@oneuptime/common 11.3.9 → 11.3.10

package/Server/Services/MutableMetricService.ts

@@ -0,0 +1,275 @@
+import AnalyticsDatabaseService, {
+  DbJSONResponse,
+  Results,
+} from "./AnalyticsDatabaseService";
+import ClickhouseDatabase from "../Infrastructure/ClickhouseDatabase";
+import { SQL, Statement } from "../Utils/AnalyticsDatabase/Statement";
+import AlertMetricType from "../../Types/Alerts/AlertMetricType";
+import AnalyticsTableName from "../../Types/AnalyticsDatabase/AnalyticsTableName";
+import TableColumnType from "../../Types/AnalyticsDatabase/TableColumnType";
+import IncidentMetricType from "../../Types/Incident/IncidentMetricType";
+import { JSONObject } from "../../Types/JSON";
+import ObjectID from "../../Types/ObjectID";
+import ServiceType from "../../Types/Telemetry/ServiceType";
+import MutableMetric from "../../Models/AnalyticsModels/MutableMetric";
+import { MetricPointType } from "../../Models/AnalyticsModels/Metric";
+import OneUptimeDate from "../../Types/Date";
+import Includes from "../../Types/BaseDatabase/Includes";
+
+type MutableMetricPointIdentity = {
+  name: string;
+  metricPointId: string;
+};
+
+export class MutableMetricService extends AnalyticsDatabaseService<MutableMetric> {
+  private static readonly mutableMetricNames: Set<string> = new Set<string>([
+    ...Object.values(AlertMetricType),
+    ...Object.values(IncidentMetricType),
+  ]);
+
+  public constructor(clickhouseDatabase?: ClickhouseDatabase | undefined) {
+    super({ modelType: MutableMetric, database: clickhouseDatabase });
+  }
+
+  public static isMutableMetricName(metricName: string | undefined): boolean {
+    if (!metricName) {
+      return false;
+    }
+
+    return MutableMetricService.mutableMetricNames.has(metricName);
+  }
+
+  public static getMutableMetricNames(): Array<string> {
+    return Array.from(MutableMetricService.mutableMetricNames);
+  }
+
+  public async createMutableMetrics(data: {
+    metrics: Array<MutableMetric>;
+  }): Promise<void> {
+    if (data.metrics.length === 0) {
+      return;
+    }
+
+    const version: number = this.getNextVersion();
+
+    for (const metric of data.metrics) {
+      metric.version = metric.version || version;
+      metric.isDeleted = metric.isDeleted || false;
+    }
+
+    await this.createMany({
+      items: data.metrics,
+      props: {
+        isRoot: true,
+      },
+    });
+  }
+
+  public async replaceEntityMetrics(data: {
+    projectId: ObjectID;
+    primaryEntityId: ObjectID;
+    primaryEntityType: ServiceType;
+    metricNames: Array<string>;
+    metrics: Array<MutableMetric>;
+    retentionDate: Date;
+  }): Promise<void> {
+    const version: number = this.getNextVersion();
+    const desiredMetricKeys: Set<string> = new Set<string>();
+
+    for (const metric of data.metrics) {
+      const metricName: string | undefined = metric.name;
+      const metricPointId: string | undefined = metric.metricPointId;
+
+      if (!metricName || !metricPointId) {
+        continue;
+      }
+
+      desiredMetricKeys.add(
+        this.getMetricPointKey({
+          name: metricName,
+          metricPointId: metricPointId,
+        }),
+      );
+      metric.version = version;
+      metric.isDeleted = false;
+    }
+
+    const existingMetricPointIdentities: Array<MutableMetricPointIdentity> =
+      await this.getActiveMetricPointIdentities({
+        projectId: data.projectId,
+        primaryEntityId: data.primaryEntityId,
+        primaryEntityType: data.primaryEntityType,
+        metricNames: data.metricNames,
+      });
+
+    const tombstoneMetrics: Array<MutableMetric> = [];
+
+    for (const existingMetricPointIdentity of existingMetricPointIdentities) {
+      const metricPointKey: string = this.getMetricPointKey(
+        existingMetricPointIdentity,
+      );
+
+      if (desiredMetricKeys.has(metricPointKey)) {
+        continue;
+      }
+
+      tombstoneMetrics.push(
+        this.createTombstoneMetric({
+          projectId: data.projectId,
+          primaryEntityId: data.primaryEntityId,
+          primaryEntityType: data.primaryEntityType,
+          name: existingMetricPointIdentity.name,
+          metricPointId: existingMetricPointIdentity.metricPointId,
+          version: version,
+          retentionDate: data.retentionDate,
+        }),
+      );
+    }
+
+    await this.createMutableMetrics({
+      metrics: [...data.metrics, ...tombstoneMetrics],
+    });
+  }
+
+  public async tombstoneEntityMetrics(data: {
+    projectId: ObjectID;
+    primaryEntityId: ObjectID;
+    primaryEntityType: ServiceType;
+    metricNames: Array<string>;
+    retentionDate: Date;
+  }): Promise<void> {
+    const existingMetricPointIdentities: Array<MutableMetricPointIdentity> =
+      await this.getActiveMetricPointIdentities({
+        projectId: data.projectId,
+        primaryEntityId: data.primaryEntityId,
+        primaryEntityType: data.primaryEntityType,
+        metricNames: data.metricNames,
+      });
+
+    const version: number = this.getNextVersion();
+    const tombstoneMetrics: Array<MutableMetric> =
+      existingMetricPointIdentities.map(
+        (
+          existingMetricPointIdentity: MutableMetricPointIdentity,
+        ): MutableMetric => {
+          return this.createTombstoneMetric({
+            projectId: data.projectId,
+            primaryEntityId: data.primaryEntityId,
+            primaryEntityType: data.primaryEntityType,
+            name: existingMetricPointIdentity.name,
+            metricPointId: existingMetricPointIdentity.metricPointId,
+            version: version,
+            retentionDate: data.retentionDate,
+          });
+        },
+      );
+
+    await this.createMutableMetrics({
+      metrics: tombstoneMetrics,
+    });
+  }
+
+  private async getActiveMetricPointIdentities(data: {
+    projectId: ObjectID;
+    primaryEntityId: ObjectID;
+    primaryEntityType: ServiceType;
+    metricNames: Array<string>;
+  }): Promise<Array<MutableMetricPointIdentity>> {
+    if (data.metricNames.length === 0) {
+      return [];
+    }
+
+    if (!this.database) {
+      this.useDefaultDatabase();
+    }
+
+    const databaseName: string = this.database.getDatasourceOptions().database!;
+    const tableName: string = AnalyticsTableName.MutableMetric;
+
+    const statement: Statement = SQL`
+      SELECT name, metricPointId
+      FROM (
+        SELECT
+          name,
+          metricPointId,
+          argMax(isDeleted, version) AS isDeleted
+        FROM ${databaseName}.${tableName}
+        WHERE projectId = ${{
+          value: data.projectId,
+          type: TableColumnType.ObjectID,
+        }}
+        AND primaryEntityId = ${{
+          value: data.primaryEntityId,
+          type: TableColumnType.ObjectID,
+        }}
+        AND primaryEntityType = ${{
+          value: data.primaryEntityType,
+          type: TableColumnType.Text,
+        }}
+        AND name IN ${{
+          value: new Includes(data.metricNames),
+          type: TableColumnType.Text,
+        }}
+        GROUP BY name, metricPointId
+      )
+      WHERE isDeleted = false
+    `;
+
+    const dbResult: Results = await this.executeQuery(statement);
+    const response: DbJSONResponse = await dbResult.json<{
+      data?: Array<JSONObject>;
+    }>();
+
+    return (response.data || []).map(
+      (row: JSONObject): MutableMetricPointIdentity => {
+        return {
+          name: row["name"] as string,
+          metricPointId: row["metricPointId"] as string,
+        };
+      },
+    );
+  }
+
+  private createTombstoneMetric(data: {
+    projectId: ObjectID;
+    primaryEntityId: ObjectID;
+    primaryEntityType: ServiceType;
+    name: string;
+    metricPointId: string;
+    version: number;
+    retentionDate: Date;
+  }): MutableMetric {
+    const now: Date = OneUptimeDate.getCurrentDate();
+    const metric: MutableMetric = new MutableMetric();
+
+    metric.projectId = data.projectId;
+    metric.primaryEntityId = data.primaryEntityId;
+    metric.primaryEntityType = data.primaryEntityType;
+    metric.name = data.name;
+    metric.metricPointId = data.metricPointId;
+    metric.value = 0;
+    metric.time = now;
+    metric.timeUnixNano = OneUptimeDate.toUnixNano(now);
+    metric.metricPointType = MetricPointType.Sum;
+    metric.attributes = {};
+    metric.attributeKeys = [];
+    metric.version = data.version;
+    metric.isDeleted = true;
+    metric.retentionDate = data.retentionDate;
+
+    return metric;
+  }
+
+  private getMetricPointKey(data: {
+    name: string;
+    metricPointId: string;
+  }): string {
+    return `${data.name}|${data.metricPointId}`;
+  }
+
+  private getNextVersion(): number {
+    return Date.now() * 1000 + Number(process.hrtime.bigint() % BigInt(1000));
+  }
+}
+
+export default new MutableMetricService();

package/Server/Services/TelemetryAttributeService.ts

@@ -4,6 +4,9 @@ import LogDatabaseService from "./LogService";
 import MetricDatabaseService from "./MetricService";
 import SpanDatabaseService from "./SpanService";
 import ExceptionInstanceService from "./ExceptionInstanceService";
+import MutableMetricDatabaseService, {
+  MutableMetricService as MutableMetricServiceClass,
+} from "./MutableMetricService";
 import TableColumnType from "../../Types/AnalyticsDatabase/TableColumnType";
 import { JSONObject } from "../../Types/JSON";
 import ObjectID from "../../Types/ObjectID";
@@ -25,6 +28,7 @@ type TelemetrySource = {
    */
   attributeKeysColumn?: string | undefined;
   timeColumn: string;
+  isMutableMetricSource?: boolean | undefined;
 };
 
 type TelemetryAttributesCacheEntry = {
@@ -51,6 +55,7 @@ export class TelemetryAttributeService {
 
   private getTelemetrySource(
     telemetryType: TelemetryType,
+    metricName?: string | undefined,
   ): TelemetrySource | null {
     switch (telemetryType) {
       case TelemetryType.Log:
@@ -62,6 +67,17 @@ export class TelemetryAttributeService {
           timeColumn: "time",
         };
       case TelemetryType.Metric:
+        if (MutableMetricServiceClass.isMutableMetricName(metricName)) {
+          return {
+            service: MutableMetricDatabaseService,
+            tableName: MutableMetricDatabaseService.model.tableName,
+            attributesColumn: "attributes",
+            attributeKeysColumn: "attributeKeys",
+            timeColumn: "time",
+            isMutableMetricSource: true,
+          };
+        }
+
         return {
           service: MetricDatabaseService,
           tableName: MetricDatabaseService.model.tableName,
@@ -98,6 +114,7 @@ export class TelemetryAttributeService {
   }): Promise<string[]> {
     const source: TelemetrySource | null = this.getTelemetrySource(
       data.telemetryType,
+      data.metricName,
     );
 
     if (!source) {
@@ -108,6 +125,7 @@ export class TelemetryAttributeService {
       data.projectId,
       data.telemetryType,
       data.metricName,
+      source.tableName,
     );
 
     const cachedEntry: TelemetryAttributesCacheEntry | null =
@@ -149,8 +167,11 @@ export class TelemetryAttributeService {
     projectId: ObjectID,
     telemetryType: TelemetryType,
     metricName?: string | undefined,
+    sourceTableName?: string | undefined,
   ): string {
-    const base: string = `${projectId.toString()}:${telemetryType}`;
+    const base: string = `${projectId.toString()}:${telemetryType}:${
+      sourceTableName || "default"
+    }`;
     if (metricName) {
       return `${base}:${metricName}`;
     }
@@ -251,10 +272,23 @@ export class TelemetryAttributeService {
     attributeKeysColumn?: string | undefined;
     timeColumn: string;
     metricName?: string | undefined;
+    isMutableMetricSource?: boolean | undefined;
   }): Statement {
     const lookbackStartDate: Date =
       TelemetryAttributeService.getLookbackStartDate();
 
+    if (data.isMutableMetricSource) {
+      return TelemetryAttributeService.buildMutableMetricAttributesStatement({
+        projectId: data.projectId,
+        tableName: data.tableName,
+        attributesColumn: data.attributesColumn,
+        attributeKeysColumn: data.attributeKeysColumn,
+        timeColumn: data.timeColumn,
+        metricName: data.metricName,
+        lookbackStartDate,
+      });
+    }
+
     /*
      * Two notable choices here:
      *
@@ -332,6 +366,7 @@ export class TelemetryAttributeService {
         attributeKeysColumn: data.source.attributeKeysColumn,
         timeColumn: data.source.timeColumn,
         metricName: data.metricName,
+        isMutableMetricSource: data.source.isMutableMetricSource,
       });
 
     const dbResult: Results = await data.source.service.executeQuery(statement);
@@ -377,6 +412,7 @@ export class TelemetryAttributeService {
   }): Promise<string[]> {
     const source: TelemetrySource | null = this.getTelemetrySource(
       data.telemetryType,
+      data.metricName,
     );
 
     if (!source) {
@@ -402,6 +438,19 @@ export class TelemetryAttributeService {
     const lookbackStartDate: Date =
       TelemetryAttributeService.getLookbackStartDate();
 
+    if (data.source.isMutableMetricSource) {
+      return TelemetryAttributeService.buildMutableMetricAttributeValuesStatement(
+        {
+          projectId: data.projectId,
+          source: data.source,
+          metricName: data.metricName,
+          attributeKey: data.attributeKey,
+          searchText: data.searchText,
+          lookbackStartDate,
+        },
+      );
+    }
+
     const statement: Statement = SQL`
       SELECT DISTINCT ${data.source.attributesColumn}[${{
         type: TableColumnType.Text,
@@ -499,6 +548,139 @@ export class TelemetryAttributeService {
         return Boolean(val);
       });
   }
+
+  private static buildMutableMetricAttributesStatement(data: {
+    projectId: ObjectID;
+    tableName: string;
+    attributesColumn: string;
+    attributeKeysColumn?: string | undefined;
+    timeColumn: string;
+    metricName?: string | undefined;
+    lookbackStartDate: Date;
+  }): Statement {
+    const attributeKeysColumn: string =
+      data.attributeKeysColumn || "attributeKeys";
+
+    const statement: Statement = SQL`
+      SELECT arrayDistinct(arrayFlatten(groupUniqArrayArray(${attributeKeysColumn}))) AS keys
+      FROM (
+        SELECT
+          argMax(${data.timeColumn}, version) AS ${data.timeColumn},
+          argMax(${data.attributesColumn}, version) AS ${data.attributesColumn},
+          argMax(${attributeKeysColumn}, version) AS ${attributeKeysColumn},
+          argMax(retentionDate, version) AS retentionDate,
+          argMax(isDeleted, version) AS isDeleted
+        FROM ${data.tableName}
+        WHERE projectId = ${{
+          type: TableColumnType.ObjectID,
+          value: data.projectId,
+        }}`;
+
+    if (data.metricName) {
+      statement.append(
+        SQL`
+        AND name = ${{
+          type: TableColumnType.Text,
+          value: data.metricName,
+        }}`,
+      );
+    }
+
+    statement.append(SQL`
+        GROUP BY projectId, name, primaryEntityId, primaryEntityType, metricPointId
+      )
+      WHERE isDeleted = false
+        AND retentionDate >= now()
+        AND ${data.timeColumn} >= ${{
+          type: TableColumnType.Date,
+          value: data.lookbackStartDate,
+        }}
+        AND NOT empty(${attributeKeysColumn})`);
+
+    statement.append(
+      " SETTINGS max_execution_time = 45, timeout_overflow_mode = 'break'",
+    );
+
+    return statement;
+  }
+
+  private static buildMutableMetricAttributeValuesStatement(data: {
+    projectId: ObjectID;
+    source: TelemetrySource;
+    metricName?: string | undefined;
+    attributeKey: string;
+    searchText?: string | undefined;
+    lookbackStartDate: Date;
+  }): Statement {
+    const statement: Statement = SQL`
+      SELECT DISTINCT ${data.source.attributesColumn}[${{
+        type: TableColumnType.Text,
+        value: data.attributeKey,
+      }}] AS attributeValue
+      FROM (
+        SELECT
+          argMax(${data.source.timeColumn}, version) AS ${data.source.timeColumn},
+          argMax(${data.source.attributesColumn}, version) AS ${data.source.attributesColumn},
+          argMax(retentionDate, version) AS retentionDate,
+          argMax(isDeleted, version) AS isDeleted
+        FROM ${data.source.tableName}
+        WHERE projectId = ${{
+          type: TableColumnType.ObjectID,
+          value: data.projectId,
+        }}`;
+
+    if (data.metricName) {
+      statement.append(
+        SQL`
+        AND name = ${{
+          type: TableColumnType.Text,
+          value: data.metricName,
+        }}`,
+      );
+    }
+
+    statement.append(SQL`
+        GROUP BY projectId, name, primaryEntityId, primaryEntityType, metricPointId
+      )
+      WHERE isDeleted = false
+        AND retentionDate >= now()
+        AND ${data.source.timeColumn} >= ${{
+          type: TableColumnType.Date,
+          value: data.lookbackStartDate,
+        }}
+        AND mapContains(${data.source.attributesColumn}, ${{
+          type: TableColumnType.Text,
+          value: data.attributeKey,
+        }})`);
+
+    if (data.searchText && data.searchText.trim().length > 0) {
+      statement.append(
+        SQL`
+        AND ${data.source.attributesColumn}[${{
+          type: TableColumnType.Text,
+          value: data.attributeKey,
+        }}] ILIKE ${{
+          type: TableColumnType.Text,
+          value: `%${data.searchText.trim()}%`,
+        }}`,
+      );
+    }
+
+    statement.append(
+      SQL`
+      ORDER BY attributeValue ASC
+      LIMIT ${{
+        type: TableColumnType.Number,
+        value: TelemetryAttributeService.ATTRIBUTE_VALUES_LIMIT,
+      }}`,
+    );
+
+    statement.append(
+      " SETTINGS max_execution_time = 45, timeout_overflow_mode = 'break'",
+    );
+
+    return statement;
+  }
 }
 
 export default new TelemetryAttributeService();

package/Server/Utils/AnalyticsDatabase/ClusterConfig.ts

@@ -82,6 +82,8 @@ export function getStorageEngine(engine: AnalyticsTableEngine): string {
   switch (engine) {
     case AnalyticsTableEngine.AggregatingMergeTree:
       return "ReplicatedAggregatingMergeTree";
+    case AnalyticsTableEngine.ReplacingMergeTree:
+      return "ReplicatedReplacingMergeTree(version)";
     case AnalyticsTableEngine.MergeTree:
     default:
       return "ReplicatedMergeTree";

package/Server/Utils/AnalyticsDatabase/Statement.ts

@@ -194,6 +194,8 @@ export class Statement implements BaseQueryParams {
       [TableColumnType.ArrayNumber]: "Array(Int32)",
       [TableColumnType.ArrayText]: "Array(String)",
       [TableColumnType.LongNumber]: "Int128",
+      [TableColumnType.BigNumber]: "Int64",
+      [TableColumnType.UInt64]: "UInt64",
     };
 
     if ((statementParam as StatementParameter).value instanceof Includes) {