@oneuptime/common 12.0.15 → 12.0.17

package/Server/Services/LogCountBaselineService.ts

@@ -0,0 +1,163 @@
+import AnalyticsDatabaseService from "./AnalyticsDatabaseService";
+import ClickhouseDatabase from "../Infrastructure/ClickhouseDatabase";
+import LogCountBaseline from "../../Models/AnalyticsModels/LogCountBaseline";
+import { MetricBaselineService } from "./MetricBaselineService";
+import CountAnomaly, {
+  CountBaselineStats,
+  CountBaselineSummary,
+} from "../Utils/Monitor/Criteria/CountAnomaly";
+import logger, { LogAttributes } from "../Utils/Logger";
+import ObjectID from "../../Types/ObjectID";
+
+/**
+ * Read-side service for the `LogCountBaseline` MV target table — the
+ * log-volume peer of `SpanCountBaselineService` (see that service and
+ * `MetricBaselineService` for the shared design notes).
+ *
+ * `getBaseline(...)` fetches the per-minute log counts of one
+ * hour-of-week cell over the rolling window and computes the stats in
+ * app code via {@link CountAnomaly.computeStats}. Window/minSamples
+ * defaults and caps are shared with MetricBaselineService so every
+ * anomaly criterion behaves alike.
+ */
+export class LogCountBaselineService extends AnalyticsDatabaseService<LogCountBaseline> {
+  public constructor(clickhouseDatabase?: ClickhouseDatabase | undefined) {
+    super({ modelType: LogCountBaseline, database: clickhouseDatabase });
+  }
+
+  /**
+   * Fetch the rolling-window baseline of logs-per-minute for one
+   * (project, [services], [severities], hour-of-week) cell. Returns
+   * null when no data exists.
+   *
+   * `telemetryServiceIds` empty/omitted aggregates across every service
+   * in the project; `severityTexts` empty/omitted aggregates across
+   * every severity — matching how the Logs monitor query itself
+   * broadens when those filters are unset.
+   */
+  public async getBaseline(input: {
+    projectId: ObjectID | string;
+    telemetryServiceIds?: Array<ObjectID | string> | undefined;
+    severityTexts?: Array<string> | undefined;
+    hourOfWeek: number;
+    windowDays?: number | undefined;
+    minSamples?: number | undefined;
+  }): Promise<CountBaselineSummary | null> {
+    const windowDays: number = Math.min(
+      input.windowDays || MetricBaselineService.DEFAULT_WINDOW_DAYS,
+      MetricBaselineService.MAX_WINDOW_DAYS,
+    );
+    const minSamples: number =
+      input.minSamples || MetricBaselineService.DEFAULT_MIN_SAMPLES;
+
+    const projectIdStr: string = this.escapeString(
+      input.projectId instanceof ObjectID
+        ? input.projectId.toString()
+        : input.projectId,
+    );
+    const hour: number = Math.max(
+      0,
+      Math.min(167, Math.floor(input.hourOfWeek)),
+    );
+
+    const serviceIds: Array<string> = (input.telemetryServiceIds || []).map(
+      (id: ObjectID | string) => {
+        return this.escapeString(id instanceof ObjectID ? id.toString() : id);
+      },
+    );
+    const serviceIdClause: string =
+      serviceIds.length > 0
+        ? `AND primaryEntityId IN (${serviceIds
+            .map((id: string) => {
+              return `'${id}'`;
+            })
+            .join(", ")})`
+        : "";
+
+    const severities: Array<string> = (input.severityTexts || [])
+      .filter((severity: string) => {
+        return Boolean(severity);
+      })
+      .map((severity: string) => {
+        return this.escapeString(severity);
+      });
+    const severityClause: string =
+      severities.length > 0
+        ? `AND severityText IN (${severities
+            .map((severity: string) => {
+              return `'${severity}'`;
+            })
+            .join(", ")})`
+        : "";
+
+    /*
+     * One row per non-empty minute cell; each row's merged count is a
+     * raw baseline sample ("logs in that minute"). GROUP BY day +
+     * minuteOfHour also folds across services/severities when the IN
+     * clauses are broad, so a multi-service monitor baselines its
+     * combined volume — matching what its observed logCount measures.
+     */
+    const sql: string = `
+      SELECT
+        countMerge(logCountState) AS c
+      FROM LogCountBaseline
+      WHERE projectId = '${projectIdStr}'
+        ${serviceIdClause}
+        ${severityClause}
+        AND hourOfWeek = ${hour}
+        AND day >= today() - INTERVAL ${windowDays} DAY
+      GROUP BY day, minuteOfHour
+    `;
+
+    const resultSet: {
+      json: () => Promise<{ data: Array<{ c: number | string }> }>;
+    } = (await this.executeQuery(sql)) as unknown as {
+      json: () => Promise<{ data: Array<{ c: number | string }> }>;
+    };
+
+    const parsed: { data: Array<{ c: number | string }> } =
+      await resultSet.json();
+
+    const counts: Array<number> = parsed.data.map(
+      (row: { c: number | string }) => {
+        return this.toNumber(row.c);
+      },
+    );
+
+    const stats: CountBaselineStats | null = CountAnomaly.computeStats(counts);
+    if (!stats) {
+      return null;
+    }
+
+    const summary: CountBaselineSummary = {
+      ...stats,
+      isReliable: stats.sampleCount >= minSamples,
+      windowDays,
+      hourOfWeek: hour,
+    };
+
+    logger.debug("LogCountBaselineService.getBaseline", {
+      projectId: projectIdStr,
+      hourOfWeek: hour,
+      sampleCount: summary.sampleCount,
+      isReliable: summary.isReliable,
+    } as LogAttributes);
+
+    return summary;
+  }
+
+  private toNumber(v: number | string | undefined): number {
+    if (typeof v === "number") {
+      return v;
+    }
+    const n: number = Number(v);
+    return Number.isFinite(n) ? n : 0;
+  }
+
+  private escapeString(v: string): string {
+    // ClickHouse string-literal escape for backslash and single quote.
+    return v.replace(/\\/g, "\\\\").replace(/'/g, "\\'");
+  }
+}
+
+export default new LogCountBaselineService();

package/Server/Services/MutableMetricService.ts

@@ -4,10 +4,8 @@ import AnalyticsDatabaseService, {
 } from "./AnalyticsDatabaseService";
 import ClickhouseDatabase from "../Infrastructure/ClickhouseDatabase";
 import { SQL, Statement } from "../Utils/AnalyticsDatabase/Statement";
-import AlertMetricType from "../../Types/Alerts/AlertMetricType";
 import AnalyticsTableName from "../../Types/AnalyticsDatabase/AnalyticsTableName";
 import TableColumnType from "../../Types/AnalyticsDatabase/TableColumnType";
-import IncidentMetricType from "../../Types/Incident/IncidentMetricType";
 import { JSONObject } from "../../Types/JSON";
 import ObjectID from "../../Types/ObjectID";
 import ServiceType from "../../Types/Telemetry/ServiceType";
@@ -22,10 +20,29 @@ type MutableMetricPointIdentity = {
 };
 
 export class MutableMetricService extends AnalyticsDatabaseService<MutableMetric> {
-  private static readonly mutableMetricNames: Set<string> = new Set<string>([
-    ...Object.values(AlertMetricType),
-    ...Object.values(IncidentMetricType),
-  ]);
+  /*
+   * Reserved namespaces whose metrics live in the mutable table.
+   *
+   * This used to be a Set built at module load from the two hard-coded
+   * metric-type enums, which made user-defined metric names impossible: a
+   * name the Set did not contain was silently routed to the immutable Metric
+   * table by all three consumers below and returned nothing -- an empty
+   * chart, never an error. A prefix test keeps the routing decision
+   * synchronous and cache-free (these are hot read paths) while letting a
+   * project name its own measurements.
+   *
+   * Consumers: MetricService.tryBuildMutableMetricAggregateStatement,
+   * MetricAggregationService facet-source selection, and
+   * TelemetryAttributeService attribute autocomplete.
+   *
+   * The cost of this is that `oneuptime.` is now a reserved prefix that
+   * user-supplied metric names may not claim.
+   */
+  private static readonly mutableMetricNamePrefixes: ReadonlyArray<string> = [
+    "oneuptime.incident.",
+    "oneuptime.alert.",
+    "oneuptime.scheduled-maintenance.",
+  ];
 
   public constructor(clickhouseDatabase?: ClickhouseDatabase | undefined) {
     super({ modelType: MutableMetric, database: clickhouseDatabase });
@@ -36,11 +53,15 @@ export class MutableMetricService extends AnalyticsDatabaseService<MutableMetric
       return false;
     }
 
-    return MutableMetricService.mutableMetricNames.has(metricName);
+    return MutableMetricService.mutableMetricNamePrefixes.some(
+      (prefix: string) => {
+        return metricName.startsWith(prefix);
+      },
+    );
   }
 
-  public static getMutableMetricNames(): Array<string> {
-    return Array.from(MutableMetricService.mutableMetricNames);
+  public static getMutableMetricNamePrefixes(): ReadonlyArray<string> {
+    return MutableMetricService.mutableMetricNamePrefixes;
   }
 
   public async createMutableMetrics(data: {

package/Server/Services/ScheduledMaintenanceMeasurementService.ts

@@ -0,0 +1,311 @@
+import CreateBy from "../Types/Database/CreateBy";
+import UpdateBy from "../Types/Database/UpdateBy";
+import DeleteBy from "../Types/Database/DeleteBy";
+import { OnCreate, OnUpdate, OnDelete } from "../Types/Database/Hooks";
+import SortOrder from "../../Types/BaseDatabase/SortOrder";
+import DatabaseService from "./DatabaseService";
+import Model from "../../Models/DatabaseModels/ScheduledMaintenanceMeasurement";
+import ScheduledMaintenanceState from "../../Models/DatabaseModels/ScheduledMaintenanceState";
+import ScheduledMaintenanceStateService from "./ScheduledMaintenanceStateService";
+import ScheduledMaintenanceMeasurementAnchorType from "../../Types/ScheduledMaintenance/ScheduledMaintenanceMeasurementAnchorType";
+import MeasurementDefinitionValidator from "../Utils/Measurement/MeasurementDefinitionValidator";
+import BadDataException from "../../Types/Exception/BadDataException";
+import ObjectID from "../../Types/ObjectID";
+import OneUptimeDate from "../../Types/Date";
+import { LIMIT_PER_PROJECT } from "../../Types/Database/LimitMax";
+import CaptureSpan from "../Utils/Telemetry/CaptureSpan";
+
+/*
+ * The three backfill columns onBeforeUpdate rewrites, named so they are still
+ * checked against the model. Spelt out rather than reached through
+ * Partial<Model>: this model carries enough relations that instantiating a
+ * mapped type over all of it blows tsc's instantiation depth limit.
+ */
+interface BackfillFields {
+  backfillRequestedAt?: Date | undefined;
+  backfillCursorCreatedAt?: Date | undefined;
+  backfillCompletedAt?: Date | undefined;
+}
+
+/*
+ * Which column each timestamp anchor actually reads. Several anchors are
+ * aliases for the same instant, so the validator compares these rather than
+ * the enum values -- otherwise a definition like "Created At -> Timeline
+ * Start" would be accepted and report a constant zero on every entity.
+ */
+const TIMESTAMP_ANCHOR_SOURCES: Record<string, string> = {
+  [ScheduledMaintenanceMeasurementAnchorType.TimelineStart]: "createdAt",
+  [ScheduledMaintenanceMeasurementAnchorType.CreatedAt]: "createdAt",
+  [ScheduledMaintenanceMeasurementAnchorType.ScheduledStartsAt]: "startsAt",
+  [ScheduledMaintenanceMeasurementAnchorType.ScheduledEndsAt]: "endsAt",
+};
+
+export class Service extends DatabaseService<Model> {
+  public static readonly METRIC_NAME_PREFIX: string =
+    "oneuptime.scheduled-maintenance.measurement.";
+
+  public constructor() {
+    super(Model);
+  }
+
+  @CaptureSpan()
+  protected override async onBeforeCreate(
+    createBy: CreateBy<Model>,
+  ): Promise<OnCreate<Model>> {
+    MeasurementDefinitionValidator.validateKey(createBy.data.key);
+
+    /*
+     * Derived here rather than from the slug: DatabaseService generates the
+     * slug AFTER onBeforeCreate runs, and Slug.getSlug appends ten random
+     * digits, so a slug-derived metric name would be both unavailable at
+     * this point and unreadable if it were.
+     */
+    createBy.data.metricName = Service.METRIC_NAME_PREFIX + createBy.data.key;
+
+    await this.validateAnchors({
+      projectId: createBy.data.projectId!,
+      startAnchorType: createBy.data.startAnchorType,
+      endAnchorType: createBy.data.endAnchorType,
+      startStateId: createBy.data.startScheduledMaintenanceStateId,
+      endStateId: createBy.data.endScheduledMaintenanceStateId,
+      startStateRole: createBy.data.startScheduledMaintenanceStateRole,
+      endStateRole: createBy.data.endScheduledMaintenanceStateRole,
+      startOccurrence: createBy.data.startStateOccurrence,
+      endOccurrence: createBy.data.endStateOccurrence,
+    });
+
+    if (!createBy.data.order) {
+      const highest: Model | null = await this.findOneBy({
+        query: { projectId: createBy.data.projectId! },
+        select: { order: true },
+        sort: { order: SortOrder.Descending },
+        props: { isRoot: true },
+      });
+
+      createBy.data.order = (highest?.order || 0) + 1;
+    }
+
+    /*
+     * A definition created today must apply to maintenance events that have
+     * already run, or the page it appears on is empty for every completed
+     * event forever. The backfill worker picks this stamp up.
+     */
+    createBy.data.backfillRequestedAt = OneUptimeDate.getCurrentDate();
+
+    return { createBy, carryForward: null };
+  }
+
+  @CaptureSpan()
+  protected override async onBeforeUpdate(
+    updateBy: UpdateBy<Model>,
+  ): Promise<OnUpdate<Model>> {
+    /*
+     * Read as a bag of keys rather than as Partial<Model>: this model carries
+     * enough relations that checking assignability against it blows the
+     * instantiation depth limit, and the two uses below -- key presence and a
+     * spread into `merged` -- need nothing more than this.
+     */
+    const data: Record<string, unknown> = updateBy.data as Record<
+      string,
+      unknown
+    >;
+
+    const definitionKeys: Array<string> = [
+      "startAnchorType",
+      "endAnchorType",
+      "startScheduledMaintenanceStateId",
+      "endScheduledMaintenanceStateId",
+      "startScheduledMaintenanceStateRole",
+      "endScheduledMaintenanceStateRole",
+      "startStateOccurrence",
+      "endStateOccurrence",
+      "isEnabled",
+    ];
+
+    const touchesDefinition: boolean = definitionKeys.some((key: string) => {
+      return Object.prototype.hasOwnProperty.call(data, key);
+    });
+
+    if (!touchesDefinition) {
+      return { updateBy, carryForward: null };
+    }
+
+    const existingItems: Array<Model> = await this.findBy({
+      query: updateBy.query,
+      select: {
+        projectId: true,
+        startAnchorType: true,
+        endAnchorType: true,
+        startScheduledMaintenanceStateId: true,
+        endScheduledMaintenanceStateId: true,
+        startScheduledMaintenanceStateRole: true,
+        endScheduledMaintenanceStateRole: true,
+        startStateOccurrence: true,
+        endStateOccurrence: true,
+      },
+      limit: LIMIT_PER_PROJECT,
+      skip: 0,
+      props: { isRoot: true },
+    });
+
+    for (const existing of existingItems) {
+      const merged: Record<string, unknown> = {
+        ...existing,
+        ...data,
+      } as Record<string, unknown>;
+
+      await this.validateAnchors({
+        projectId: existing.projectId!,
+        startAnchorType: merged[
+          "startAnchorType"
+        ] as ScheduledMaintenanceMeasurementAnchorType,
+        endAnchorType: merged[
+          "endAnchorType"
+        ] as ScheduledMaintenanceMeasurementAnchorType,
+        startStateId: merged["startScheduledMaintenanceStateId"] as ObjectID,
+        endStateId: merged["endScheduledMaintenanceStateId"] as ObjectID,
+        startStateRole: merged["startScheduledMaintenanceStateRole"] as string,
+        endStateRole: merged["endScheduledMaintenanceStateRole"] as string,
+        startOccurrence: merged["startStateOccurrence"] as string,
+        endOccurrence: merged["endStateOccurrence"] as string,
+      });
+    }
+
+    /*
+     * Changing what a measurement means rewrites its history in place, under
+     * the same metric point identity -- one series, one current definition.
+     * Keeping the old numbers means creating a new definition instead, which
+     * the settings page says.
+     */
+    const backfillFields: BackfillFields =
+      updateBy.data as unknown as BackfillFields;
+
+    backfillFields.backfillRequestedAt = OneUptimeDate.getCurrentDate();
+    // Cleared, not left stale: the backfill must restart from the beginning.
+    backfillFields.backfillCursorCreatedAt = null as unknown as Date;
+    backfillFields.backfillCompletedAt = null as unknown as Date;
+
+    return { updateBy, carryForward: null };
+  }
+
+  @CaptureSpan()
+  protected override async onBeforeDelete(
+    deleteBy: DeleteBy<Model>,
+  ): Promise<OnDelete<Model>> {
+    const items: Array<Model> = await this.findBy({
+      query: deleteBy.query,
+      select: { isSystemDefined: true, name: true },
+      limit: LIMIT_PER_PROJECT,
+      skip: 0,
+      props: { isRoot: true },
+    });
+
+    for (const item of items) {
+      if (item.isSystemDefined) {
+        throw new BadDataException(
+          `"${item.name}" is a built-in measurement and cannot be deleted. Disable it instead.`,
+        );
+      }
+    }
+
+    return { deleteBy, carryForward: null };
+  }
+
+  /*
+   * Every measurement metric name in the project, enabled or not.
+   *
+   * Disabled definitions must stay in this list: the tombstone pass diffs
+   * live metric points against the desired set scoped to these names, so a
+   * name that is omitted keeps its points live in every chart until the
+   * retention date rather than disappearing when the user disables it.
+   */
+  @CaptureSpan()
+  public async getMetricNamesForProject(
+    projectId: ObjectID,
+  ): Promise<Array<string>> {
+    const measurements: Array<Model> = await this.findBy({
+      query: { projectId: projectId },
+      select: { metricName: true },
+      limit: LIMIT_PER_PROJECT,
+      skip: 0,
+      props: { isRoot: true },
+    });
+
+    return measurements
+      .map((measurement: Model) => {
+        return measurement.metricName;
+      })
+      .filter((name: string | undefined) => {
+        return Boolean(name);
+      }) as Array<string>;
+  }
+
+  private async validateAnchors(data: {
+    projectId: ObjectID;
+    startAnchorType?: ScheduledMaintenanceMeasurementAnchorType | undefined;
+    endAnchorType?: ScheduledMaintenanceMeasurementAnchorType | undefined;
+    startStateId?: ObjectID | undefined;
+    endStateId?: ObjectID | undefined;
+    startStateRole?: string | undefined;
+    endStateRole?: string | undefined;
+    startOccurrence?: string | undefined;
+    endOccurrence?: string | undefined;
+  }): Promise<void> {
+    MeasurementDefinitionValidator.validateAnchorPair({
+      timestampAnchorSources: TIMESTAMP_ANCHOR_SOURCES,
+      startAnchorType: data.startAnchorType,
+      endAnchorType: data.endAnchorType,
+      stateEnteredAnchor:
+        ScheduledMaintenanceMeasurementAnchorType.StateEntered,
+      stateRoleEnteredAnchor:
+        ScheduledMaintenanceMeasurementAnchorType.StateRoleEntered,
+      startStateId: data.startStateId?.toString(),
+      endStateId: data.endStateId?.toString(),
+      startStateRole: data.startStateRole,
+      endStateRole: data.endStateRole,
+      startOccurrence: data.startOccurrence,
+      endOccurrence: data.endOccurrence,
+    });
+
+    if (
+      data.startAnchorType !==
+        ScheduledMaintenanceMeasurementAnchorType.StateEntered ||
+      data.endAnchorType !==
+        ScheduledMaintenanceMeasurementAnchorType.StateEntered ||
+      !data.startStateId ||
+      !data.endStateId
+    ) {
+      return;
+    }
+
+    const states: Array<ScheduledMaintenanceState> =
+      await ScheduledMaintenanceStateService.findBy({
+        query: { projectId: data.projectId },
+        select: { _id: true, name: true, order: true },
+        limit: LIMIT_PER_PROJECT,
+        skip: 0,
+        props: { isRoot: true },
+      });
+
+    const startState: ScheduledMaintenanceState | undefined = states.find(
+      (state: ScheduledMaintenanceState) => {
+        return state._id?.toString() === data.startStateId!.toString();
+      },
+    );
+
+    const endState: ScheduledMaintenanceState | undefined = states.find(
+      (state: ScheduledMaintenanceState) => {
+        return state._id?.toString() === data.endStateId!.toString();
+      },
+    );
+
+    MeasurementDefinitionValidator.validateStateOrder({
+      startStateName: startState?.name,
+      startStateOrder: startState?.order,
+      endStateName: endState?.name,
+      endStateOrder: endState?.order,
+    });
+  }
+}
+
+export default new Service();