@oneuptime/common 13.0.5 → 13.0.7

package/Server/Utils/Slo/SloLegacyMonitorLabelAdoption.ts

@@ -0,0 +1,173 @@
+import PostgresAppInstance, {
+  DatabaseSource,
+} from "../../Infrastructure/PostgresDatabase";
+import {
+  MIGRATED_MONITOR_RULE_DESCRIPTION,
+  MIGRATED_MONITOR_RULE_NAME,
+} from "../../Infrastructure/Postgres/SchemaMigrations/1793200000000-BackfillSloMonitorRulesAndAffectedResources";
+import DatabaseNotConnectedException from "../../../Types/Exception/DatabaseNotConnectedException";
+import ObjectID from "../../../Types/ObjectID";
+import { EntityManager } from "typeorm";
+
+/*
+ * Turns an SLO's deprecated "Auto-Add Monitors With Labels" list into a real
+ * SLO Monitor Rule, for SLOs the one-shot backfill could not have seen.
+ *
+ * WHY THIS EXISTS. Migration 1793200000000 converts every label list into a
+ * rule once, when the migrate Job runs. The label table itself is kept so API
+ * pods from the previous release keep working during a rolling deploy - and
+ * those pods keep WRITING it: their create and edit forms still ask for
+ * auto-add labels, and their engine attaches the matching monitors as
+ * autoAddedMonitors. A dashboard tab opened before the upgrade, an API client
+ * or a workflow can also still send the list to the NEW pods, because the
+ * column stays in the API. Anything written that way after the backfill has
+ * no rule, and the new engine reads only rules: the next sync of that SLO
+ * released every monitor the list had attached, and the SLO quietly stopped
+ * measuring the monitors its owner configured.
+ *
+ * WHY NOT A DATA MIGRATION. A Workers data migration does not run "after the
+ * deploy": the migrate Job runs the Postgres schema migrations and then the
+ * data migrations back to back, in one process. With the chart's default
+ * migrate.hook=false that happens while the pods roll; with hook=true it
+ * happens before they roll at all. Either way previous-release pods keep
+ * serving after it finishes, so a re-run there closes nothing.
+ *
+ * So the new code adopts the list itself, at each moment it would otherwise
+ * lose it:
+ *
+ *   - a monitor changed, and an SLO still records it as rule-attached but has
+ *     no enabled rule (ServiceLevelObjectiveMonitorRuleEngineService
+ *     .syncSlosForMonitor);
+ *   - somebody is creating an SLO's first monitor rule, whose sync would
+ *     otherwise run with only that rule and detach everything the list
+ *     attached (ServiceLevelObjectiveMonitorRuleService.onBeforeCreate - before
+ *     the new rule row exists, so "no rule row" still means what it says);
+ *   - the list itself was just written through the deprecated column
+ *     (ServiceLevelObjectiveService, after the create or update).
+ *
+ * WHICH SLOs ARE ADOPTED. Only those with label rows AND no monitor rule row
+ * at all - enabled or disabled - AND, unless the list was just written,
+ * monitors still recorded as rule-attached. Those conditions are what keep a
+ * rule the user deleted from coming back:
+ *
+ *   - An SLO the backfill already converted still has its label rows (they
+ *     are never deleted, for the reason below). If its user disables the
+ *     rule, a rule row still exists, so nothing is adopted and the disable
+ *     means what it says.
+ *   - If its user deletes every rule, the delete's own sync releases every
+ *     rule-attached monitor. That SLO has label rows and no rule rows - the
+ *     same as a list a previous-release pod wrote - so the rule-attached
+ *     monitors are the evidence that tells the two apart: only the legacy
+ *     engine leaves monitors attached by a list with no rule behind it. Only a
+ *     delete whose sync failed and left monitors behind can still be adopted,
+ *     and then the rule that comes back describes monitors that really are
+ *     attached.
+ *   - A list that was just written needs no such evidence: the write is the
+ *     owner saying what the SLO should measure now, and a brand-new list has
+ *     not attached anything yet. ServiceLevelObjectiveService only asks for
+ *     this when the write changed the stored list, so an out-of-date form
+ *     re-submitting the list it loaded does not bring a deleted rule back.
+ *
+ * The rule-side sync (syncMonitorsForSlo) never adopts, and neither do rule
+ * edits and deletes: after a rule delete, "no rule rows, monitors still
+ * attached" is exactly what a deliberate delete looks like, and an edited
+ * rule is itself a rule row.
+ *
+ * The label rows are left in place, like the backfill leaves them. Deleting
+ * them here would hand previous-release pods an SLO with an empty label list,
+ * and their SLO edit form re-submits that list - its sync would then release
+ * every rule-attached monitor. The rule written here makes them irrelevant to
+ * the new engine.
+ *
+ * The rule has the backfill's name and description, so it reads the same as
+ * every other converted rule. It is written in SQL, with no hooks, as the
+ * backfill is; the caller re-syncs the SLO right after, and that sync is what
+ * posts the membership feed items.
+ *
+ * Safe to run concurrently. Two monitor edits, or a monitor edit and a rule
+ * create, can reach the same SLO at once, and "no rule exists" is a
+ * read-then-write. The SLO rows are locked first (in id order, so two runs
+ * cannot deadlock), and under READ COMMITTED the adopt statement takes its
+ * snapshot after the lock is granted - so a second run sees the first run's
+ * rule and inserts nothing.
+ */
+
+export const SLO_LEGACY_MONITOR_LABEL_LOCK_STATEMENT: string = `SELECT "_id" FROM "ServiceLevelObjective" WHERE "_id" = ANY($1::uuid[]) AND "projectId" = $2 ORDER BY "_id" FOR UPDATE`;
+
+export const SLO_LEGACY_MONITOR_LABEL_ADOPT_STATEMENT: string = `WITH "adoptedRules" AS (INSERT INTO "ServiceLevelObjectiveMonitorRule" ("_id", "version", "projectId", "serviceLevelObjectiveId", "name", "description", "isEnabled") SELECT uuid_generate_v4(), 1, slo."projectId", slo."_id", $3, $4, true FROM "ServiceLevelObjective" slo WHERE slo."_id" = ANY($1::uuid[]) AND slo."projectId" = $2 AND slo."deletedAt" IS NULL AND EXISTS (SELECT 1 FROM "ServiceLevelObjectiveMonitorLabel" sml WHERE sml."serviceLevelObjectiveId" = slo."_id") AND NOT EXISTS (SELECT 1 FROM "ServiceLevelObjectiveMonitorRule" existing WHERE existing."serviceLevelObjectiveId" = slo."_id") AND (NOT $5::boolean OR EXISTS (SELECT 1 FROM "ServiceLevelObjectiveAutoAddedMonitor" attached WHERE attached."serviceLevelObjectiveId" = slo."_id")) RETURNING "_id", "serviceLevelObjectiveId"), "adoptedLabels" AS (INSERT INTO "ServiceLevelObjectiveMonitorRuleMonitorLabel" ("serviceLevelObjectiveMonitorRuleId", "labelId") SELECT "adoptedRules"."_id", sml."labelId" FROM "adoptedRules" INNER JOIN "ServiceLevelObjectiveMonitorLabel" sml ON sml."serviceLevelObjectiveId" = "adoptedRules"."serviceLevelObjectiveId" ON CONFLICT DO NOTHING) SELECT "serviceLevelObjectiveId" FROM "adoptedRules"`;
+
+interface AdoptedRuleRow {
+  serviceLevelObjectiveId: string;
+}
+
+export default class SloLegacyMonitorLabelAdoption {
+  /*
+   * Adopts the label lists of those `serviceLevelObjectiveIds` (in
+   * `projectId`) that have one and no monitor rule, and returns the ids of the
+   * SLOs a rule was written for, lower-cased. Everything else is left
+   * untouched, so calling it for an SLO that needs nothing is a no-op.
+   *
+   * `requireRuleAttachedMonitors` (default true) additionally requires the SLO
+   * to still hold monitors as rule-attached - the evidence that the list was
+   * written by a previous-release pod rather than left behind when the user
+   * deleted the rule it had become. Pass false only when the caller has just
+   * seen the list itself change through the deprecated column.
+   */
+  public static async adoptLegacyMonitorLabels(data: {
+    projectId: ObjectID;
+    serviceLevelObjectiveIds: Array<ObjectID | string>;
+    requireRuleAttachedMonitors?: boolean | undefined;
+  }): Promise<Array<string>> {
+    const sloIds: Array<string> = Array.from(
+      new Set<string>(
+        data.serviceLevelObjectiveIds
+          .map((id: ObjectID | string): string => {
+            return id.toString().toLowerCase();
+          })
+          .filter((id: string): boolean => {
+            return id.length > 0;
+          }),
+      ),
+    );
+
+    if (sloIds.length === 0) {
+      return [];
+    }
+
+    const dataSource: DatabaseSource | null =
+      PostgresAppInstance.getDataSource();
+
+    if (!dataSource) {
+      throw new DatabaseNotConnectedException();
+    }
+
+    const projectId: string = data.projectId.toString();
+
+    const requireRuleAttachedMonitors: boolean =
+      data.requireRuleAttachedMonitors !== false;
+
+    return await dataSource.transaction(
+      async (manager: EntityManager): Promise<Array<string>> => {
+        await manager.query(SLO_LEGACY_MONITOR_LABEL_LOCK_STATEMENT, [
+          sloIds,
+          projectId,
+        ]);
+
+        const rows: Array<AdoptedRuleRow> = await manager.query(
+          SLO_LEGACY_MONITOR_LABEL_ADOPT_STATEMENT,
+          [
+            sloIds,
+            projectId,
+            MIGRATED_MONITOR_RULE_NAME,
+            MIGRATED_MONITOR_RULE_DESCRIPTION,
+            requireRuleAttachedMonitors,
+          ],
+        );
+
+        return rows.map((row: AdoptedRuleRow): string => {
+          return row.serviceLevelObjectiveId.toString().toLowerCase();
+        });
+      },
+    );
+  }
+}

package/Server/Utils/Slo/SloMetricUtil.ts

@@ -0,0 +1,327 @@
+import logger from "../Logger";
+import CaptureSpan from "../Telemetry/CaptureSpan";
+import TelemetryUtil from "../Telemetry/Telemetry";
+import GlobalConfigService from "../../Services/GlobalConfigService";
+import MetricService from "../../Services/MetricService";
+import { MetricPointType } from "../../../Models/AnalyticsModels/Metric";
+import GlobalConfig from "../../../Models/DatabaseModels/GlobalConfig";
+import Label from "../../../Models/DatabaseModels/Label";
+import MetricType from "../../../Models/DatabaseModels/MetricType";
+import OneUptimeDate from "../../../Types/Date";
+import Dictionary from "../../../Types/Dictionary";
+import { JSONObject } from "../../../Types/JSON";
+import ObjectID from "../../../Types/ObjectID";
+import SloMetricType from "../../../Types/ServiceLevelObjective/SloMetricType";
+import ServiceType from "../../../Types/Telemetry/ServiceType";
+import MetricResourceAttributeUtil from "../../../Utils/Metrics/MetricResourceAttributeUtil";
+import SloMetricTypeUtil, {
+  SLO_METRIC_PROJECT_ID_ATTRIBUTE,
+  SLO_METRIC_SLO_ID_ATTRIBUTE,
+  SLO_METRIC_SLO_NAME_ATTRIBUTE,
+} from "../../../Utils/Slo/SloMetricType";
+
+/*
+ * Posts one SLO evaluation as `oneuptime.slo.*` metrics, so an SLO can be
+ * charted, filtered and dashboarded like any monitor series.
+ *
+ * Modelled on NetworkDeviceMetricUtil: append-only MetricItemV3 rows keyed to
+ * the SLO itself (primaryEntityId = the SLO id, primaryEntityType =
+ * ServiceType.ServiceLevelObjective), not to any of its monitors. The names
+ * start with `oneuptime.slo.`, which MutableMetricService does NOT route to
+ * the mutable table, so reads find these rows in MetricItemV3 where they are
+ * written.
+ *
+ * This does not replace SloHistory. SloHistory keeps 400 days in monthly
+ * partitions for the long-range error-budget charts; these rows follow the
+ * monitor-metric retention (daily partitions with ttl_only_drop_parts), which
+ * is why they must never be given a history-length retention - every row
+ * would pin its daily partition for a year.
+ */
+export default class SloMetricUtil {
+  // Retention handling mirrors MonitorMetricUtil (shared GlobalConfig knob).
+  private static readonly DEFAULT_RETENTION_DAYS: number = 30;
+
+  /*
+   * The worker evaluates every SLO every few minutes, so thousands of SLOs
+   * would otherwise read GlobalConfig hundreds of times a minute for a value
+   * an admin changes a few times a year.
+   */
+  private static cachedRetentionDays: number | null = null;
+  private static lastCacheRefresh: Date | null = null;
+  private static readonly CACHE_TTL_MS: number = 5 * 60 * 1000;
+
+  private static async getRetentionDays(): Promise<number> {
+    const now: Date = OneUptimeDate.getCurrentDate();
+
+    if (
+      this.cachedRetentionDays !== null &&
+      this.lastCacheRefresh !== null &&
+      now.getTime() - this.lastCacheRefresh.getTime() < this.CACHE_TTL_MS
+    ) {
+      return this.cachedRetentionDays;
+    }
+
+    try {
+      const globalConfig: GlobalConfig | null =
+        await GlobalConfigService.findOneBy({
+          query: {
+            _id: ObjectID.getZeroObjectID().toString(),
+          },
+          props: {
+            isRoot: true,
+          },
+          select: {
+            monitorMetricRetentionInDays: true,
+          },
+        });
+
+      /*
+       * Zero or a negative number would stamp a retention date at or before
+       * the row's own timestamp, and ClickHouse would drop the evaluation the
+       * moment the part merged. Treat it as unset, as MonitorMetricUtil does.
+       */
+      if (
+        globalConfig &&
+        globalConfig.monitorMetricRetentionInDays !== undefined &&
+        globalConfig.monitorMetricRetentionInDays !== null &&
+        globalConfig.monitorMetricRetentionInDays > 0
+      ) {
+        this.cachedRetentionDays = globalConfig.monitorMetricRetentionInDays;
+      } else {
+        this.cachedRetentionDays = this.DEFAULT_RETENTION_DAYS;
+      }
+    } catch (err) {
+      logger.error(
+        "Error fetching monitor metric retention config for SLO metrics, using default:",
+      );
+      logger.error(err);
+      this.cachedRetentionDays = this.DEFAULT_RETENTION_DAYS;
+    }
+
+    this.lastCacheRefresh = now;
+    return this.cachedRetentionDays;
+  }
+
+  @CaptureSpan()
+  public static async saveSloMetrics(data: {
+    projectId: ObjectID;
+    sloId: ObjectID;
+    sloName?: string | undefined;
+    labels?: Array<Label> | undefined;
+    // null, undefined and non-finite values are skipped, never written as 0.
+    values: Partial<Record<SloMetricType, number | null | undefined>>;
+  }): Promise<void> {
+    const metricsToWrite: Array<{ metricType: SloMetricType; value: number }> =
+      [];
+
+    /*
+     * Walk the util's list rather than the caller's object, so the rows come
+     * out in one stable order and only names that have catalog metadata can
+     * ever be written.
+     */
+    for (const metricType of SloMetricTypeUtil.getAll()) {
+      const value: number | null | undefined = data.values[metricType];
+
+      /*
+       * A NaN burn rate or an Infinity budget percentage (a zero-length
+       * window) has no meaningful point to chart, and ClickHouse would store
+       * it as a value every aggregate then has to step around.
+       */
+      if (typeof value !== "number" || !isFinite(value)) {
+        continue;
+      }
+
+      metricsToWrite.push({ metricType: metricType, value: value });
+    }
+
+    if (metricsToWrite.length === 0) {
+      return;
+    }
+
+    const baseAttributes: JSONObject = {
+      [SLO_METRIC_SLO_ID_ATTRIBUTE]: data.sloId.toString(),
+      [SLO_METRIC_PROJECT_ID_ATTRIBUTE]: data.projectId.toString(),
+    };
+
+    if (data.sloName && data.sloName.trim().length > 0) {
+      baseAttributes[SLO_METRIC_SLO_NAME_ATTRIBUTE] = data.sloName;
+    }
+
+    /*
+     * Labels are merged LAST (oneuptime.label.* cannot collide with the bare
+     * keys above anyway) and attributeKeys is computed from the merged set,
+     * because the dashboard's attribute pickers read attributeKeys, not the
+     * attributes map.
+     */
+    const attributes: JSONObject =
+      MetricResourceAttributeUtil.mergeResourceAttributes(
+        baseAttributes,
+        MetricResourceAttributeUtil.getResourceAttributes({
+          labels: data.labels,
+        }),
+      );
+
+    const retentionDays: number = await SloMetricUtil.getRetentionDays();
+
+    /*
+     * One timestamp for the whole evaluation, so the SLI and its target (and
+     * every other series) land on exactly the same x and overlay cleanly.
+     */
+    const ingestionDate: Date = OneUptimeDate.getCurrentDate();
+    const retentionDate: Date = OneUptimeDate.addRemoveDays(
+      ingestionDate,
+      retentionDays,
+    );
+
+    const metricRows: Array<JSONObject> = [];
+    const metricNameServiceNameMap: Dictionary<MetricType> = {};
+
+    for (const metricToWrite of metricsToWrite) {
+      metricRows.push(
+        SloMetricUtil.buildSloMetricRow({
+          projectId: data.projectId,
+          sloId: data.sloId,
+          metricName: metricToWrite.metricType,
+          value: metricToWrite.value,
+          attributes: attributes,
+          ingestionDate: ingestionDate,
+          retentionDate: retentionDate,
+        }),
+      );
+
+      /*
+       * Registering the name is what puts the series in the dashboard metric
+       * picker (it reads MetricType rows, not any enum). Description and unit
+       * come from the same util the SLO Metrics page charts with.
+       */
+      const metricType: MetricType = new MetricType();
+      metricType.name = metricToWrite.metricType;
+      metricType.description = SloMetricTypeUtil.getDescription(
+        metricToWrite.metricType,
+      );
+      metricType.unit = SloMetricTypeUtil.getUnit(metricToWrite.metricType);
+
+      metricNameServiceNameMap[metricToWrite.metricType] = metricType;
+    }
+
+    // One insert per evaluation, however many series it carries.
+    await MetricService.insertJsonRows(metricRows);
+
+    /*
+     * Fire-and-forget: the catalog is fenced by a cache and reconciles on the
+     * next evaluation, so a slow or failed index write must not hold up the
+     * worker's notifications and burn-rate rules.
+     */
+    TelemetryUtil.indexMetricNameServiceNameMap({
+      projectId: data.projectId,
+      metricNameServiceNameMap: metricNameServiceNameMap,
+    }).catch((err: Error) => {
+      logger.error(err);
+    });
+  }
+
+  /*
+   * What the worker posts when a guard stops it measuring an SLO: Paused
+   * (every monitor disabled), Misconfigured (no monitors, monitors gone, an
+   * unusable target) and the zero-data guard (monitors with no status history
+   * yet). Only the TARGET is written.
+   *
+   * Why anything at all: the SLO dashboard template's toolbar picker lists the
+   * `sloName` values posted in the last day (TelemetryAttributeService's
+   * lookback). Posting nothing on these paths left a never-measured SLO out
+   * of the picker for good, and dropped a Paused or Misconfigured one a day
+   * after its last good evaluation, while the SLO List beside the picker
+   * still showed it.
+   *
+   * Why only the target: it is configuration, true whether or not anything
+   * was measured, so a Metrics page charting it shows the objective with no
+   * SLI under it. A budget, burn rate or status point would claim a
+   * measurement that did not happen, which is why the Status series keeps no
+   * value for Paused and Misconfigured (SloMetricTypeUtil). The rows are the
+   * same shape, attributes and retention as an evaluation's, so the picker,
+   * the sloId filters and the label pickers read them identically.
+   *
+   * Never throws: this runs on a path that must still resolve open burn-rate
+   * alerts and record the guard status, so a metric store failure is logged
+   * and swallowed here rather than left to every call site.
+   */
+  @CaptureSpan()
+  public static async saveSloGuardMetrics(data: {
+    projectId: ObjectID;
+    sloId: ObjectID;
+    sloName?: string | undefined;
+    labels?: Array<Label> | undefined;
+    targetPercentage?: number | null | undefined;
+  }): Promise<void> {
+    try {
+      await SloMetricUtil.saveSloMetrics({
+        projectId: data.projectId,
+        sloId: data.sloId,
+        sloName: data.sloName,
+        labels: data.labels,
+        values: {
+          [SloMetricType.TargetPercent]: data.targetPercentage,
+        },
+      });
+    } catch (err) {
+      logger.error(
+        `SloMetricUtil - Error writing guard-path SLO metrics for SLO ${data.sloId.toString()}:`,
+      );
+      logger.error(err);
+    }
+  }
+
+  // Row shape must stay in lockstep with MonitorMetricUtil.buildMonitorMetricRow.
+  private static buildSloMetricRow(data: {
+    projectId: ObjectID;
+    sloId: ObjectID;
+    metricName: string;
+    value: number;
+    attributes: JSONObject;
+    ingestionDate: Date;
+    retentionDate: Date;
+  }): JSONObject {
+    const ingestionTimestamp: string = OneUptimeDate.toClickhouseDateTime(
+      data.ingestionDate,
+    );
+    const timeUnixNano: string = OneUptimeDate.toUnixNano(
+      data.ingestionDate,
+    ).toString();
+
+    // A copy per row, so no two rows share (and can mutate) one object.
+    const attributes: JSONObject = { ...data.attributes };
+    const attributeKeys: Array<string> =
+      TelemetryUtil.getAttributeKeys(attributes);
+
+    return {
+      _id: ObjectID.generateTimeOrdered().toString(),
+      createdAt: ingestionTimestamp,
+      projectId: data.projectId.toString(),
+      primaryEntityId: data.sloId.toString(),
+      primaryEntityType: ServiceType.ServiceLevelObjective,
+      name: data.metricName,
+      aggregationTemporality: null,
+      /*
+       * Every SLO series is a point-in-time reading, i.e. a Gauge. Monitor
+       * rows use Sum, but MetricService aggregates Sum and Gauge identically
+       * for scalar queries, so the honest type costs nothing.
+       */
+      metricPointType: MetricPointType.Gauge,
+      time: ingestionTimestamp,
+      startTime: null,
+      timeUnixNano: timeUnixNano,
+      startTimeUnixNano: null,
+      attributes: attributes,
+      attributeKeys: attributeKeys,
+      isMonotonic: null,
+      count: null,
+      sum: null,
+      min: null,
+      max: null,
+      bucketCounts: [],
+      explicitBounds: [],
+      value: data.value,
+      retentionDate: OneUptimeDate.toClickhouseDateTime(data.retentionDate),
+    } as JSONObject;
+  }
+}