@oneuptime/common 12.0.15 → 12.0.16

package/Server/Utils/Marketing/Providers/Reddit.ts

@@ -1,5 +1,4 @@
 import axios, { AxiosResponse } from "axios";
-import crypto from "crypto";
 import {
   RedditAdsAccountId,
   RedditAdsOAuthClientId,
@@ -8,9 +7,11 @@ import {
 } from "../../../EnvironmentConfig";
 import ConversionUploadProvider, {
   ConversionSkip,
+  ConversionTypeMapping,
   ConversionUploadBatchResult,
 } from "../ConversionUploadProvider";
 import { JSONObject } from "../../../../Types/JSON";
+import { MarketingConversionType } from "../../../../Types/Marketing/MarketingConversion";
 import MarketingConversion from "../../../../Models/DatabaseModels/MarketingConversion";
 
 const REQUEST_TIMEOUT_MS: number = 30000;
@@ -21,9 +22,10 @@ const REDDIT_MAX_EVENT_AGE_IN_DAYS: number = 7;
 /*
  * Reddit Conversions API
  * (ads-api.reddit.com/api/v2.0/conversions/events/{accountId}) using the
- * rdt_cid click id. conversion_id (the conversion row id) makes retried
- * uploads idempotent on Reddit's side. OAuth2 refresh-token flow with HTTP
- * basic auth against reddit.com.
+ * rdt_cid click id or a SHA-256 email, whichever the conversion carries.
+ * conversion_id (the conversion row id) makes retried uploads idempotent on
+ * Reddit's side. OAuth2 refresh-token flow with HTTP basic auth against
+ * reddit.com.
  */
 export default class RedditProvider extends ConversionUploadProvider {
   public override readonly key: string = "reddit";
@@ -44,9 +46,19 @@ export default class RedditProvider extends ConversionUploadProvider {
   protected override getProviderSkipReason(
     conversion: MarketingConversion,
   ): ConversionSkip | null {
-    if (!this.getClickId(conversion, "rdt_cid")) {
+    if (
+      !this.getClickId(conversion, "rdt_cid") &&
+      !this.getHashedEmail(conversion)
+    ) {
       return {
-        reason: "No Reddit click id (rdt_cid)",
+        reason: "No Reddit click id (rdt_cid) and no email to match on",
+        isPermanent: true,
+      };
+    }
+
+    if (!this.getTrackingType(conversion)) {
+      return {
+        reason: "No Reddit tracking type mapped for this conversion type",
         isPermanent: true,
       };
     }
@@ -61,11 +73,19 @@ export default class RedditProvider extends ConversionUploadProvider {
     return null;
   }
 
-  private hashEmail(email: string): string {
-    return crypto
-      .createHash("sha256")
-      .update(email.trim().toLowerCase())
-      .digest("hex");
+  /*
+   * Reddit standard tracking types. A booked meeting is `Lead`: Reddit has no
+   * distinct "meeting booked" type, and reporting it as Purchase would put a
+   * demo into the same optimisation pool as revenue.
+   */
+  private getTrackingType(conversion: MarketingConversion): string | undefined {
+    const mapping: ConversionTypeMapping<string> = {
+      [MarketingConversionType.SignUp]: "SignUp",
+      [MarketingConversionType.MeetingBooked]: "Lead",
+      [MarketingConversionType.PaidSubscription]: "Purchase",
+    };
+
+    return this.resolveByConversionType(conversion, mapping);
   }
 
   private async getAccessToken(): Promise<string> {
@@ -136,16 +156,26 @@ export default class RedditProvider extends ConversionUploadProvider {
 
         const payload: JSONObject = {
           event_at: eventAt.toISOString(),
-          click_id: this.getClickId(conversion, "rdt_cid") || "",
           event_type: {
-            tracking_type: this.isSignUp(conversion) ? "SignUp" : "Purchase",
+            tracking_type: this.getTrackingType(conversion) || "Lead",
           },
           event_metadata: eventMetadata,
         };
 
-        if (conversion.email) {
+        const clickId: string | undefined = this.getClickId(
+          conversion,
+          "rdt_cid",
+        );
+
+        if (clickId) {
+          payload["click_id"] = clickId;
+        }
+
+        const hashedEmail: string | undefined = this.getHashedEmail(conversion);
+
+        if (hashedEmail) {
           payload["user"] = {
-            email: this.hashEmail(conversion.email),
+            email: hashedEmail,
           };
         }
 

package/Server/Utils/Measurement/MeasurementDefinitionValidator.ts

@@ -0,0 +1,209 @@
+import BadDataException from "../../../Types/Exception/BadDataException";
+import MeasurementOccurrence from "../../../Types/Measurement/MeasurementOccurrence";
+
+/*
+ * Validation shared by the incident, alert and scheduled maintenance
+ * measurement services.
+ *
+ * Every rule here exists to stop a definition that would silently produce a
+ * confidently wrong number -- a constant zero, or a duration that can never
+ * complete with nothing anywhere saying so.
+ */
+export default class MeasurementDefinitionValidator {
+  /*
+   * The key becomes part of the metric name, which lands in ClickHouse, in
+   * the dashboard chart picker and in the API. It is immutable after
+   * creation, so it is worth rejecting a bad one loudly at the only moment
+   * it can still be changed.
+   */
+  public static readonly KEY_PATTERN: RegExp = /^[a-z0-9][a-z0-9-]{0,49}$/;
+
+  public static validateKey(key: string | undefined): void {
+    if (!key) {
+      throw new BadDataException(
+        "A measurement needs a key. Use lowercase letters, numbers and hyphens, for example: time-to-detect",
+      );
+    }
+
+    if (!MeasurementDefinitionValidator.KEY_PATTERN.test(key)) {
+      throw new BadDataException(
+        `"${key}" is not a valid measurement key. Keys start with a lowercase letter or number, contain only lowercase letters, numbers and hyphens, and are at most 50 characters. For example: time-to-detect`,
+      );
+    }
+  }
+
+  public static validateAnchorPair(data: {
+    startAnchorType?: string | undefined;
+    endAnchorType?: string | undefined;
+    stateEnteredAnchor: string;
+    stateRoleEnteredAnchor: string;
+    /*
+     * Maps each timestamp anchor type onto the column it actually reads.
+     * Several anchors are aliases -- an alert's "Timeline Start" and
+     * "Created At" are both createdAt -- so comparing the enum values alone
+     * would wave through a definition that is a constant zero by
+     * construction.
+     */
+    timestampAnchorSources?: Record<string, string> | undefined;
+    startStateId?: string | undefined;
+    endStateId?: string | undefined;
+    startStateRole?: string | undefined;
+    endStateRole?: string | undefined;
+    startOccurrence?: string | undefined;
+    endOccurrence?: string | undefined;
+  }): void {
+    if (!data.startAnchorType) {
+      throw new BadDataException("A measurement needs a starting point.");
+    }
+
+    if (!data.endAnchorType) {
+      throw new BadDataException("A measurement needs an ending point.");
+    }
+
+    if (
+      data.startAnchorType === data.stateEnteredAnchor &&
+      !data.startStateId
+    ) {
+      throw new BadDataException(
+        "Pick the state this measurement starts from.",
+      );
+    }
+
+    if (data.endAnchorType === data.stateEnteredAnchor && !data.endStateId) {
+      throw new BadDataException("Pick the state this measurement ends at.");
+    }
+
+    if (
+      data.startAnchorType === data.stateRoleEnteredAnchor &&
+      !data.startStateRole
+    ) {
+      throw new BadDataException(
+        "Pick the kind of state this measurement starts from.",
+      );
+    }
+
+    if (
+      data.endAnchorType === data.stateRoleEnteredAnchor &&
+      !data.endStateRole
+    ) {
+      throw new BadDataException(
+        "Pick the kind of state this measurement ends at.",
+      );
+    }
+
+    MeasurementDefinitionValidator.rejectIdenticalAnchors(data);
+  }
+
+  /*
+   * Two identical endpoints always measure zero. Shipping that produces a
+   * chart that reads "we do this instantly" on every incident -- which is
+   * strictly worse than having no chart, because it looks like an answer.
+   */
+  private static rejectIdenticalAnchors(data: {
+    startAnchorType?: string | undefined;
+    endAnchorType?: string | undefined;
+    stateEnteredAnchor: string;
+    stateRoleEnteredAnchor: string;
+    timestampAnchorSources?: Record<string, string> | undefined;
+    startStateId?: string | undefined;
+    endStateId?: string | undefined;
+    startStateRole?: string | undefined;
+    endStateRole?: string | undefined;
+    startOccurrence?: string | undefined;
+    endOccurrence?: string | undefined;
+  }): void {
+    const startSource: string =
+      MeasurementDefinitionValidator.resolveAnchorSource(
+        data.startAnchorType,
+        data.timestampAnchorSources,
+      );
+    const endSource: string =
+      MeasurementDefinitionValidator.resolveAnchorSource(
+        data.endAnchorType,
+        data.timestampAnchorSources,
+      );
+
+    if (startSource !== endSource) {
+      return;
+    }
+
+    const startOccurrence: string =
+      data.startOccurrence || MeasurementOccurrence.First;
+    const endOccurrence: string =
+      data.endOccurrence || MeasurementOccurrence.First;
+
+    if (data.startAnchorType === data.stateEnteredAnchor) {
+      if (
+        data.startStateId === data.endStateId &&
+        startOccurrence === endOccurrence
+      ) {
+        throw new BadDataException(
+          "This measurement starts and ends at the same point, so it would always be zero. Pick two different points.",
+        );
+      }
+
+      return;
+    }
+
+    if (data.startAnchorType === data.stateRoleEnteredAnchor) {
+      if (
+        data.startStateRole === data.endStateRole &&
+        startOccurrence === endOccurrence
+      ) {
+        throw new BadDataException(
+          "This measurement starts and ends at the same point, so it would always be zero. Pick two different points.",
+        );
+      }
+
+      return;
+    }
+
+    // Two identical timestamp anchors. Always zero, no exceptions.
+    throw new BadDataException(
+      "This measurement starts and ends at the same point, so it would always be zero. Pick two different points.",
+    );
+  }
+
+  /*
+   * The column an anchor reads, so two differently-named anchors that read
+   * the same column are recognised as the same point in time.
+   */
+  private static resolveAnchorSource(
+    anchorType: string | undefined,
+    sources: Record<string, string> | undefined,
+  ): string {
+    if (!anchorType) {
+      return "";
+    }
+
+    return sources?.[anchorType] || anchorType;
+  }
+
+  /*
+   * States are entered in strictly increasing order, so an end state ordered
+   * at or before the start state describes a duration that can never
+   * complete. Without this the definition looks fine and every incident
+   * reports "not applicable" forever.
+   */
+  public static validateStateOrder(data: {
+    startStateName?: string | undefined;
+    startStateOrder?: number | undefined;
+    endStateName?: string | undefined;
+    endStateOrder?: number | undefined;
+  }): void {
+    if (
+      data.startStateOrder === undefined ||
+      data.endStateOrder === undefined
+    ) {
+      return;
+    }
+
+    if (data.endStateOrder > data.startStateOrder) {
+      return;
+    }
+
+    throw new BadDataException(
+      `"${data.endStateName}" comes before "${data.startStateName}" in this project's state order, so this measurement could never complete. Swap the two, or pick a later ending state.`,
+    );
+  }
+}

package/Server/Utils/Measurement/MeasurementMetricWriter.ts

@@ -0,0 +1,189 @@
+import MetricType from "../../../Models/DatabaseModels/MetricType";
+import MutableMetric from "../../../Models/AnalyticsModels/MutableMetric";
+import { MetricPointType } from "../../../Models/AnalyticsModels/Metric";
+import GlobalConfigService from "../../Services/GlobalConfigService";
+import MutableMetricService from "../../Services/MutableMetricService";
+import GlobalConfig from "../../../Models/DatabaseModels/GlobalConfig";
+import ObjectID from "../../../Types/ObjectID";
+import OneUptimeDate from "../../../Types/Date";
+import ServiceType from "../../../Types/Telemetry/ServiceType";
+import TelemetryUtil from "../Telemetry/Telemetry";
+import { JSONObject } from "../../../Types/JSON";
+import logger from "../Logger";
+
+export interface MeasurementMetricPoint {
+  metricName: string;
+  measurementId: string;
+  measurementKey: string;
+  measurementName: string;
+  description?: string | undefined;
+  unit?: string | undefined;
+  valueInSeconds: number;
+  time: Date;
+}
+
+const DEFAULT_METRIC_RETENTION_IN_DAYS: number = 180;
+
+/*
+ * Writes measurement values into the existing mutable-metric pipeline.
+ *
+ * The one thing this must get right is the tombstone scope.
+ * MutableMetricService.replaceEntityMetrics tombstones every live metric
+ * point whose name is in `metricNames` and which is not in the desired set it
+ * was handed. So the measurement writer owns its OWN name list -- the
+ * project's measurement metric names, and nothing else -- and never appends
+ * those names to the built-in refresh's replace list. Doing the latter would
+ * make every state change tombstone every measurement point, silently, since
+ * the built-in refresh has no measurement points in its desired set.
+ */
+export default class MeasurementMetricWriter {
+  public static async write(data: {
+    projectId: ObjectID;
+    primaryEntityId: ObjectID;
+    primaryEntityType: ServiceType;
+    // Every measurement metric name in the project, enabled or not.
+    allMeasurementMetricNames: Array<string>;
+    // Only the points that actually have a value; disabled ones are omitted.
+    points: Array<MeasurementMetricPoint>;
+    baseAttributes: JSONObject;
+  }): Promise<void> {
+    if (data.allMeasurementMetricNames.length === 0) {
+      return;
+    }
+
+    const retentionDate: Date = OneUptimeDate.addRemoveDays(
+      OneUptimeDate.getCurrentDate(),
+      await MeasurementMetricWriter.getMetricRetentionInDays(),
+    );
+
+    const metrics: Array<MutableMetric> = [];
+    const metricTypesMap: Record<string, MetricType> = {};
+
+    for (const point of data.points) {
+      const metric: MutableMetric = new MutableMetric();
+
+      metric.projectId = data.projectId;
+      metric.primaryEntityId = data.primaryEntityId;
+      metric.primaryEntityType = data.primaryEntityType;
+      metric.name = point.metricName;
+
+      /*
+       * Per-definition point identity, mirroring the built-in
+       * `time-in-state:<timelineRowId>` shape. Sharing one identity across
+       * definitions would make each refresh overwrite the others.
+       */
+      metric.metricPointId = `measurement:${point.measurementId}`;
+      metric.value = point.valueInSeconds;
+      metric.time = point.time;
+      metric.timeUnixNano = OneUptimeDate.toUnixNano(point.time);
+      metric.metricPointType = MetricPointType.Sum;
+      metric.retentionDate = retentionDate;
+
+      metric.attributes = {
+        ...data.baseAttributes,
+        /*
+         * Namespaced, following the oneuptime.label.* / oneuptime.customField.*
+         * convention, so they cannot collide with the unprefixed camelCase
+         * dimensions the built-in metrics carry.
+         */
+        "oneuptime.measurement.id": point.measurementId,
+        "oneuptime.measurement.key": point.measurementKey,
+        "oneuptime.measurement.name": point.measurementName,
+      };
+      metric.attributeKeys = TelemetryUtil.getAttributeKeys(metric.attributes);
+
+      metrics.push(metric);
+
+      const metricType: MetricType = new MetricType();
+      metricType.name = point.metricName;
+      metricType.description =
+        point.description ||
+        `${point.measurementName}, a measurement defined for this project`;
+      metricType.unit = point.unit || "seconds";
+      metricTypesMap[point.metricName] = metricType;
+    }
+
+    await MutableMetricService.replaceEntityMetrics({
+      projectId: data.projectId,
+      primaryEntityId: data.primaryEntityId,
+      primaryEntityType: data.primaryEntityType,
+      metricNames: data.allMeasurementMetricNames,
+      metrics: metrics,
+      retentionDate: retentionDate,
+    });
+
+    if (Object.keys(metricTypesMap).length === 0) {
+      return;
+    }
+
+    /*
+     * Registering the name is what makes a measurement appear in the
+     * dashboard chart picker -- the picker reads MetricType rows, not any
+     * enum, so no dashboard code is needed for a new measurement.
+     */
+    TelemetryUtil.indexMetricNameServiceNameMap({
+      metricNameServiceNameMap: metricTypesMap,
+      projectId: data.projectId,
+    }).catch((err: Error) => {
+      logger.error(err);
+    });
+  }
+
+  /*
+   * Tombstones every measurement point for one entity. Used when the entity
+   * is deleted, so its points leave the charts rather than lingering until
+   * the retention date.
+   */
+  public static async tombstoneAll(data: {
+    projectId: ObjectID;
+    primaryEntityId: ObjectID;
+    primaryEntityType: ServiceType;
+    allMeasurementMetricNames: Array<string>;
+  }): Promise<void> {
+    if (data.allMeasurementMetricNames.length === 0) {
+      return;
+    }
+
+    const retentionDate: Date = OneUptimeDate.addRemoveDays(
+      OneUptimeDate.getCurrentDate(),
+      await MeasurementMetricWriter.getMetricRetentionInDays(),
+    );
+
+    await MutableMetricService.tombstoneEntityMetrics({
+      projectId: data.projectId,
+      primaryEntityId: data.primaryEntityId,
+      primaryEntityType: data.primaryEntityType,
+      metricNames: data.allMeasurementMetricNames,
+      retentionDate: retentionDate,
+    });
+  }
+
+  private static async getMetricRetentionInDays(): Promise<number> {
+    try {
+      const globalConfig: GlobalConfig | null =
+        await GlobalConfigService.findOneBy({
+          query: {
+            _id: ObjectID.getZeroObjectID().toString(),
+          },
+          props: {
+            isRoot: true,
+          },
+          select: {
+            monitorMetricRetentionInDays: true,
+          },
+        });
+
+      if (
+        globalConfig &&
+        globalConfig.monitorMetricRetentionInDays !== undefined &&
+        globalConfig.monitorMetricRetentionInDays !== null
+      ) {
+        return globalConfig.monitorMetricRetentionInDays;
+      }
+    } catch (err) {
+      logger.error(err as Error);
+    }
+
+    return DEFAULT_METRIC_RETENTION_IN_DAYS;
+  }
+}

package/Server/Utils/Monitor/Criteria/CountAnomaly.ts

@@ -0,0 +1,182 @@
+import {
+  AnomalyDetectionMethod,
+  FilterType,
+} from "../../../../Types/Monitor/CriteriaFilter";
+
+/**
+ * Consistency constant that maps a Median Absolute Deviation to the
+ * standard deviation of a Gaussian (σ ≈ 1.4826 × MAD), so the same
+ * sigmaCount sensitivity levels apply to both anomaly methods.
+ */
+export const MAD_TO_SIGMA: number = 1.4826;
+
+/**
+ * Descriptive statistics of a set of per-minute telemetry counts —
+ * one hour-of-week baseline cell aggregated over a rolling window.
+ * Computed in app code (not SQL) so the math is pure and unit-testable;
+ * the input is at most `60 × ceil(windowDays / 7)` numbers.
+ */
+export interface CountBaselineStats {
+  /** Number of non-empty minute cells that contributed. */
+  sampleCount: number;
+  mean: number;
+  /** Population standard deviation. */
+  stddev: number;
+  median: number;
+  /** MAD × 1.4826 → σ-equivalent, used by the MedianMad method. */
+  madSigma: number;
+  minObserved: number;
+  maxObserved: number;
+}
+
+/**
+ * Baseline summary returned by the span/log count baseline services:
+ * the stats plus the reliability verdict and lookup coordinates.
+ * Mirrors `BaselineSummary` from MetricBaselineService.
+ */
+export interface CountBaselineSummary extends CountBaselineStats {
+  /**
+   * Whether this cell meets the minimum sample threshold to be trusted.
+   * Callers must refuse to evaluate against an unreliable baseline —
+   * the "Learning" cold-start state.
+   */
+  isReliable: boolean;
+  /** Window the baseline was computed over (days). */
+  windowDays: number;
+  hourOfWeek: number;
+}
+
+export interface CountAnomalyEvaluation {
+  breaches: boolean;
+  /** (observed - center) / spread — signed sigma distance. */
+  observedSigma: number;
+  expectedHigh: number;
+  expectedLow: number;
+  /** Baseline center actually compared against (mean or median). */
+  center: number;
+  /** Baseline spread actually compared against (stddev or MAD×1.4826). */
+  spread: number;
+}
+
+/**
+ * Pure math for count-based anomaly criteria (CheckOn.SpanCount /
+ * CheckOn.LogCount with FilterType.AnomalouslyHigh/Low/Anomalous).
+ * The services compute stats through here so the write-side MV cells
+ * and the eval-time comparison agree on what a "sample" is.
+ */
+export default class CountAnomaly {
+  /**
+   * Compute baseline stats over a cell's per-minute counts. Returns
+   * null for an empty input — the caller treats that as "no baseline"
+   * (Learning), same as a missing row.
+   */
+  public static computeStats(counts: Array<number>): CountBaselineStats | null {
+    if (counts.length === 0) {
+      return null;
+    }
+
+    let sum: number = 0;
+    let min: number = counts[0]!;
+    let max: number = counts[0]!;
+    for (const value of counts) {
+      sum += value;
+      if (value < min) {
+        min = value;
+      }
+      if (value > max) {
+        max = value;
+      }
+    }
+    const mean: number = sum / counts.length;
+
+    let sumSquaredDeviation: number = 0;
+    for (const value of counts) {
+      sumSquaredDeviation += (value - mean) * (value - mean);
+    }
+    const stddev: number = Math.sqrt(sumSquaredDeviation / counts.length);
+
+    const median: number = CountAnomaly.medianOf(counts);
+    const absoluteDeviations: Array<number> = counts.map((value: number) => {
+      return Math.abs(value - median);
+    });
+    const madSigma: number =
+      CountAnomaly.medianOf(absoluteDeviations) * MAD_TO_SIGMA;
+
+    return {
+      sampleCount: counts.length,
+      mean,
+      stddev,
+      median,
+      madSigma,
+      minObserved: min,
+      maxObserved: max,
+    };
+  }
+
+  /**
+   * Compare an observed per-minute rate to a baseline cell. Returns
+   * null when the baseline has zero spread — any deviation at all would
+   * fire, so the caller must skip rather than alert (mirrors the metric
+   * evaluator's zero-variance guard).
+   */
+  public static evaluate(input: {
+    observedPerMinute: number;
+    stats: CountBaselineStats;
+    sigmaCount: number;
+    method: AnomalyDetectionMethod;
+    filterType: FilterType | undefined;
+  }): CountAnomalyEvaluation | null {
+    const isMedianMad: boolean =
+      input.method === AnomalyDetectionMethod.MedianMad;
+    const center: number = isMedianMad ? input.stats.median : input.stats.mean;
+    const spread: number = isMedianMad
+      ? input.stats.madSigma
+      : input.stats.stddev;
+
+    if (!Number.isFinite(spread) || spread <= 0) {
+      return null;
+    }
+
+    const expectedHigh: number = center + input.sigmaCount * spread;
+    const expectedLow: number = center - input.sigmaCount * spread;
+    const observedSigma: number = (input.observedPerMinute - center) / spread;
+
+    const isHighBreach: boolean = input.observedPerMinute > expectedHigh;
+    const isLowBreach: boolean = input.observedPerMinute < expectedLow;
+
+    let breaches: boolean = false;
+    switch (input.filterType) {
+      case FilterType.AnomalouslyHigh:
+        breaches = isHighBreach;
+        break;
+      case FilterType.AnomalouslyLow:
+        breaches = isLowBreach;
+        break;
+      case FilterType.Anomalous:
+        breaches = isHighBreach || isLowBreach;
+        break;
+      default:
+        breaches = false;
+    }
+
+    return {
+      breaches,
+      observedSigma,
+      expectedHigh,
+      expectedLow,
+      center,
+      spread,
+    };
+  }
+
+  private static medianOf(values: Array<number>): number {
+    const sorted: Array<number> = [...values].sort((a: number, b: number) => {
+      return a - b;
+    });
+    const mid: number = Math.floor(sorted.length / 2);
+    if (sorted.length % 2 === 0) {
+      return (sorted[mid - 1]! + sorted[mid]!) / 2;
+    }
+    return sorted[mid]!;
+  }
+}