@oneuptime/common 12.0.22 → 12.0.24

package/Server/Services/SecurityEventService.ts

@@ -13,10 +13,49 @@ import { getQuerySettings } from "../Utils/AnalyticsDatabase/QuerySettingsHelper
 export interface DetectionMatchGroup {
   groupValue: string;
   matchCount: number;
+  /*
+   * uniqExact of the rule's distinctCountField over the group's matches;
+   * 0 whenever the rule has no distinct-count field (the query emits a
+   * constant, not an aggregate, in that case).
+   */
+  distinctCount: number;
   sampleMessage: string;
   sampleObservables: Array<string>;
 }
 
+export interface FindDetectionMatchesData {
+  projectId: ObjectID;
+  startTime: Date;
+  endTime: Date;
+  whereFragment: Statement;
+  groupByExpression: Statement | null;
+  distinctCountExpression: Statement | null;
+  minMatchCount: number;
+  maxGroups: number;
+}
+
+/*
+ * The in-process twin of the HAVING clause findDetectionMatches emits —
+ * exported so the evaluator's re-filter and the SQL builder live in one
+ * file, and a change to either side of the firing predicate is reviewed
+ * next to the other.
+ */
+export function doesGroupMeetDetectionThreshold(data: {
+  group: DetectionMatchGroup;
+  usesDistinctCount: boolean;
+  minMatchCount: number;
+}): boolean {
+  if (data.group.matchCount <= 0) {
+    return false;
+  }
+
+  const effectiveCount: number = data.usesDistinctCount
+    ? data.group.distinctCount
+    : data.group.matchCount;
+
+  return effectiveCount >= Math.max(1, data.minMatchCount);
+}
+
 export class SecurityEventService extends AnalyticsDatabaseService<SecurityEvent> {
   public constructor(clickhouseDatabase?: ClickhouseDatabase | undefined) {
     super({ modelType: SecurityEvent, database: clickhouseDatabase });
@@ -24,18 +63,21 @@ export class SecurityEventService extends AnalyticsDatabaseService<SecurityEvent
 
   /*
    * Detection-engine query: count matching events in a window, grouped by
-   * an optional field expression. `whereFragment` and
-   * `groupByExpression` are Statements built by SigmaClickhouseCompiler —
-   * every rule-controlled value inside them is a bound parameter.
+   * an optional field expression. `whereFragment`, `groupByExpression`
+   * and `distinctCountExpression` are Statements built by
+   * SigmaClickhouseCompiler — every rule-controlled value inside them is
+   * a bound parameter.
+   *
+   * When distinctCountExpression is set, the threshold applies to
+   * uniqExact of that field, not the raw match count — and empty values
+   * are excluded via nullIf, so "5 distinct usernames" never counts rows
+   * that carry no username. The threshold is enforced in HAVING so the
+   * maxGroups LIMIT keeps only qualifying groups instead of letting
+   * noisy-but-under-threshold groups crowd out real ones.
    */
-  public async findDetectionMatches(data: {
-    projectId: ObjectID;
-    startTime: Date;
-    endTime: Date;
-    whereFragment: Statement;
-    groupByExpression: Statement | null;
-    maxGroups: number;
-  }): Promise<Array<DetectionMatchGroup>> {
+  public async findDetectionMatches(
+    data: FindDetectionMatchesData,
+  ): Promise<Array<DetectionMatchGroup>> {
     if (!this.database) {
       this.useDefaultDatabase();
     }
@@ -51,8 +93,18 @@ export class SecurityEventService extends AnalyticsDatabaseService<SecurityEvent
       statement.append("''");
     }
 
+    statement.append(" AS groupValue, count() AS matchCount, ");
+
+    if (data.distinctCountExpression) {
+      statement.append("uniqExact(nullIf(");
+      statement.append(data.distinctCountExpression);
+      statement.append(", ''))");
+    } else {
+      statement.append("0");
+    }
+
     statement.append(
-      ` AS groupValue, count() AS matchCount, any(message) AS sampleMessage, arrayDistinct(arrayFlatten(groupArray(20)(observables))) AS sampleObservables FROM ${databaseName}.${this.model.tableName} WHERE `,
+      ` AS distinctCount, any(message) AS sampleMessage, arrayDistinct(arrayFlatten(groupArray(20)(observables))) AS sampleObservables FROM ${databaseName}.${this.model.tableName} WHERE `,
     );
 
     statement.append(
@@ -69,7 +121,31 @@ export class SecurityEventService extends AnalyticsDatabaseService<SecurityEvent
     );
 
     statement.append(data.whereFragment);
-    statement.append(") GROUP BY groupValue ORDER BY matchCount DESC LIMIT ");
+    statement.append(") GROUP BY groupValue");
+
+    /*
+     * The effective count — what the rule's threshold means — is the
+     * distinct count when a distinct expression was given, else the raw
+     * match count. A distinct-count rule always gets a HAVING clause,
+     * even at threshold 1: a group whose matches all lack the counted
+     * field has distinctCount 0 and must not fire.
+     */
+    const effectiveCountColumn: string = data.distinctCountExpression
+      ? "distinctCount"
+      : "matchCount";
+    const minMatchCount: number = Math.max(1, data.minMatchCount);
+
+    if (data.distinctCountExpression || minMatchCount > 1) {
+      statement.append(` HAVING ${effectiveCountColumn} >= `);
+      statement.append(
+        SQL`${{
+          type: TableColumnType.Number,
+          value: minMatchCount,
+        }}`,
+      );
+    }
+
+    statement.append(` ORDER BY ${effectiveCountColumn} DESC LIMIT `);
     statement.append(
       SQL`${{
         type: TableColumnType.Number,
@@ -88,6 +164,7 @@ export class SecurityEventService extends AnalyticsDatabaseService<SecurityEvent
       return {
         groupValue: String(row["groupValue"] ?? ""),
         matchCount: Number(row["matchCount"] || 0),
+        distinctCount: Number(row["distinctCount"] || 0),
         sampleMessage: String(row["sampleMessage"] ?? ""),
         sampleObservables: Array.isArray(row["sampleObservables"])
           ? (row["sampleObservables"] as Array<unknown>).map(

package/Server/Services/StatusPageResourceService.ts

@@ -17,6 +17,82 @@ import Model from "../../Models/DatabaseModels/StatusPageResource";
 import Monitor from "../../Models/DatabaseModels/Monitor";
 import MonitorGroupResource from "../../Models/DatabaseModels/MonitorGroupResource";
 
+/**
+ * The ways a create or update can name the resource's target. Everything
+ * server side sets the foreign key column, but the dashboard's resource form
+ * posts the relation - and the relation does NOT arrive in the same shape on
+ * both write paths:
+ *
+ *   - create goes through BaseAPI.createItem, which revives the body with
+ *     BaseModel.fromJSON, so `monitor` is a real Monitor and `monitor.id` is
+ *     an ObjectID;
+ *   - update goes through BaseAPI.updateItem, which only runs
+ *     JSONFunctions.deserialize. That revives ObjectID/DateTime values, but
+ *     never nested models, so `monitor` stays the plain `{ _id: "<uuid>" }`
+ *     the browser sent and has no `id` at all.
+ *
+ * Reading only `.id` therefore silently saw nothing on every edit-form save,
+ * which is the whole path the update guard exists for - so this accepts the
+ * id however it arrives.
+ */
+type StatusPageResourceTargetValue =
+  | ObjectID
+  | string
+  | { id?: unknown; _id?: unknown }
+  | null
+  | undefined;
+
+interface StatusPageResourceTargetInput {
+  monitorId?: StatusPageResourceTargetValue;
+  monitor?: StatusPageResourceTargetValue;
+  monitorGroupId?: StatusPageResourceTargetValue;
+  monitorGroup?: StatusPageResourceTargetValue;
+}
+
+function toTargetObjectID(
+  value: StatusPageResourceTargetValue,
+): ObjectID | null {
+  if (!value) {
+    return null;
+  }
+
+  if (value instanceof ObjectID) {
+    return value;
+  }
+
+  if (typeof value === "string") {
+    return new ObjectID(value);
+  }
+
+  if (typeof value === "object") {
+    return (
+      toTargetObjectID(value.id as StatusPageResourceTargetValue) ||
+      toTargetObjectID(value._id as StatusPageResourceTargetValue)
+    );
+  }
+
+  return null;
+}
+
+interface StatusPageResourceTarget {
+  monitorId: ObjectID | null;
+  monitorGroupId: ObjectID | null;
+}
+
+/**
+ * Named after what the operator sees rather than what the column is, because
+ * this reads back to them on the resource form.
+ */
+function duplicateResourceException(
+  target: StatusPageResourceTarget,
+): BadDataException {
+  const thing: string = target.monitorId ? "monitor" : "monitor group";
+
+  return new BadDataException(
+    `This ${thing} is already added to this status page. A ${thing} can only be added once so it is not shown twice to your customers.`,
+  );
+}
+
 export class Service extends DatabaseService<Model> {
   public constructor() {
     super(Model);
@@ -115,6 +191,76 @@ export class Service extends DatabaseService<Model> {
     return statusPageResources;
   }
 
+  /**
+   * The id of the thing a resource points at, whichever way the caller
+   * expressed it. The dashboard's resource form posts the relation
+   * (`monitor: { _id }`) while everything server side sets the foreign key
+   * column, and both mean the same resource.
+   */
+  private getResourceMonitorTarget(
+    data: StatusPageResourceTargetInput,
+  ): StatusPageResourceTarget {
+    return {
+      monitorId:
+        toTargetObjectID(data.monitorId) || toTargetObjectID(data.monitor),
+      monitorGroupId:
+        toTargetObjectID(data.monitorGroupId) ||
+        toTargetObjectID(data.monitorGroup),
+    };
+  }
+
+  /**
+   * A status page lists a monitor once.
+   *
+   * Nothing stopped the same monitor being added twice, so re-adding a label's
+   * monitors after a new one joined that label created a second resource for
+   * every monitor already there, and the public page listed each of them twice
+   * (issue #3420). The rule engine has always refused to add a monitor that is
+   * already on the page for exactly this reason; this makes the same promise
+   * hold for every other way a resource is created - the resource form, the
+   * bulk add modal, and the API.
+   *
+   * The check is status-page-wide rather than per group: a monitor in two
+   * groups is still a monitor a visitor sees twice.
+   */
+  @CaptureSpan()
+  public async isResourceAlreadyOnStatusPage(data: {
+    statusPageId: ObjectID;
+    monitorId?: ObjectID | null | undefined;
+    monitorGroupId?: ObjectID | null | undefined;
+    excludeResourceId?: ObjectID | null | undefined;
+  }): Promise<boolean> {
+    if (!data.monitorId && !data.monitorGroupId) {
+      return false;
+    }
+
+    const query: Query<Model> = {
+      statusPageId: data.statusPageId,
+    };
+
+    if (data.monitorId) {
+      query.monitorId = data.monitorId;
+    } else {
+      query.monitorGroupId = data.monitorGroupId!;
+    }
+
+    if (data.excludeResourceId) {
+      query._id = QueryHelper.notEquals(data.excludeResourceId.toString());
+    }
+
+    const existingResource: Model | null = await this.findOneBy({
+      query: query,
+      select: {
+        _id: true,
+      },
+      props: {
+        isRoot: true,
+      },
+    });
+
+    return Boolean(existingResource);
+  }
+
   @CaptureSpan()
   protected override async onBeforeCreate(
     createBy: CreateBy<Model>,
@@ -125,6 +271,20 @@ export class Service extends DatabaseService<Model> {
       );
     }
 
+    const target: StatusPageResourceTarget = this.getResourceMonitorTarget(
+      createBy.data as unknown as StatusPageResourceTargetInput,
+    );
+
+    if (
+      await this.isResourceAlreadyOnStatusPage({
+        statusPageId: createBy.data.statusPageId,
+        monitorId: target.monitorId,
+        monitorGroupId: target.monitorGroupId,
+      })
+    ) {
+      throw duplicateResourceException(target);
+    }
+
     if (!createBy.data.order) {
       const query: Query<Model> = {
         statusPageId: createBy.data.statusPageId,
@@ -222,6 +382,69 @@ export class Service extends DatabaseService<Model> {
   protected override async onBeforeUpdate(
     updateBy: UpdateBy<Model>,
   ): Promise<OnUpdate<Model>> {
+    /*
+     * Pointing an existing resource at a monitor the page already lists is the
+     * same duplicate onBeforeCreate refuses, just reached from the edit form.
+     */
+    const updatedTarget: StatusPageResourceTarget =
+      this.getResourceMonitorTarget(
+        updateBy.data as unknown as StatusPageResourceTargetInput,
+      );
+
+    if (
+      (updatedTarget.monitorId || updatedTarget.monitorGroupId) &&
+      updateBy.query._id
+    ) {
+      const resourceBeingUpdated: Model | null = await this.findOneBy({
+        query: {
+          _id: updateBy.query._id!,
+        },
+        props: {
+          isRoot: true,
+        },
+        select: {
+          _id: true,
+          statusPageId: true,
+          monitorId: true,
+          monitorGroupId: true,
+        },
+      });
+
+      const currentTarget: StatusPageResourceTarget =
+        this.getResourceMonitorTarget(
+          (resourceBeingUpdated ||
+            {}) as unknown as StatusPageResourceTargetInput,
+        );
+
+      /*
+       * The edit form is a ModelForm, so it posts every field it collects -
+       * the monitor included - even when all the operator changed was the
+       * display name. Checking an unchanged target would refuse those saves
+       * on a status page that already carries a duplicate from before this
+       * rule existed, which would leave both of its rows uneditable.
+       */
+      const isTargetUnchanged: boolean =
+        (!updatedTarget.monitorId ||
+          updatedTarget.monitorId.toString() ===
+            currentTarget.monitorId?.toString()) &&
+        (!updatedTarget.monitorGroupId ||
+          updatedTarget.monitorGroupId.toString() ===
+            currentTarget.monitorGroupId?.toString());
+
+      if (
+        resourceBeingUpdated?.statusPageId &&
+        !isTargetUnchanged &&
+        (await this.isResourceAlreadyOnStatusPage({
+          statusPageId: resourceBeingUpdated.statusPageId,
+          monitorId: updatedTarget.monitorId,
+          monitorGroupId: updatedTarget.monitorGroupId,
+          excludeResourceId: resourceBeingUpdated.id,
+        }))
+      ) {
+        throw duplicateResourceException(updatedTarget);
+      }
+    }
+
     if (updateBy.data.order && !updateBy.props.isRoot && updateBy.query._id) {
       const resource: Model | null = await this.findOneBy({
         query: {

package/Server/Services/TelemetryAttributeService.ts

@@ -4,6 +4,7 @@ import LogDatabaseService from "./LogService";
 import MetricDatabaseService from "./MetricService";
 import SpanDatabaseService from "./SpanService";
 import ExceptionInstanceService from "./ExceptionInstanceService";
+import SecurityEventService from "./SecurityEventService";
 import MutableMetricDatabaseService, {
   MutableMetricService as MutableMetricServiceClass,
 } from "./MutableMetricService";
@@ -101,6 +102,22 @@ export class TelemetryAttributeService {
           attributeKeysColumn: "attributeKeys",
           timeColumn: "time",
         };
+      /*
+       * Security events carry the whole source payload flattened into
+       * `attributes`, which is where every field the OCSF schema does not
+       * have a typed column for ends up (device.hostname,
+       * finding_info.title, metadata.product.name, ...). The security
+       * events table offers those as optional columns, and this is the
+       * list it offers.
+       */
+      case TelemetryType.SecurityEvent:
+        return {
+          service: SecurityEventService,
+          tableName: SecurityEventService.model.tableName,
+          attributesColumn: "attributes",
+          attributeKeysColumn: "attributeKeys",
+          timeColumn: "time",
+        };
       default:
         return null;
     }

package/Server/Services/UserService.ts

@@ -42,7 +42,9 @@ import TeamMember from "../../Models/DatabaseModels/TeamMember";
 import Model from "../../Models/DatabaseModels/User";
 import SlackUtil from "../Utils/Workspace/Slack/Slack";
 import ProductAnalytics from "../Utils/ProductAnalytics";
-import MarketingEventUtil from "../Utils/Marketing/MarketingEventUtil";
+import MarketingEventUtil, {
+  utmAnalyticsProperties,
+} from "../Utils/Marketing/MarketingEventUtil";
 import { MarketingEventType } from "../../Types/Marketing/MarketingEvent";
 import UserTotpAuthService from "./UserTotpAuthService";
 import UserTwoFactorBackupCodeService from "./UserTwoFactorBackupCodeService";
@@ -242,11 +244,7 @@ export class Service extends DatabaseService<Model> {
         distinctId: createdItem.email.toString(),
         properties: {
           has_password: Boolean(createdItem.password),
-          utm_source: createdItem.utmSource || "",
-          utm_medium: createdItem.utmMedium || "",
-          utm_campaign: createdItem.utmCampaign || "",
-          utm_term: createdItem.utmTerm || "",
-          utm_content: createdItem.utmContent || "",
+          ...utmAnalyticsProperties(createdItem as unknown as JSONObject),
           utm_url: createdItem.utmUrl || "",
           click_ids: createdItem.clickIds || {},
           first_touch: createdItem.firstTouchAttribution || {},