@oneuptime/common 12.0.23 → 12.0.24

package/Server/Services/NetworkDeviceService.ts

@@ -303,6 +303,24 @@ export class Service extends DatabaseService<Model> {
             await this.applySiteAssignmentRulesToDevice(createdItem.id!);
           }
         })
+        .then(async () => {
+          /*
+           * A device created monitor-backed adopts its monitor's CURRENT
+           * status right away rather than waiting for that monitor's next
+           * status CHANGE. Skipped for SNMP devices, which are judged by
+           * their own walk.
+           */
+          if (
+            NetworkDeviceMonitoringMethodUtil.isMonitorBacked(
+              createdItem.monitoringMethod,
+            )
+          ) {
+            await this.refreshStampedMonitorStatus({
+              deviceId: createdItem.id!,
+              clearWhenNotMonitorBacked: false,
+            });
+          }
+        })
         .catch((error: Error) => {
           logger.error(
             `Error applying network device rules in NetworkDeviceService.onCreateSuccess: ${error}`,
@@ -453,6 +471,109 @@ export class Service extends DatabaseService<Model> {
     };
   }
 
+  /*
+   * Re-derives one device's stamped `currentMonitorStatusId` from its
+   * binding, and refreshes its site chain if the stamp moved.
+   *
+   * The stamp is what every monitor-backed surface reads — the device list
+   * pill, the site rollup, the topology node — and until this existed the
+   * ONLY thing that ever wrote it was
+   * `NetworkSiteService.onMonitorStatusChanged`, which fires on a monitor's
+   * next status CHANGE. Bind a device to a Ping monitor that is already Up
+   * and staying Up and nothing writes the stamp at all, so the device sits
+   * on "Pending" until the monitor happens to go down — which is
+   * OneUptime/oneuptime#3392. Binding is itself an event that decides the
+   * device's status, so it stamps here.
+   *
+   * `clearWhenNotMonitorBacked` is for the write that moves a device OFF
+   * monitor-backed: the ping monitor's verdict must not outlive the binding
+   * (DeviceHealthStateUtil lets a stamped status beat reachability, so a
+   * stale one would poison the site rollup of a device that is now walked).
+   * It is deliberately NOT set when a write only touches the monitor
+   * binding of an SNMP device, because an SNMP device's stamp comes from
+   * the Network Device monitor that watches it, and that binding lives in
+   * the monitor's step data rather than in this column.
+   */
+  @CaptureSpan()
+  public async refreshStampedMonitorStatus(data: {
+    deviceId: ObjectID;
+    clearWhenNotMonitorBacked: boolean;
+  }): Promise<void> {
+    const device: Model | null = await this.findOneById({
+      id: data.deviceId,
+      select: {
+        _id: true,
+        projectId: true,
+        siteId: true,
+        monitoringMethod: true,
+        monitorId: true,
+        currentMonitorStatusId: true,
+      },
+      props: {
+        isRoot: true,
+      },
+    });
+
+    if (!device || !device.projectId) {
+      return;
+    }
+
+    const isMonitorBacked: boolean =
+      NetworkDeviceMonitoringMethodUtil.isMonitorBacked(
+        device.monitoringMethod,
+      );
+
+    if (!isMonitorBacked && !data.clearWhenNotMonitorBacked) {
+      return;
+    }
+
+    let monitorStatusId: ObjectID | null = null;
+
+    if (isMonitorBacked && device.monitorId) {
+      /*
+       * Scoped to the device's project on the read as well as on the write
+       * path in onBeforeCreate/onBeforeUpdate: a monitor from another
+       * tenant must never be able to stamp a status here.
+       */
+      const monitor: Monitor | null = await MonitorService.findOneBy({
+        query: {
+          _id: device.monitorId,
+          projectId: device.projectId,
+        },
+        select: {
+          _id: true,
+          currentMonitorStatusId: true,
+        },
+        props: {
+          isRoot: true,
+        },
+      });
+
+      monitorStatusId = monitor?.currentMonitorStatusId || null;
+    }
+
+    const currentStampId: string | null =
+      device.currentMonitorStatusId?.toString() || null;
+    const nextStampId: string | null = monitorStatusId?.toString() || null;
+
+    if (currentStampId === nextStampId) {
+      return;
+    }
+
+    await this.updateColumnsByIdWithoutHooks({
+      id: data.deviceId,
+      data: {
+        currentMonitorStatusId: monitorStatusId,
+      },
+    });
+
+    if (device.siteId) {
+      await NetworkSiteService.recomputeRollupForSiteAndAncestors(
+        device.siteId,
+      );
+    }
+  }
+
   /*
    * Site maintenance after device updates. Resilient by design: a rollup
    * or rule-engine failure must never fail the device update itself.
@@ -462,6 +583,33 @@ export class Service extends DatabaseService<Model> {
     onUpdate: OnUpdate<Model>,
     updatedItemIds: Array<ObjectID>,
   ): Promise<OnUpdate<Model>> {
+    /*
+     * Re-stamp before the site maintenance below, and in its own try: the
+     * two are independent, and a rollup failure must not cost the device
+     * its status (nor the other way round).
+     */
+    try {
+      const dataKeys: Array<string> = Object.keys(onUpdate.updateBy.data || {});
+      const isMethodWrite: boolean = dataKeys.includes("monitoringMethod");
+      const isMonitorWrite: boolean = RelationIdUtil.isWritten(
+        dataKeys,
+        MONITOR_KEYS,
+      );
+
+      if (isMethodWrite || isMonitorWrite) {
+        for (const deviceId of updatedItemIds) {
+          await this.refreshStampedMonitorStatus({
+            deviceId: deviceId,
+            clearWhenNotMonitorBacked: isMethodWrite,
+          });
+        }
+      }
+    } catch (error) {
+      logger.error(
+        `Error in NetworkDeviceService.onUpdateSuccess monitor status refresh: ${error}`,
+      );
+    }
+
     try {
       const dataKeys: Array<string> = Object.keys(onUpdate.updateBy.data || {});
 

package/Server/Services/OnCallDutyPolicyEscalationRuleScheduleService.ts

@@ -31,6 +31,48 @@ export class Service extends DatabaseService<Model> {
     super(Model);
   }
 
+  /**
+   * Re-resolve and persist the schedule's roster after it is attached to, or
+   * detached from, an on-call policy.
+   *
+   * The persisted roster columns are derived state, and one of their inputs is
+   * WHICH policies escalate to the schedule: a schedule attached to exactly one
+   * policy is resolved in that policy's context, so its policy-scoped user
+   * overrides count, and a schedule attached to none or to several is resolved
+   * policy-blind (OnCallDutyPolicyScheduleService.getSingleAttachedPolicyId).
+   * Attaching or detaching therefore changes the correct answer, and every other
+   * input — layers, layer users, overrides — already refreshes on change.
+   *
+   * Without this the roster stayed on the pre-attach answer until the next
+   * natural hand-off, so the schedule page's "X is currently on the roster"
+   * banner and the "On Call Now" column named the originally-scheduled user for
+   * the whole override window while alert routing paged the substitute.
+   * https://github.com/OneUptime/oneuptime/issues/3411
+   *
+   * Best-effort: an attach must not fail because a roster could not be
+   * recomputed, and the refresh is idempotent (an unaffected schedule simply
+   * resolves to the same roster and diffs to "no change", so no duplicate
+   * hand-off notification is sent).
+   */
+  private async refreshScheduleRoster(
+    onCallDutyPolicyScheduleId: ObjectID | null | undefined,
+  ): Promise<void> {
+    if (!onCallDutyPolicyScheduleId) {
+      return;
+    }
+
+    try {
+      await OnCallDutyPolicyScheduleService.refreshCurrentUserIdAndHandoffTimeInSchedule(
+        onCallDutyPolicyScheduleId,
+      );
+    } catch (err) {
+      logger.error(
+        "Error refreshing the roster after an on-call schedule was attached to or detached from a policy (best-effort).",
+      );
+      logger.error(err);
+    }
+  }
+
   protected override async onCreateSuccess(
     _onCreate: OnCreate<Model>,
     createdItem: Model,
@@ -75,6 +117,12 @@ export class Service extends DatabaseService<Model> {
       );
     }
 
+    /*
+     * Bring the persisted roster in line with the new policy attachment before
+     * anything below can return early. See refreshScheduleRoster.
+     */
+    await this.refreshScheduleRoster(createdModel.onCallDutyPolicyScheduleId);
+
     // send notification to the new current user.
 
     const userOnSchedule: ObjectID | null =
@@ -324,6 +372,23 @@ export class Service extends DatabaseService<Model> {
   ): Promise<OnDelete<Model>> {
     const deletedItems: Array<Model> = onDelete.carryForward.deletedItems;
 
+    /*
+     * Detaching changes the schedule's policy context just as attaching does, so
+     * its roster has to be re-resolved here too. Deduped by schedule: removing a
+     * schedule from several escalation rules of one policy is one context
+     * change, not several. See refreshScheduleRoster.
+     */
+    const refreshedScheduleIds: Set<string> = new Set<string>();
+    for (const deletedItem of deletedItems) {
+      const scheduleId: string | undefined =
+        deletedItem.onCallDutyPolicyScheduleId?.toString();
+      if (!scheduleId || refreshedScheduleIds.has(scheduleId)) {
+        continue;
+      }
+      refreshedScheduleIds.add(scheduleId);
+      await this.refreshScheduleRoster(deletedItem.onCallDutyPolicyScheduleId);
+    }
+
     for (const deletedItem of deletedItems) {
       const userOnSchedule: ObjectID | null =
         await OnCallDutyPolicyScheduleService.getCurrentUserIdInSchedule(

package/Server/Services/OnCallDutyPolicyFeedService.ts

@@ -19,7 +19,7 @@ export class Service extends DatabaseService<OnCallDutyPolicyFeed> {
     super(OnCallDutyPolicyFeed);
 
     if (IsBillingEnabled) {
-      this.hardDeleteItemsOlderThanInDays("createdAt", 90);
+      this.hardDeleteItemsOlderThanInDays("createdAt", 3 * 365); // 3 years
     }
   }
 

package/Server/Services/ProjectService.ts

@@ -16,8 +16,14 @@ import UpdateBy from "../Types/Database/UpdateBy";
 import logger, { LogAttributes } from "../Utils/Logger";
 import Errors from "../Utils/Errors";
 import ProductAnalytics from "../Utils/ProductAnalytics";
-import MarketingEventUtil from "../Utils/Marketing/MarketingEventUtil";
+import MarketingEventUtil, {
+  utmAnalyticsProperties,
+} from "../Utils/Marketing/MarketingEventUtil";
 import { MarketingEventType } from "../../Types/Marketing/MarketingEvent";
+import {
+  UtmPropertyKeys,
+  UtmUrlPropertyKey,
+} from "../../Types/Marketing/Attribution";
 import SessionReplayGateCacheStore from "../Utils/SessionReplay/SessionReplayGateCacheStore";
 import AccessTokenService from "./AccessTokenService";
 import BillingService from "./BillingService";
@@ -198,6 +204,10 @@ export class ProjectService extends DatabaseService<Model> {
         utmMedium: true,
         utmTerm: true,
         utmContent: true,
+        utmId: true,
+        utmSourcePlatform: true,
+        utmCreativeFormat: true,
+        utmMarketingTactic: true,
         utmUrl: true,
         clickIds: true,
         firstTouchAttribution: true,
@@ -348,13 +358,17 @@ export class ProjectService extends DatabaseService<Model> {
     data.data.createdOwnerPhone = user.companyPhoneNumber!;
     data.data.createdOwnerCompanyName = user.companyName!;
 
-    // UTM info.
-    data.data.utmCampaign = user.utmCampaign!;
-    data.data.utmSource = user.utmSource!;
-    data.data.utmMedium = user.utmMedium!;
-    data.data.utmTerm = user.utmTerm!;
-    data.data.utmContent = user.utmContent!;
-    data.data.utmUrl = user.utmUrl!;
+    /*
+     * UTM info, copied field by field off the shared contract rather than
+     * hand-listed. Hand-listing is how Project ended up with four columns
+     * (utmId and its GA4 siblings) that the schema had but nothing ever wrote.
+     */
+    const projectRow: JSONObject = data.data as unknown as JSONObject;
+    const userRow: JSONObject = user as unknown as JSONObject;
+
+    for (const propertyKey of [...UtmPropertyKeys, UtmUrlPropertyKey]) {
+      projectRow[propertyKey] = userRow[propertyKey];
+    }
 
     // Ad attribution info (click IDs + first touch).
     data.data.clickIds = user.clickIds!;
@@ -572,6 +586,10 @@ export class ProjectService extends DatabaseService<Model> {
         utmCampaign: true,
         utmTerm: true,
         utmContent: true,
+        utmId: true,
+        utmSourcePlatform: true,
+        utmCreativeFormat: true,
+        utmMarketingTactic: true,
         utmUrl: true,
         clickIds: true,
         firstTouchAttribution: true,
@@ -950,11 +968,7 @@ These are no longer recorded against the project and have to be cancelled by han
        */
       is_paid_conversion: true,
       has_custom_pricing: data.plan.isCustomPricing(),
-      utm_source: data.project.utmSource || "",
-      utm_medium: data.project.utmMedium || "",
-      utm_campaign: data.project.utmCampaign || "",
-      utm_term: data.project.utmTerm || "",
-      utm_content: data.project.utmContent || "",
+      ...utmAnalyticsProperties(data.project as unknown as JSONObject),
       click_ids: data.project.clickIds || {},
     };
 
@@ -1065,9 +1079,7 @@ These are no longer recorded against the project and have to be cancelled by han
         oldPlanOrder !== null && newPlanOrder === oldPlanOrder,
       is_paid_conversion: oldMonthlyAmountInUSD === 0 && newPlanIsPaid,
       has_custom_pricing: data.newPlan.isCustomPricing(),
-      utm_source: data.project.utmSource || "",
-      utm_medium: data.project.utmMedium || "",
-      utm_campaign: data.project.utmCampaign || "",
+      ...utmAnalyticsProperties(data.project as unknown as JSONObject),
       click_ids: data.project.clickIds || {},
     };
 
@@ -1436,9 +1448,7 @@ These are no longer recorded against the project and have to be cancelled by han
           project_id: createdItem.id?.toString() || "",
           project_name: createdItem.name?.toString() || "",
           plan: createdItem.planName?.toString() || "",
-          utm_source: createdItem.utmSource || "",
-          utm_medium: createdItem.utmMedium || "",
-          utm_campaign: createdItem.utmCampaign || "",
+          ...utmAnalyticsProperties(createdItem as unknown as JSONObject),
           click_ids: createdItem.clickIds || {},
         },
       });

package/Server/Services/SecurityEventService.ts

@@ -13,10 +13,49 @@ import { getQuerySettings } from "../Utils/AnalyticsDatabase/QuerySettingsHelper
 export interface DetectionMatchGroup {
   groupValue: string;
   matchCount: number;
+  /*
+   * uniqExact of the rule's distinctCountField over the group's matches;
+   * 0 whenever the rule has no distinct-count field (the query emits a
+   * constant, not an aggregate, in that case).
+   */
+  distinctCount: number;
   sampleMessage: string;
   sampleObservables: Array<string>;
 }
 
+export interface FindDetectionMatchesData {
+  projectId: ObjectID;
+  startTime: Date;
+  endTime: Date;
+  whereFragment: Statement;
+  groupByExpression: Statement | null;
+  distinctCountExpression: Statement | null;
+  minMatchCount: number;
+  maxGroups: number;
+}
+
+/*
+ * The in-process twin of the HAVING clause findDetectionMatches emits —
+ * exported so the evaluator's re-filter and the SQL builder live in one
+ * file, and a change to either side of the firing predicate is reviewed
+ * next to the other.
+ */
+export function doesGroupMeetDetectionThreshold(data: {
+  group: DetectionMatchGroup;
+  usesDistinctCount: boolean;
+  minMatchCount: number;
+}): boolean {
+  if (data.group.matchCount <= 0) {
+    return false;
+  }
+
+  const effectiveCount: number = data.usesDistinctCount
+    ? data.group.distinctCount
+    : data.group.matchCount;
+
+  return effectiveCount >= Math.max(1, data.minMatchCount);
+}
+
 export class SecurityEventService extends AnalyticsDatabaseService<SecurityEvent> {
   public constructor(clickhouseDatabase?: ClickhouseDatabase | undefined) {
     super({ modelType: SecurityEvent, database: clickhouseDatabase });
@@ -24,18 +63,21 @@ export class SecurityEventService extends AnalyticsDatabaseService<SecurityEvent
 
   /*
    * Detection-engine query: count matching events in a window, grouped by
-   * an optional field expression. `whereFragment` and
-   * `groupByExpression` are Statements built by SigmaClickhouseCompiler —
-   * every rule-controlled value inside them is a bound parameter.
+   * an optional field expression. `whereFragment`, `groupByExpression`
+   * and `distinctCountExpression` are Statements built by
+   * SigmaClickhouseCompiler — every rule-controlled value inside them is
+   * a bound parameter.
+   *
+   * When distinctCountExpression is set, the threshold applies to
+   * uniqExact of that field, not the raw match count — and empty values
+   * are excluded via nullIf, so "5 distinct usernames" never counts rows
+   * that carry no username. The threshold is enforced in HAVING so the
+   * maxGroups LIMIT keeps only qualifying groups instead of letting
+   * noisy-but-under-threshold groups crowd out real ones.
    */
-  public async findDetectionMatches(data: {
-    projectId: ObjectID;
-    startTime: Date;
-    endTime: Date;
-    whereFragment: Statement;
-    groupByExpression: Statement | null;
-    maxGroups: number;
-  }): Promise<Array<DetectionMatchGroup>> {
+  public async findDetectionMatches(
+    data: FindDetectionMatchesData,
+  ): Promise<Array<DetectionMatchGroup>> {
     if (!this.database) {
       this.useDefaultDatabase();
     }
@@ -51,8 +93,18 @@ export class SecurityEventService extends AnalyticsDatabaseService<SecurityEvent
       statement.append("''");
     }
 
+    statement.append(" AS groupValue, count() AS matchCount, ");
+
+    if (data.distinctCountExpression) {
+      statement.append("uniqExact(nullIf(");
+      statement.append(data.distinctCountExpression);
+      statement.append(", ''))");
+    } else {
+      statement.append("0");
+    }
+
     statement.append(
-      ` AS groupValue, count() AS matchCount, any(message) AS sampleMessage, arrayDistinct(arrayFlatten(groupArray(20)(observables))) AS sampleObservables FROM ${databaseName}.${this.model.tableName} WHERE `,
+      ` AS distinctCount, any(message) AS sampleMessage, arrayDistinct(arrayFlatten(groupArray(20)(observables))) AS sampleObservables FROM ${databaseName}.${this.model.tableName} WHERE `,
     );
 
     statement.append(
@@ -69,7 +121,31 @@ export class SecurityEventService extends AnalyticsDatabaseService<SecurityEvent
     );
 
     statement.append(data.whereFragment);
-    statement.append(") GROUP BY groupValue ORDER BY matchCount DESC LIMIT ");
+    statement.append(") GROUP BY groupValue");
+
+    /*
+     * The effective count — what the rule's threshold means — is the
+     * distinct count when a distinct expression was given, else the raw
+     * match count. A distinct-count rule always gets a HAVING clause,
+     * even at threshold 1: a group whose matches all lack the counted
+     * field has distinctCount 0 and must not fire.
+     */
+    const effectiveCountColumn: string = data.distinctCountExpression
+      ? "distinctCount"
+      : "matchCount";
+    const minMatchCount: number = Math.max(1, data.minMatchCount);
+
+    if (data.distinctCountExpression || minMatchCount > 1) {
+      statement.append(` HAVING ${effectiveCountColumn} >= `);
+      statement.append(
+        SQL`${{
+          type: TableColumnType.Number,
+          value: minMatchCount,
+        }}`,
+      );
+    }
+
+    statement.append(` ORDER BY ${effectiveCountColumn} DESC LIMIT `);
     statement.append(
       SQL`${{
         type: TableColumnType.Number,
@@ -88,6 +164,7 @@ export class SecurityEventService extends AnalyticsDatabaseService<SecurityEvent
       return {
         groupValue: String(row["groupValue"] ?? ""),
         matchCount: Number(row["matchCount"] || 0),
+        distinctCount: Number(row["distinctCount"] || 0),
         sampleMessage: String(row["sampleMessage"] ?? ""),
         sampleObservables: Array.isArray(row["sampleObservables"])
           ? (row["sampleObservables"] as Array<unknown>).map(