@oneuptime/common 11.7.4 → 12.0.0

package/Server/Services/NetworkSiteAssignmentRuleService.ts

@@ -1,18 +1,73 @@
 import DatabaseService from "./DatabaseService";
+import NetworkSiteService from "./NetworkSiteService";
 import Model from "../../Models/DatabaseModels/NetworkSiteAssignmentRule";
+import NetworkSite from "../../Models/DatabaseModels/NetworkSite";
 import { OnCreate, OnUpdate } from "../Types/Database/Hooks";
 import CreateBy from "../Types/Database/CreateBy";
 import UpdateBy from "../Types/Database/UpdateBy";
 import CaptureSpan from "../Utils/Telemetry/CaptureSpan";
 import LIMIT_MAX from "../../Types/Database/LimitMax";
 import BadDataException from "../../Types/Exception/BadDataException";
+import ObjectID from "../../Types/ObjectID";
 import CidrMatchUtil from "../../Utils/NetworkSite/CidrMatchUtil";
+import RelationIdUtil from "../Utils/Database/RelationIdUtil";
+
+/*
+ * The dashboard posts the `site` relation while server-side callers write the
+ * `siteId` column; read whichever is present. See RelationIdUtil.
+ */
+const SITE_KEYS: Array<string> = ["siteId", "site"];
+
+function readSiteId(data: Record<string, unknown>): ObjectID | null {
+  return RelationIdUtil.read(data, SITE_KEYS);
+}
 
 export class Service extends DatabaseService<Model> {
   public constructor() {
     super(Model);
   }
 
+  /*
+   * The FK behind siteId only requires the NetworkSite row to exist, not that
+   * it belongs to the rule's project. A rule pointing at a foreign site is
+   * both a cross-tenant read (the rules table renders `site.name`) and a rule
+   * that can never assign anything - NetworkDeviceService's own tenancy guard
+   * rejects the resulting device update. Fail at write time instead, where
+   * the user can see it.
+   */
+  private async assertSiteBelongsToProject(data: {
+    siteId: ObjectID;
+    projectId: ObjectID | undefined;
+  }): Promise<void> {
+    if (!data.projectId) {
+      return;
+    }
+
+    const site: NetworkSite | null = await NetworkSiteService.findOneById({
+      id: data.siteId,
+      select: {
+        _id: true,
+        projectId: true,
+      },
+      props: {
+        isRoot: true,
+      },
+    });
+
+    if (!site) {
+      throw new BadDataException("Network site not found.");
+    }
+
+    if (
+      site.projectId &&
+      site.projectId.toString() !== data.projectId.toString()
+    ) {
+      throw new BadDataException(
+        "Network site must belong to the same project.",
+      );
+    }
+  }
+
   /*
    * A rule with neither criterion would match nothing (or, worse, read as
    * "match everything" to a future maintainer) - reject it at write time.
@@ -26,6 +81,17 @@ export class Service extends DatabaseService<Model> {
       hostnamePattern: createBy.data.hostnamePattern,
     });
 
+    const siteId: ObjectID | null = readSiteId(
+      createBy.data as unknown as Record<string, unknown>,
+    );
+
+    if (siteId) {
+      await this.assertSiteBelongsToProject({
+        siteId: siteId,
+        projectId: createBy.data.projectId,
+      });
+    }
+
     return { createBy, carryForward: null };
   }
 
@@ -35,10 +101,14 @@ export class Service extends DatabaseService<Model> {
   ): Promise<OnUpdate<Model>> {
     const dataKeys: Array<string> = Object.keys(updateBy.data || {});
 
-    if (
-      !dataKeys.includes("subnetCidr") &&
-      !dataKeys.includes("hostnamePattern")
-    ) {
+    const isCriteriaChange: boolean =
+      dataKeys.includes("subnetCidr") || dataKeys.includes("hostnamePattern");
+
+    const newSiteId: ObjectID | null = readSiteId(
+      updateBy.data as unknown as Record<string, unknown>,
+    );
+
+    if (!isCriteriaChange && !newSiteId) {
       return { updateBy, carryForward: null };
     }
 
@@ -50,6 +120,7 @@ export class Service extends DatabaseService<Model> {
       query: updateBy.query,
       select: {
         _id: true,
+        projectId: true,
         subnetCidr: true,
         hostnamePattern: true,
       },
@@ -61,14 +132,23 @@ export class Service extends DatabaseService<Model> {
     });
 
     for (const existingRule of existingRules) {
-      this.validateCriteria({
-        subnetCidr: dataKeys.includes("subnetCidr")
-          ? ((updateBy.data as any)["subnetCidr"] as string | null)
-          : existingRule.subnetCidr,
-        hostnamePattern: dataKeys.includes("hostnamePattern")
-          ? ((updateBy.data as any)["hostnamePattern"] as string | null)
-          : existingRule.hostnamePattern,
-      });
+      if (isCriteriaChange) {
+        this.validateCriteria({
+          subnetCidr: dataKeys.includes("subnetCidr")
+            ? ((updateBy.data as any)["subnetCidr"] as string | null)
+            : existingRule.subnetCidr,
+          hostnamePattern: dataKeys.includes("hostnamePattern")
+            ? ((updateBy.data as any)["hostnamePattern"] as string | null)
+            : existingRule.hostnamePattern,
+        });
+      }
+
+      if (newSiteId) {
+        await this.assertSiteBelongsToProject({
+          siteId: newSiteId,
+          projectId: existingRule.projectId,
+        });
+      }
     }
 
     return { updateBy, carryForward: null };

package/Server/Services/NetworkSiteService.ts

@@ -34,12 +34,22 @@ import ObjectID from "../../Types/ObjectID";
 import OneUptimeDate from "../../Types/Date";
 import Text from "../../Types/Text";
 import { Raw } from "typeorm";
+import RelationIdUtil from "../Utils/Database/RelationIdUtil";
 import MaterializedPathUtil from "../../Utils/NetworkSite/MaterializedPathUtil";
 import SiteStatusRollupUtil, {
   DeviceHealthState,
   RollupStatusOption,
 } from "../../Utils/NetworkSite/SiteStatusRollupUtil";
 
+/*
+ * Both spellings of "this site's parent" in a write payload: the dashboard's
+ * site form posts the `parentSite` relation while server-side callers write
+ * the `parentSiteId` column. Watching only the column let a re-parent done
+ * from the UI skip cycle detection, the same-project guard and the subtree
+ * path rebase entirely. See RelationIdUtil.
+ */
+const PARENT_SITE_KEYS: Array<string> = ["parentSiteId", "parentSite"];
+
 /*
  * Carried from onBeforeUpdate to onUpdateSuccess when an update touches
  * parentSiteId, so the subtree rebase knows each site's previous state.
@@ -152,9 +162,18 @@ export class Service extends DatabaseService<Model> {
   ): Promise<OnCreate<Model>> {
     let parentPath: string | null = null;
 
-    if (createBy.data.parentSiteId) {
+    /*
+     * Both spellings: the dashboard's site form posts the `parentSite`
+     * relation, not the `parentSiteId` column. See RelationIdUtil.
+     */
+    const parentSiteId: ObjectID | null = RelationIdUtil.read(
+      createBy.data as unknown as Record<string, unknown>,
+      PARENT_SITE_KEYS,
+    );
+
+    if (parentSiteId) {
       const parent: Model | null = await this.findOneById({
-        id: createBy.data.parentSiteId,
+        id: parentSiteId,
         select: {
           _id: true,
           projectId: true,
@@ -178,9 +197,7 @@ export class Service extends DatabaseService<Model> {
         );
       }
 
-      parentPath = await this.getMaterializedPathForSite(
-        createBy.data.parentSiteId,
-      );
+      parentPath = await this.getMaterializedPathForSite(parentSiteId);
     }
 
     return {
@@ -228,14 +245,14 @@ export class Service extends DatabaseService<Model> {
   ): Promise<OnUpdate<Model>> {
     const dataKeys: Array<string> = Object.keys(updateBy.data || {});
 
-    if (!dataKeys.includes("parentSiteId")) {
+    if (!RelationIdUtil.isWritten(dataKeys, PARENT_SITE_KEYS)) {
       return { updateBy, carryForward: null };
     }
 
-    const newParentIdValue: unknown = (updateBy.data as any)["parentSiteId"];
-    const newParentId: ObjectID | null = newParentIdValue
-      ? new ObjectID(newParentIdValue.toString())
-      : null;
+    const newParentId: ObjectID | null = RelationIdUtil.read(
+      updateBy.data as unknown as Record<string, unknown>,
+      PARENT_SITE_KEYS,
+    );
 
     const previousItems: Array<Model> = await this.findBy({
       query: this.scopeQueryToCallerTenant(updateBy.query, updateBy.props),

package/Server/Services/ProjectService.ts

@@ -16,6 +16,7 @@ import UpdateBy from "../Types/Database/UpdateBy";
 import logger, { LogAttributes } from "../Utils/Logger";
 import Errors from "../Utils/Errors";
 import ProductAnalytics from "../Utils/ProductAnalytics";
+import SessionReplayGateCacheStore from "../Utils/SessionReplay/SessionReplayGateCacheStore";
 import AccessTokenService from "./AccessTokenService";
 import BillingService from "./BillingService";
 import DatabaseService from "./DatabaseService";
@@ -360,6 +361,52 @@ export class ProjectService extends DatabaseService<Model> {
     return Promise.resolve({ createBy: data, carryForward: null });
   }
 
+  /*
+   * Session replay is gated on the project's org-wide allow flag, and the
+   * ingest gate caches the resolved policy per pod. Turning the flag off has
+   * to reach every pod now, not after the cache TTL plus the config
+   * endpoint's browser cache - roughly six minutes - so the Redis kill key
+   * is written here. Best effort: a Redis outage only means the ordinary
+   * cache expiry applies.
+   */
+  @CaptureSpan()
+  protected override async onUpdateSuccess(
+    onUpdate: OnUpdate<Model>,
+    updatedItemIds: Array<ObjectID>,
+  ): Promise<OnUpdate<Model>> {
+    const updateData: Record<string, unknown> = onUpdate.updateBy
+      .data as unknown as Record<string, unknown>;
+
+    if (!("isSessionReplayAllowed" in updateData)) {
+      return onUpdate;
+    }
+
+    const isTurningOff: boolean = updateData["isSessionReplayAllowed"] !== true;
+
+    for (const projectId of updatedItemIds) {
+      try {
+        SessionReplayGateCacheStore.clearCache(projectId);
+
+        if (isTurningOff) {
+          await SessionReplayGateCacheStore.markProjectDisabled(projectId);
+        } else {
+          /*
+           * Re-enabling has to drop a kill key left over from an earlier
+           * disable, or the project stays refused for the rest of its TTL.
+           */
+          await SessionReplayGateCacheStore.clearProjectDisabled(projectId);
+        }
+      } catch (err) {
+        logger.warn(
+          `ProjectService: could not invalidate the session replay gate cache for project ${projectId.toString()}`,
+        );
+        logger.warn(err);
+      }
+    }
+
+    return onUpdate;
+  }
+
   @CaptureSpan()
   protected override async onBeforeUpdate(
     updateBy: UpdateBy<Model>,

package/Server/Services/RumApplicationService.ts

@@ -7,9 +7,10 @@ import QueryHelper from "../Types/Database/QueryHelper";
 import OneUptimeDate from "../../Types/Date";
 import LIMIT_MAX from "../../Types/Database/LimitMax";
 import GlobalCache from "../Infrastructure/GlobalCache";
+import SessionReplayGateCacheStore from "../Utils/SessionReplay/SessionReplayGateCacheStore";
 import logger, { LogAttributes } from "../Utils/Logger";
 import crypto from "crypto";
-import { OnCreate } from "../Types/Database/Hooks";
+import { OnCreate, OnDelete, OnUpdate } from "../Types/Database/Hooks";
 import RumApplicationLabelRuleEngineService from "./RumApplicationLabelRuleEngineService";
 import RumApplicationOwnerRuleEngineService from "./RumApplicationOwnerRuleEngineService";
 
@@ -19,6 +20,14 @@ const LAST_SEEN_THROTTLE_SECONDS: number = 60;
 const LABELS_APPLIED_CACHE_NAMESPACE: string = "rum-application-labels-applied";
 const LABELS_APPLIED_CACHE_TTL_SECONDS: number = 60;
 
+const REPLAY_CHUNK_RECEIVED_CACHE_NAMESPACE: string =
+  "rum-application-replay-chunk-received";
+const REPLAY_CHUNK_RECEIVED_THROTTLE_SECONDS: number = 60;
+
+const REPLAY_BUDGET_EXCEEDED_CACHE_NAMESPACE: string =
+  "rum-application-replay-budget-exceeded";
+const REPLAY_BUDGET_EXCEEDED_THROTTLE_SECONDS: number = 300;
+
 export class Service extends DatabaseService<Model> {
   public constructor() {
     super(Model);
@@ -54,6 +63,144 @@ export class Service extends DatabaseService<Model> {
     return createdItem;
   }
 
+  /*
+   * Session replay policy invalidation.
+   *
+   * The ingest gate caches the resolved policy for POLICY_CACHE_TTL_MS per
+   * pod, and the config endpoint is cached in the browser for 300s on top of
+   * that. Without these hooks, switching replay off for an application would
+   * keep being accepted for up to ~6 minutes across the fleet - so the kill
+   * key exists precisely to be written here.
+   *
+   * The kill key is per PROJECT, which is coarser than the per-application
+   * change that triggered it: disabling one application in a multi-app
+   * project briefly refuses the project's other applications too. That is
+   * accepted deliberately. The window is KILL_KEY_CACHE_TTL_MS plus the
+   * policy TTL, the failure mode is "a few recordings not taken", and the
+   * alternative - keying on projectId:appIdentifier - would leave the
+   * project-level `isSessionReplayAllowed` switch with no fast path at all.
+   */
+  @CaptureSpan()
+  protected override async onUpdateSuccess(
+    onUpdate: OnUpdate<Model>,
+    updatedItemIds: Array<ObjectID>,
+  ): Promise<OnUpdate<Model>> {
+    await this.invalidateSessionReplayPolicy({
+      updateData: onUpdate.updateBy.data as Record<string, unknown>,
+      itemIds: updatedItemIds,
+      tenantId: onUpdate.updateBy.props.tenantId,
+    });
+
+    return onUpdate;
+  }
+
+  @CaptureSpan()
+  protected override async onDeleteSuccess(
+    onDelete: OnDelete<Model>,
+    itemIdsBeforeDelete: Array<ObjectID>,
+  ): Promise<OnDelete<Model>> {
+    /*
+     * A deleted application can never be re-enabled, so this always takes
+     * the "turn it off now" path rather than only clearing the cache.
+     */
+    await this.invalidateSessionReplayPolicy({
+      updateData: { isSessionReplayEnabled: false },
+      itemIds: itemIdsBeforeDelete,
+      tenantId: onDelete.deleteBy.props.tenantId,
+    });
+
+    return onDelete;
+  }
+
+  private async invalidateSessionReplayPolicy(data: {
+    updateData: Record<string, unknown>;
+    itemIds: Array<ObjectID>;
+    tenantId: ObjectID | undefined;
+  }): Promise<void> {
+    /*
+     * Only replay-relevant edits invalidate. A lastSeenAt heartbeat or a
+     * label change must not write a kill key, and updateLastSeen bypasses
+     * hooks entirely for the same reason.
+     */
+    const touchesReplayPolicy: boolean = Object.keys(data.updateData).some(
+      (key: string): boolean => {
+        return (
+          key.startsWith("sessionReplay") || key === "isSessionReplayEnabled"
+        );
+      },
+    );
+
+    if (!touchesReplayPolicy) {
+      return;
+    }
+
+    const projectIds: Array<ObjectID> = await this.resolveProjectIds(data);
+
+    const isTurningOff: boolean =
+      data.updateData["isSessionReplayEnabled"] === false;
+
+    for (const projectId of projectIds) {
+      try {
+        SessionReplayGateCacheStore.clearCache(projectId);
+
+        if (isTurningOff) {
+          await SessionReplayGateCacheStore.markProjectDisabled(projectId);
+        } else {
+          /*
+           * Any other replay edit (including turning it back ON) must also
+           * drop a kill key left over from an earlier disable, or the
+           * project stays refused for the remainder of the key's TTL.
+           */
+          await SessionReplayGateCacheStore.clearProjectDisabled(projectId);
+        }
+      } catch (err) {
+        /*
+         * Best effort. The ordinary cache expiry still applies, so a Redis
+         * outage degrades the response time of a settings change rather
+         * than failing the settings change itself.
+         */
+        logger.warn(
+          `RumApplicationService: could not invalidate the session replay gate cache for project ${projectId.toString()}`,
+        );
+        logger.warn(err);
+      }
+    }
+  }
+
+  private async resolveProjectIds(data: {
+    itemIds: Array<ObjectID>;
+    tenantId: ObjectID | undefined;
+  }): Promise<Array<ObjectID>> {
+    if (data.tenantId) {
+      return [data.tenantId];
+    }
+
+    /*
+     * A root-props update carries no tenantId, so the project has to be read
+     * back off the rows that were touched.
+     */
+    const projectIds: Map<string, ObjectID> = new Map();
+
+    for (const id of data.itemIds) {
+      const app: Model | null = await this.findOneById({
+        id: id,
+        select: {
+          _id: true,
+          projectId: true,
+        },
+        props: {
+          isRoot: true,
+        },
+      });
+
+      if (app?.projectId) {
+        projectIds.set(app.projectId.toString(), app.projectId);
+      }
+    }
+
+    return Array.from(projectIds.values());
+  }
+
   @CaptureSpan()
   public async findOrCreateByAppIdentifier(data: {
     projectId: ObjectID;
@@ -196,6 +343,98 @@ export class Service extends DatabaseService<Model> {
     });
   }
 
+  /*
+   * Session replay ingest health markers.
+   *
+   * Both write columns whose names start with "sessionReplay", which the
+   * onUpdateSuccess hook treats as a policy edit (cache clear + kill-key
+   * churn) - so they MUST go through the hook-free single-statement path,
+   * exactly like updateLastSeen. They also skip the updatedAt refresh:
+   * configEpoch is derived from updatedAt, and a passive health timestamp
+   * broadcasting "configuration changed" to every live recorder once a
+   * minute would be self-inflicted config-fetch load.
+   *
+   * Throttled through GlobalCache so 20k chunks/min becomes at most one
+   * Postgres write per application per window across the whole fleet.
+   * Failures are swallowed after a warn: this bookkeeping must never make
+   * an ingest request fail.
+   */
+  @CaptureSpan()
+  public async markSessionReplayChunkReceived(
+    rumApplicationId: ObjectID,
+  ): Promise<void> {
+    await this.writeReplayHealthColumn({
+      rumApplicationId: rumApplicationId,
+      cacheNamespace: REPLAY_CHUNK_RECEIVED_CACHE_NAMESPACE,
+      throttleSeconds: REPLAY_CHUNK_RECEIVED_THROTTLE_SECONDS,
+      column: "sessionReplayLastChunkReceivedAt",
+    });
+  }
+
+  @CaptureSpan()
+  public async markSessionReplayBudgetExceeded(
+    rumApplicationId: ObjectID,
+  ): Promise<void> {
+    await this.writeReplayHealthColumn({
+      rumApplicationId: rumApplicationId,
+      cacheNamespace: REPLAY_BUDGET_EXCEEDED_CACHE_NAMESPACE,
+      throttleSeconds: REPLAY_BUDGET_EXCEEDED_THROTTLE_SECONDS,
+      column: "sessionReplayBudgetExceededAt",
+    });
+  }
+
+  private async writeReplayHealthColumn(data: {
+    rumApplicationId: ObjectID;
+    cacheNamespace: string;
+    throttleSeconds: number;
+    column:
+      | "sessionReplayLastChunkReceivedAt"
+      | "sessionReplayBudgetExceededAt";
+  }): Promise<void> {
+    const cacheKey: string = data.rumApplicationId.toString();
+
+    try {
+      const cached: string | null = await GlobalCache.getString(
+        data.cacheNamespace,
+        cacheKey,
+      );
+
+      if (cached) {
+        return; // written recently, across all pods
+      }
+    } catch {
+      /*
+       * Cache unavailable - fail open and write anyway, same stance as
+       * updateLastSeen: a stale health column is worse than an occasional
+       * extra single-row UPDATE.
+       */
+    }
+
+    try {
+      await GlobalCache.setString(data.cacheNamespace, cacheKey, "1", {
+        expiresInSeconds: data.throttleSeconds,
+      });
+    } catch {
+      // Best-effort throttle write; proceed with the DB update regardless.
+    }
+
+    try {
+      await this.updateColumnsByIdWithoutHooks({
+        id: data.rumApplicationId,
+        data: {
+          [data.column]: OneUptimeDate.getCurrentDate(),
+          // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        } as any,
+        skipUpdateDateColumn: true,
+      });
+    } catch (err) {
+      logger.warn(
+        `RumApplicationService: could not write ${data.column} for application ${cacheKey}`,
+      );
+      logger.warn(err);
+    }
+  }
+
   @CaptureSpan()
   public async attachLabels(data: {
     rumApplicationId: ObjectID;

package/Server/Services/RumSessionChunkService.ts

@@ -0,0 +1,11 @@
+import ClickhouseDatabase from "../Infrastructure/ClickhouseDatabase";
+import AnalyticsDatabaseService from "./AnalyticsDatabaseService";
+import RumSessionChunk from "../../Models/AnalyticsModels/RumSessionChunk";
+
+export class RumSessionChunkService extends AnalyticsDatabaseService<RumSessionChunk> {
+  public constructor(clickhouseDatabase?: ClickhouseDatabase | undefined) {
+    super({ modelType: RumSessionChunk, database: clickhouseDatabase });
+  }
+}
+
+export default new RumSessionChunkService();