@oneuptime/common 10.4.14 → 10.4.16

package/Server/API/BaseAnalyticsAPI.ts

@@ -29,6 +29,22 @@ import { UserPermission } from "../../Types/Permission";
 import PositiveNumber from "../../Types/PositiveNumber";
 import AggregatedResult from "../../Types/BaseDatabase/AggregatedResult";
 import CaptureSpan from "../Utils/Telemetry/CaptureSpan";
+import GlobalCache from "../Infrastructure/GlobalCache";
+import logger from "../Utils/Logger";
+
+/*
+ * Aggregate cache TTL. Dashboards typically auto-refresh every 30s+,
+ * so an 8s window collapses bursts of identical requests (e.g. 12
+ * widgets loading on the same page) onto a single ClickHouse query
+ * while still looking real-time to humans.
+ *
+ * Project-scoped only: analytics data is project-wide and the
+ * service layer enforces project-scoped read permissions, so
+ * caching across users within the same project is safe. Endpoints
+ * with row-level access scoping should override `getAggregate` to
+ * skip the cache (or shape the key to include the access scope).
+ */
+const ANALYTICS_AGGREGATE_CACHE_TTL_SECONDS: number = 8;
 
 export default class BaseAnalyticsAPI<
   TAnalyticsDataModel extends AnalyticsDataModel,
@@ -268,29 +284,46 @@ export default class BaseAnalyticsAPI<
     const databaseProps: DatabaseCommonInteractionProps =
       await CommonAPI.getDatabaseCommonInteractionProps(req);
 
-    const [list, count] = await Promise.all([
-      this.service.findBy({
-        query,
-        select,
-        skip: skip,
-        limit: limit,
-        sort: sort,
-        groupBy: groupBy,
-        props: databaseProps,
-      }),
-      this.service.countBy({
-        query,
-        groupBy: groupBy,
-        props: databaseProps,
-      }),
-    ]);
+    /*
+     * Skip the parallel countBy on analytics tables. countBy on Log /
+     * Span / Metric over wide time ranges scans every matching block
+     * (no LIMIT) and routinely dominates list-endpoint latency under
+     * heavy ingest. Instead we over-fetch by one row and derive
+     * `hasMore` from whether the extra row showed up. `count` is
+     * emitted as a lower bound (`skip + data.length + hasMore`) so
+     * older clients that read `count` keep rendering something
+     * sensible while newer clients use `hasMore` for prev/next.
+     */
+    const overfetchLimit: PositiveNumber = new PositiveNumber(
+      limit.toNumber() + 1,
+    );
+
+    const list: Array<AnalyticsDataModel> = await this.service.findBy({
+      query,
+      select,
+      skip: skip,
+      limit: overfetchLimit,
+      sort: sort,
+      groupBy: groupBy,
+      props: databaseProps,
+    });
+
+    const hasMore: boolean = list.length > limit.toNumber();
+    if (hasMore) {
+      list.length = limit.toNumber();
+    }
+
+    const lowerBoundCount: PositiveNumber = new PositiveNumber(
+      skip.toNumber() + list.length + (hasMore ? 1 : 0),
+    );
 
     return Response.sendEntityArrayResponse(
       req,
       res,
       list,
-      count,
+      lowerBoundCount,
       this.entityType,
+      { hasMore },
     );
   }
 
@@ -327,14 +360,89 @@ export default class BaseAnalyticsAPI<
     const databaseProps: DatabaseCommonInteractionProps =
       await CommonAPI.getDatabaseCommonInteractionProps(req);
 
+    /*
+     * Short-lived project-scoped cache. A dashboard refresh fires
+     * one /aggregate call per widget — typically 10+ identical or
+     * near-identical aggregations against the same time window
+     * inside a few hundred milliseconds. Cache the result for 8s
+     * so the underlying ClickHouse aggregation runs once per
+     * burst. On cache outage (Redis down, parse error, …) we fall
+     * through to a live query so behavior degrades to today's.
+     */
+    const projectId: string | undefined = databaseProps.tenantId?.toString();
+    const cacheNamespace: string = `${this.getEntityName()}-aggregate`;
+    const cacheKey: string | null = projectId
+      ? `${projectId}:${this.buildAggregateCacheKey(aggregateBy)}`
+      : null;
+
+    if (cacheKey) {
+      try {
+        const cached: JSONObject | null = await GlobalCache.getJSONObject(
+          cacheNamespace,
+          cacheKey,
+        );
+        if (cached) {
+          return Response.sendJsonObjectResponse(req, res, cached);
+        }
+      } catch (err) {
+        logger.debug(`${cacheNamespace} cache read failed`);
+        logger.debug(err);
+      }
+    }
+
     const aggregateResult: AggregatedResult = await this.service.aggregateBy({
       ...aggregateBy,
       props: databaseProps,
     });
 
-    return Response.sendJsonObjectResponse(req, res, {
-      ...(aggregateResult as any),
-    });
+    const responseBody: JSONObject = { ...(aggregateResult as any) };
+
+    if (cacheKey) {
+      try {
+        await GlobalCache.setJSON(cacheNamespace, cacheKey, responseBody, {
+          expiresInSeconds: ANALYTICS_AGGREGATE_CACHE_TTL_SECONDS,
+        });
+      } catch (err) {
+        logger.debug(`${cacheNamespace} cache write failed`);
+        logger.debug(err);
+      }
+    }
+
+    return Response.sendJsonObjectResponse(req, res, responseBody);
+  }
+
+  /*
+   * Stable serialization for the aggregate cache key. Date instances
+   * are normalized to ISO so two logically-equal time windows hit
+   * the same cache slot, and we sort object keys so the ordering is
+   * deterministic across clients and across V8 versions.
+   */
+  protected buildAggregateCacheKey(
+    aggregateBy: AggregateBy<AnalyticsDataModel>,
+  ): string {
+    return JSON.stringify(
+      aggregateBy,
+      (_key: string, value: unknown): unknown => {
+        if (value instanceof Date) {
+          return value.toISOString();
+        }
+        if (
+          value &&
+          typeof value === "object" &&
+          !Array.isArray(value) &&
+          (value as Record<string, unknown>).constructor === Object
+        ) {
+          const sorted: Record<string, unknown> = {};
+          for (const k of Object.keys(
+            value as Record<string, unknown>,
+          ).sort()) {
+            sorted[k] = (value as Record<string, unknown>)[k];
+          }
+          return sorted;
+        }
+        return value;
+      },
+    );
   }
 
   @CaptureSpan()

package/Server/API/MetricAPI.ts

@@ -1,149 +1,16 @@
-import AggregateBy from "../../Types/BaseDatabase/AggregateBy";
-import AggregatedResult from "../../Types/BaseDatabase/AggregatedResult";
-import DatabaseCommonInteractionProps from "../../Types/BaseDatabase/DatabaseCommonInteractionProps";
-import BadRequestException from "../../Types/Exception/BadRequestException";
-import { JSONObject } from "../../Types/JSON";
-import JSONFunctions from "../../Types/JSONFunctions";
 import Metric from "../../Models/AnalyticsModels/Metric";
 import { MetricService } from "../Services/MetricService";
-import GlobalCache from "../Infrastructure/GlobalCache";
-import logger from "../Utils/Logger";
-import CaptureSpan from "../Utils/Telemetry/CaptureSpan";
-import { ExpressRequest, ExpressResponse } from "../Utils/Express";
-import Response from "../Utils/Response";
-import CommonAPI from "./CommonAPI";
 import BaseAnalyticsAPI from "./BaseAnalyticsAPI";
 
 /*
- * Aggregate cache TTL. Dashboards typically auto-refresh every 30s+, so
- * an 8s window collapses bursts of identical requests (e.g. 12 widgets
- * loading on the same page) onto a single ClickHouse query while still
- * looking real-time to humans.
+ * Metric CRUD + aggregate endpoints. The 8-second project-scoped
+ * aggregate cache that used to live here has been promoted to
+ * `BaseAnalyticsAPI.getAggregate` so Log/Span/AuditLog/etc. benefit
+ * from the same dashboard-widget-burst collapse without duplicating
+ * the wrapper on every analytics API.
  */
-const AGGREGATE_CACHE_NAMESPACE: string = "metric-aggregate";
-const AGGREGATE_CACHE_TTL_SECONDS: number = 8;
-
 export default class MetricAPI extends BaseAnalyticsAPI<Metric, MetricService> {
   public constructor(service: MetricService) {
     super(Metric, service);
   }
-
-  /*
-   * Cached override of BaseAnalyticsAPI.getAggregate.
-   *
-   * Why a cache: each chart/value/gauge/table widget on a dashboard
-   * issues its own /aggregate call. With 10+ widgets and a small group
-   * of users hitting the same dashboard the underlying ClickHouse
-   * cluster sees the same heavy aggregation many times in close
-   * succession. Aggregations are read-only and pure (same input ->
-   * same output for the bucket interval), so a brief result cache is
-   * safe.
-   *
-   * Cache key: tenant project + the deserialized aggregateBy payload.
-   * We must include the project so cross-tenant collisions cannot
-   * leak data; we deliberately do NOT key on user id, because the
-   * service layer applies project-scoped read permissions and metric
-   * data is project-wide.
-   *
-   * Cache miss / Redis down: we fall through to the live query, so
-   * cache outages degrade to today's behavior, never error.
-   */
-  @CaptureSpan()
-  public override async getAggregate(
-    req: ExpressRequest,
-    res: ExpressResponse,
-  ): Promise<void> {
-    await this.onBeforeList(req, res);
-
-    let aggregateBy: AggregateBy<Metric> | null = null;
-
-    if (req.body && req.body["aggregateBy"]) {
-      aggregateBy = JSONFunctions.deserialize(
-        req.body["aggregateBy"] as JSONObject,
-      ) as any;
-    }
-
-    if (!aggregateBy) {
-      throw new BadRequestException("AggregateBy is required");
-    }
-
-    const databaseProps: DatabaseCommonInteractionProps =
-      await CommonAPI.getDatabaseCommonInteractionProps(req);
-
-    const projectId: string | undefined = databaseProps.tenantId?.toString();
-    const cacheKey: string | null = projectId
-      ? `${projectId}:${this.buildCacheKey(aggregateBy)}`
-      : null;
-
-    if (cacheKey) {
-      try {
-        const cached: JSONObject | null = await GlobalCache.getJSONObject(
-          AGGREGATE_CACHE_NAMESPACE,
-          cacheKey,
-        );
-        if (cached) {
-          return Response.sendJsonObjectResponse(req, res, cached);
-        }
-      } catch (err) {
-        // Cache fetch failed — fall through to a live query.
-        logger.debug("MetricAPI aggregate cache read failed");
-        logger.debug(err);
-      }
-    }
-
-    const aggregateResult: AggregatedResult = await this.service.aggregateBy({
-      ...aggregateBy,
-      props: databaseProps,
-    });
-
-    const responseBody: JSONObject = { ...(aggregateResult as any) };
-
-    if (cacheKey) {
-      try {
-        await GlobalCache.setJSON(
-          AGGREGATE_CACHE_NAMESPACE,
-          cacheKey,
-          responseBody,
-          { expiresInSeconds: AGGREGATE_CACHE_TTL_SECONDS },
-        );
-      } catch (err) {
-        logger.debug("MetricAPI aggregate cache write failed");
-        logger.debug(err);
-      }
-    }
-
-    return Response.sendJsonObjectResponse(req, res, responseBody);
-  }
-
-  private buildCacheKey(aggregateBy: AggregateBy<Metric>): string {
-    /*
-     * Stable serialization. Date instances are normalized to ISO so two
-     * logically-equal time windows hit the same cache slot, and we sort
-     * keys via JSON.stringify replacer to keep ordering deterministic
-     * across clients and across versions of V8.
-     */
-    return JSON.stringify(
-      aggregateBy,
-      (_key: string, value: unknown): unknown => {
-        if (value instanceof Date) {
-          return value.toISOString();
-        }
-        if (
-          value &&
-          typeof value === "object" &&
-          !Array.isArray(value) &&
-          (value as Record<string, unknown>).constructor === Object
-        ) {
-          const sorted: Record<string, unknown> = {};
-          for (const k of Object.keys(
-            value as Record<string, unknown>,
-          ).sort()) {
-            sorted[k] = (value as Record<string, unknown>)[k];
-          }
-          return sorted;
-        }
-        return value;
-      },
-    );
-  }
 }

package/Server/API/StatusAPI.ts

@@ -1,4 +1,5 @@
 import BadRequestException from "../../Types/Exception/BadRequestException";
+import InMemoryTTLCache from "../Infrastructure/InMemoryTTLCache";
 import LocalCache from "../Infrastructure/LocalCache";
 import Express, {
   ExpressRequest,
@@ -20,7 +21,93 @@ export interface StatusAPIOptions {
   databaseCheck?: (() => Promise<void>) | undefined;
 }
 
+/**
+ * Result of a recently executed health check, cached for HEALTH_CHECK_CACHE_TTL_MS.
+ * We cache both success AND failure: caching failure protects an already
+ * unhealthy backend from being hammered by retry traffic during an outage. The
+ * 5s TTL is short enough that k8s probe semantics (default 10s interval,
+ * failureThreshold 3 → ~30s to unready) are essentially unchanged.
+ */
+type CachedHealthCheckResult = { ok: true } | { ok: false; error: Error };
+
 export default class StatusAPI {
+  /**
+   * Cache of recent health check results, keyed by check name. Each entry
+   * lives for HEALTH_CHECK_CACHE_TTL_MS. Bounded to a small max size — there
+   * are only ~5 distinct check names in this API.
+   */
+  private static checkResultCache: InMemoryTTLCache<CachedHealthCheckResult> =
+    new InMemoryTTLCache<CachedHealthCheckResult>(64);
+
+  /**
+   * In-flight check promises keyed by check name. When a cache miss occurs
+   * and multiple concurrent requests arrive, they all attach to the same
+   * promise instead of each triggering their own DB query. The entry is
+   * cleared as soon as the check settles.
+   */
+  private static inflightChecks: Map<string, Promise<void>> = new Map();
+
+  /**
+   * Cache TTL for health-check results. Chosen so that:
+   *   - Two-thirds of typical k8s probes (default periodSeconds=10) hit
+   *     the cache, removing constant DB load from liveness/readiness traffic.
+   *   - Time-to-detect for a failing dependency only grows by ≤5s, which is
+   *     well within the failureThreshold window k8s probes already tolerate.
+   */
+  private static readonly HEALTH_CHECK_CACHE_TTL_MS: number = 5000;
+
+  /**
+   * Runs `checkFn` with two layers of protection:
+   *   1. TTL cache — if the same check ran in the last HEALTH_CHECK_CACHE_TTL_MS
+   *      ms, reuse its result (success or failure) without re-running.
+   *   2. Single-flight — if a check is already in flight, concurrent callers
+   *      await the same promise instead of starting their own.
+   *
+   * On cache hit this is effectively free; on cache miss we run the check
+   * exactly once regardless of how many requests arrived concurrently.
+   */
+  private static async runCachedCheck(
+    checkName: string,
+    checkFn: () => Promise<void>,
+  ): Promise<void> {
+    const cached: CachedHealthCheckResult | undefined =
+      this.checkResultCache.get(checkName);
+    if (cached) {
+      if (cached.ok) {
+        return;
+      }
+      throw cached.error;
+    }
+
+    let inflight: Promise<void> | undefined =
+      this.inflightChecks.get(checkName);
+    if (!inflight) {
+      inflight = (async (): Promise<void> => {
+        try {
+          await checkFn();
+          this.checkResultCache.set(
+            checkName,
+            { ok: true },
+            this.HEALTH_CHECK_CACHE_TTL_MS,
+          );
+        } catch (e) {
+          const error: Error = e instanceof Error ? e : new Error(String(e));
+          this.checkResultCache.set(
+            checkName,
+            { ok: false, error },
+            this.HEALTH_CHECK_CACHE_TTL_MS,
+          );
+          throw error;
+        } finally {
+          this.inflightChecks.delete(checkName);
+        }
+      })();
+      this.inflightChecks.set(checkName, inflight);
+    }
+
+    await inflight;
+  }
+
   @CaptureSpan()
   public static init(options: StatusAPIOptions): ExpressRouter {
     const statusCheckSuccessCounter: TelemetryCounter = Telemetry.getCounter({
@@ -127,8 +214,11 @@ export default class StatusAPI {
     res: ExpressResponse,
   ): Promise<void> {
     try {
-      logger.info("Ready check: Init", getLogAttributesFromRequest(req as any));
-      await options.readyCheck();
+      /*
+       * Cached for HEALTH_CHECK_CACHE_TTL_MS so k8s probe traffic does not
+       * hammer the underlying check on every request.
+       */
+      await this.runCachedCheck("ready", options.readyCheck);
       logger.info("Ready check: ok", getLogAttributesFromRequest(req as any));
       stausReadySuccess.add(1);
 
@@ -160,8 +250,8 @@ export default class StatusAPI {
     res: ExpressResponse,
   ): Promise<void> {
     try {
-      logger.info("Live check: Init", getLogAttributesFromRequest(req as any));
-      await options.liveCheck();
+      // Cached for HEALTH_CHECK_CACHE_TTL_MS — see runCachedCheck for rationale.
+      await this.runCachedCheck("live", options.liveCheck);
       logger.info("Live check: ok", getLogAttributesFromRequest(req as any));
       stausLiveSuccess.add(1);
 
@@ -195,7 +285,8 @@ export default class StatusAPI {
         getLogAttributesFromRequest(req as any),
       );
       if (options.globalCacheCheck) {
-        await options.globalCacheCheck();
+        // Cached — see runCachedCheck for rationale.
+        await this.runCachedCheck("global-cache", options.globalCacheCheck);
       } else {
         throw new BadRequestException("Global cache check not implemented");
       }
@@ -230,7 +321,11 @@ export default class StatusAPI {
         getLogAttributesFromRequest(req as any),
       );
       if (options.analyticsDatabaseCheck) {
-        await options.analyticsDatabaseCheck();
+        // Cached — see runCachedCheck for rationale.
+        await this.runCachedCheck(
+          "analytics-database",
+          options.analyticsDatabaseCheck,
+        );
       } else {
         throw new BadRequestException(
           "Analytics database check not implemented",
@@ -265,7 +360,8 @@ export default class StatusAPI {
       logger.debug("Database check", getLogAttributesFromRequest(req as any));
 
       if (options.databaseCheck) {
-        await options.databaseCheck();
+        // Cached — see runCachedCheck for rationale.
+        await this.runCachedCheck("database", options.databaseCheck);
       } else {
         throw new BadRequestException("Database check not implemented");
       }

package/Server/Infrastructure/Postgres/SchemaMigrations/1779536271671-AddFacetsToTableView.ts

@@ -0,0 +1,13 @@
+import { MigrationInterface, QueryRunner } from "typeorm";
+
+export class AddFacetsToTableView1779536271671 implements MigrationInterface {
+  public name: string = "AddFacetsToTableView1779536271671";
+
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(`ALTER TABLE "TableView" ADD "facets" jsonb`);
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(`ALTER TABLE "TableView" DROP COLUMN "facets"`);
+  }
+}

package/Server/Infrastructure/Postgres/SchemaMigrations/1779540427366-AddIsMemberNotifiedIndex.ts

@@ -0,0 +1,34 @@
+import { MigrationInterface, QueryRunner } from "typeorm";
+
+/*
+ * Adds an index on `IncidentMember.isMemberNotified`. The
+ * `IncidentMembers/SendMemberAddedNotification` cron polls
+ * `WHERE isMemberNotified = false` every minute; without the
+ * index that's a full table scan that scales with every
+ * incident-member assignment ever made.
+ *
+ * The autogenerated diff also picked up unrelated drift between
+ * the local dev database and the current entity models (drops
+ * of `User.themePreference`, `StatusPage.showGridView`,
+ * `StatusPageResource.statusPageGridColumnId`, and ALTER COLUMN
+ * default changes on `OnCallDutyPolicyScheduleLayer`). Those
+ * changes belong to other branches and must not run here — they
+ * were stripped to keep this migration tightly scoped.
+ */
+export class AddIsMemberNotifiedIndex1779540427366
+  implements MigrationInterface
+{
+  public name: string = "AddIsMemberNotifiedIndex1779540427366";
+
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(
+      `CREATE INDEX "IDX_0c2358ce5ea46732d76c4cbb26" ON "IncidentMember" ("isMemberNotified") `,
+    );
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(
+      `DROP INDEX "public"."IDX_0c2358ce5ea46732d76c4cbb26"`,
+    );
+  }
+}

package/Server/Infrastructure/Postgres/SchemaMigrations/1779619108628-AddDropdownOptionsToCustomFields.ts

@@ -0,0 +1,67 @@
+import { MigrationInterface, QueryRunner } from "typeorm";
+
+export class AddDropdownOptionsToCustomFields1779619108628
+  implements MigrationInterface
+{
+  public name: string = "AddDropdownOptionsToCustomFields1779619108628";
+
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(
+      `ALTER TABLE "IncidentCustomField" ADD "dropdownOptions" character varying(500)`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "MonitorCustomField" ADD "dropdownOptions" character varying(500)`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "OnCallDutyPolicyCustomField" ADD "dropdownOptions" character varying(500)`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "ScheduledMaintenanceCustomField" ADD "dropdownOptions" character varying(500)`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "StatusPageCustomField" ADD "dropdownOptions" character varying(500)`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "TeamMemberCustomField" ADD "dropdownOptions" character varying(500)`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "AlertCustomField" ADD "dropdownOptions" character varying(500)`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "OnCallDutyPolicyScheduleLayer" ALTER COLUMN "rotation" SET DEFAULT '{"_type":"Recurring","value":{"intervalType":"Day","intervalCount":{"_type":"PositiveNumber","value":1}}}'`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "OnCallDutyPolicyScheduleLayer" ALTER COLUMN "restrictionTimes" SET DEFAULT '{"_type":"RestrictionTimes","value":{"restictionType":"None","dayRestrictionTimes":null,"weeklyRestrictionTimes":[]}}'`,
+    );
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(
+      `ALTER TABLE "OnCallDutyPolicyScheduleLayer" ALTER COLUMN "restrictionTimes" SET DEFAULT '{"_type": "RestrictionTimes", "value": {"restictionType": "None", "dayRestrictionTimes": null, "weeklyRestrictionTimes": []}}'`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "OnCallDutyPolicyScheduleLayer" ALTER COLUMN "rotation" SET DEFAULT '{"_type": "Recurring", "value": {"intervalType": "Day", "intervalCount": {"_type": "PositiveNumber", "value": 1}}}'`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "AlertCustomField" DROP COLUMN "dropdownOptions"`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "TeamMemberCustomField" DROP COLUMN "dropdownOptions"`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "StatusPageCustomField" DROP COLUMN "dropdownOptions"`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "ScheduledMaintenanceCustomField" DROP COLUMN "dropdownOptions"`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "OnCallDutyPolicyCustomField" DROP COLUMN "dropdownOptions"`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "MonitorCustomField" DROP COLUMN "dropdownOptions"`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "IncidentCustomField" DROP COLUMN "dropdownOptions"`,
+    );
+  }
+}

package/Server/Infrastructure/Postgres/SchemaMigrations/Index.ts

@@ -344,6 +344,9 @@ import { AttachKubernetesAndDockerToIncidentAndAlert1779302536475 } from "./1779
 import { AttachServiceToIncidentAndAlert1779303924241 } from "./1779303924241-AttachServiceToIncidentAndAlert";
 import { AddAgentVersionToKubernetesDockerHost1779392865146 } from "./1779392865146-AddAgentVersionToKubernetesDockerHost";
 import { AddPerformanceIndexes1779392970424 } from "./1779392970424-AddPerformanceIndexes";
+import { AddFacetsToTableView1779536271671 } from "./1779536271671-AddFacetsToTableView";
+import { AddIsMemberNotifiedIndex1779540427366 } from "./1779540427366-AddIsMemberNotifiedIndex";
+import { AddDropdownOptionsToCustomFields1779619108628 } from "./1779619108628-AddDropdownOptionsToCustomFields";
 export default [
   InitialMigration,
   MigrationName1717678334852,
@@ -691,4 +694,7 @@ export default [
   AttachServiceToIncidentAndAlert1779303924241,
   AddAgentVersionToKubernetesDockerHost1779392865146,
   AddPerformanceIndexes1779392970424,
+  AddFacetsToTableView1779536271671,
+  AddIsMemberNotifiedIndex1779540427366,
+  AddDropdownOptionsToCustomFields1779619108628,
 ];

package/Server/Services/AccessTokenService.ts

@@ -199,7 +199,7 @@ export class AccessTokenService extends BaseService {
 
     await GlobalCache.setJSON(
       PermissionNamespace.ProjectPermission,
-      userId.toString() + projectId.toString(),
+      UserPermissionUtil.buildTenantPermissionCacheKey(userId, projectId),
       permission,
     );
 

package/Server/Services/AnalyticsDatabaseService.ts

@@ -535,9 +535,19 @@ export default class AnalyticsDatabaseService<
   ): Promise<Array<TBaseModel>> {
     try {
       if (!findBy.sort || Object.keys(findBy.sort).length === 0) {
+        /*
+         * Default sort uses the model's declared `defaultSortColumn`
+         * (e.g. `time` for Log, `startTime` for Span) so the query
+         * streams from the ClickHouse sort key. The historical
+         * fallback of `createdAt` is not in the sort key on most
+         * analytics tables, which triggered a full sort even on
+         * small LIMITed queries.
+         */
+        const defaultSortColumn: string =
+          this.model.defaultSortColumn || "createdAt";
         findBy.sort = {
-          createdAt: SortOrder.Descending,
-        };
+          [defaultSortColumn]: SortOrder.Descending,
+        } as any;
 
         if (!findBy.select) {
           findBy.select = {} as any;
@@ -910,10 +920,20 @@ export default class AnalyticsDatabaseService<
       deleteBy.query
     );
 
+    /*
+     * Use ClickHouse lightweight deletes (`DELETE FROM`) rather than
+     * `ALTER TABLE … DELETE`. The latter creates an async mutation that
+     * rewrites whole parts and is bounded by `number_of_mutations_to_throw`
+     * (default 1000). Customers with chatty state transitions hit that
+     * ceiling and every subsequent delete fails with TOO_MANY_MUTATIONS.
+     * Lightweight deletes mark rows via the hidden `_row_exists` column
+     * and are reconciled during normal merges, so they don't accumulate
+     * in the mutations queue.
+     */
     /* eslint-disable prettier/prettier */
     const statement: Statement = SQL`
-            ALTER TABLE ${databaseName}.${this.model.tableName}
-            DELETE WHERE TRUE `.append(whereStatement);
+            DELETE FROM ${databaseName}.${this.model.tableName}
+            WHERE TRUE `.append(whereStatement);
 
     logger.debug(`${this.model.tableName} Delete Statement`, { tableName: this.model.tableName } as LogAttributes);
     logger.debug(statement, { tableName: this.model.tableName } as LogAttributes);