@oneuptime/common 12.0.7 → 12.0.9

package/Server/Infrastructure/Postgres/SchemaMigrations/1786900000000-AddInventoryItemArchiveAndCustomFields.ts

@@ -0,0 +1,107 @@
+import { MigrationInterface, QueryRunner } from "typeorm";
+
+/*
+ * Two additions to the Inventory product:
+ *
+ *   Archiving — `isArchived` and its bookkeeping columns. This is the
+ *   disposal route that actually works for this table: deleting a discovered
+ *   or mirrored row does not stick, because ingest re-creates it on the next
+ *   reconcile and the poller re-mirrors it on the next sweep. Archiving is a
+ *   user annotation the reconciler never writes, so it survives
+ *   re-registration.
+ *
+ *   Custom fields — a `customFields` bag on the item plus the
+ *   `InventoryItemCustomField` table that defines their shape, following the
+ *   same two-part pattern as Monitor / Incident / ScheduledMaintenance.
+ *
+ * Hand-written for the same reason as the rename that precedes it: no
+ * database was reachable to run the generator against. Index and foreign key
+ * names are TypeORM's own (`"IDX_"/"FK_" + sha1(table_sortedColumns)`,
+ * truncated to 26 and 27 characters respectively), so the Schema Drift job
+ * finds nothing left to generate. The accompanying test recomputes every one
+ * of them from TypeORM's DefaultNamingStrategy.
+ */
+export class AddInventoryItemArchiveAndCustomFields1786900000000
+  implements MigrationInterface
+{
+  public name = "AddInventoryItemArchiveAndCustomFields1786900000000";
+
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(
+      `ALTER TABLE "InventoryItem" ADD "isArchived" boolean NOT NULL DEFAULT false`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "InventoryItem" ADD "archivedAt" TIMESTAMP WITH TIME ZONE`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "InventoryItem" ADD "archivedByUserId" uuid`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "InventoryItem" ADD "customFields" jsonb`,
+    );
+
+    /*
+     * Every list view filters on isArchived alongside the tenant column, so
+     * the two are indexed together rather than leaving isArchived to a scan.
+     */
+    await queryRunner.query(
+      `CREATE INDEX "IDX_0fef41416ab141b1f344fc96c3" ON "InventoryItem" ("projectId", "isArchived") `,
+    );
+
+    await queryRunner.query(
+      `ALTER TABLE "InventoryItem" ADD CONSTRAINT "FK_8567c6339bf4889d00d121b3047" FOREIGN KEY ("archivedByUserId") REFERENCES "User"("_id") ON DELETE SET NULL ON UPDATE NO ACTION`,
+    );
+
+    await queryRunner.query(
+      `CREATE TABLE "InventoryItemCustomField" ("_id" uuid NOT NULL DEFAULT uuid_generate_v4(), "createdAt" TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(), "updatedAt" TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(), "deletedAt" TIMESTAMP WITH TIME ZONE, "version" integer NOT NULL, "projectId" uuid NOT NULL, "name" character varying(100) NOT NULL, "description" character varying(500), "customFieldType" character varying(100), "dropdownOptions" character varying(500), "createdByUserId" uuid, "deletedByUserId" uuid, CONSTRAINT "PK_InventoryItemCustomField" PRIMARY KEY ("_id"))`,
+    );
+    await queryRunner.query(
+      `CREATE INDEX "IDX_9f6a4cbd2f57a69e5d6fab9854" ON "InventoryItemCustomField" ("projectId") `,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "InventoryItemCustomField" ADD CONSTRAINT "FK_9f6a4cbd2f57a69e5d6fab9854d" FOREIGN KEY ("projectId") REFERENCES "Project"("_id") ON DELETE CASCADE ON UPDATE NO ACTION`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "InventoryItemCustomField" ADD CONSTRAINT "FK_8f121d4daca9d74f5f948511aef" FOREIGN KEY ("createdByUserId") REFERENCES "User"("_id") ON DELETE SET NULL ON UPDATE NO ACTION`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "InventoryItemCustomField" ADD CONSTRAINT "FK_85f9a41e4ecf947560491917527" FOREIGN KEY ("deletedByUserId") REFERENCES "User"("_id") ON DELETE SET NULL ON UPDATE NO ACTION`,
+    );
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(
+      `ALTER TABLE "InventoryItemCustomField" DROP CONSTRAINT "FK_85f9a41e4ecf947560491917527"`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "InventoryItemCustomField" DROP CONSTRAINT "FK_8f121d4daca9d74f5f948511aef"`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "InventoryItemCustomField" DROP CONSTRAINT "FK_9f6a4cbd2f57a69e5d6fab9854d"`,
+    );
+    await queryRunner.query(
+      `DROP INDEX "public"."IDX_9f6a4cbd2f57a69e5d6fab9854"`,
+    );
+    await queryRunner.query(`DROP TABLE "InventoryItemCustomField"`);
+
+    await queryRunner.query(
+      `ALTER TABLE "InventoryItem" DROP CONSTRAINT "FK_8567c6339bf4889d00d121b3047"`,
+    );
+    await queryRunner.query(
+      `DROP INDEX "public"."IDX_0fef41416ab141b1f344fc96c3"`,
+    );
+
+    await queryRunner.query(
+      `ALTER TABLE "InventoryItem" DROP COLUMN "customFields"`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "InventoryItem" DROP COLUMN "archivedByUserId"`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "InventoryItem" DROP COLUMN "archivedAt"`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "InventoryItem" DROP COLUMN "isArchived"`,
+    );
+  }
+}

package/Server/Infrastructure/Postgres/SchemaMigrations/1787000000000-AddOnCallNotificationFallbackColumns.ts

@@ -0,0 +1,90 @@
+import { MigrationInterface, QueryRunner } from "typeorm";
+
+/*
+ * Schema for the on-call notification fallback: a page routed to a responder who
+ * has no notification rule matching the event's rule type and severity now goes
+ * out on whatever verified method they do have, instead of dead-ending with an
+ * Error status nobody reads.
+ *
+ * Hand-written rather than emitted by `npm run generate-postgres-migration`. The
+ * generator diffs the entire entity set against the live database, and several
+ * models in this worktree are being changed in parallel, so a generated file
+ * would have swept up unrelated in-flight columns. The statements below are
+ * exactly what the generator produces for these column definitions — compare
+ * `enableWhatsAppNotifications` (1759943124812) and `enableTelegramNotifications`
+ * (1776761171349) for the Project booleans, and `isFeatureFlagMonitorGroupsEnabled`
+ * in the InitialMigration for the nullable-with-default shape.
+ *
+ * UserNotificationRule.isOptOut is nullable because a rule that predates this
+ * column is simply not an opt-out; the DEFAULT covers rows written afterwards and
+ * the backfill for existing rows is the default itself. The two Project columns
+ * are NOT NULL like every other project boolean, so existing projects land on
+ * false: the fallback on, and no owner notification recorded as sent yet.
+ *
+ * UserOnCallLogTimeline.userNotificationRuleId is dropped to NULL-able because
+ * the fallback delivers through a UserNotificationRule constructed in memory and
+ * never saved — there is deliberately no stored rule behind a fallback page, so
+ * there is no id to record. It has been NOT NULL since the InitialMigration
+ * (1717605043663), which means that without this statement every fallback insert
+ * fails its required-field check and the feature cannot deliver anything at all.
+ * The foreign key on the column stays as it is: Postgres does not check a FK when
+ * the referencing column is NULL, so a NULL row is legal against the existing
+ * constraint and no index or constraint needs rebuilding.
+ */
+export class AddOnCallNotificationFallbackColumns1787000000000
+  implements MigrationInterface
+{
+  public name: string = "AddOnCallNotificationFallbackColumns1787000000000";
+
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(
+      `ALTER TABLE "UserNotificationRule" ADD "isOptOut" boolean DEFAULT false`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "Project" ADD "disableOnCallNotificationFallback" boolean NOT NULL DEFAULT false`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "Project" ADD "onCallFallbackUsedNotificationSentToOwners" boolean NOT NULL DEFAULT false`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "UserOnCallLogTimeline" ALTER COLUMN "userNotificationRuleId" DROP NOT NULL`,
+    );
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<void> {
+    /*
+     * Restoring NOT NULL is only possible once nothing violates it, and after the
+     * fallback has run there will be timeline rows with a NULL rule id — one per
+     * fallback page delivered. A bare `SET NOT NULL` would abort the whole
+     * rollback on the first such row, so the rows are deleted first.
+     *
+     * This deletion is destructive and worth understanding before running down().
+     * What it removes is exactly the set of rows that could not have existed
+     * before this migration: fallback delivery records. Their history is real —
+     * they describe pages that were genuinely sent — but there is no way to
+     * express them in the pre-migration schema, so reverting the schema means
+     * giving them up. An operator who wants to keep the record should copy them
+     * out first:
+     *
+     *   SELECT * FROM "UserOnCallLogTimeline" WHERE "userNotificationRuleId" IS NULL;
+     *
+     * Ordinary rows are untouched: before this migration the column was NOT NULL,
+     * so nothing predating it can match.
+     */
+    await queryRunner.query(
+      `DELETE FROM "UserOnCallLogTimeline" WHERE "userNotificationRuleId" IS NULL`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "UserOnCallLogTimeline" ALTER COLUMN "userNotificationRuleId" SET NOT NULL`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "Project" DROP COLUMN "onCallFallbackUsedNotificationSentToOwners"`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "Project" DROP COLUMN "disableOnCallNotificationFallback"`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "UserNotificationRule" DROP COLUMN "isOptOut"`,
+    );
+  }
+}

package/Server/Infrastructure/Postgres/SchemaMigrations/1787100000000-AddAIConversationPageContext.ts

@@ -0,0 +1,39 @@
+import { MigrationInterface, QueryRunner } from "typeorm";
+
+/*
+ * AIConversation.pageContext: the conversation's subject — the dashboard page
+ * (usually one specific entity, e.g. an incident) the user was looking at
+ * when the conversation started. ChatAgentRunner persists it from the first
+ * turn that carries a sanitized page context, and turns that arrive without
+ * one fall back to it, so "this incident" keeps resolving after the user
+ * navigates away mid-conversation.
+ *
+ * Hand-written rather than emitted by `npm run generate-postgres-migration`.
+ * The generator diffs the entire entity set against the live database, and
+ * several models in this worktree are being changed in parallel, so a
+ * generated file would have swept up unrelated in-flight columns. The
+ * statement below is exactly what the generator produces for a nullable JSON
+ * column — compare "widgets" and "pausedState" in
+ * 1783453297388-AddAIChatWriteActionsAndWidgets.
+ *
+ * Nullable with no default: a conversation without a pinned subject is the
+ * normal state (the user opened Ask AI from nowhere in particular), and every
+ * pre-existing conversation is exactly that.
+ */
+export class AddAIConversationPageContext1787100000000
+  implements MigrationInterface
+{
+  public name: string = "AddAIConversationPageContext1787100000000";
+
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(
+      `ALTER TABLE "AIConversation" ADD "pageContext" jsonb`,
+    );
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(
+      `ALTER TABLE "AIConversation" DROP COLUMN "pageContext"`,
+    );
+  }
+}

package/Server/Infrastructure/Postgres/SchemaMigrations/1787200000000-AddAIChatMessageFeedback.ts

@@ -0,0 +1,33 @@
+import { MigrationInterface, QueryRunner } from "typeorm";
+
+/*
+ * AIConversationMessage.userFeedback: thumbs feedback ("Up" | "Down") the user
+ * left on an assistant message via POST /ai-chat/message-feedback. Nullable
+ * with no default — no feedback is the normal state, and every pre-existing
+ * message is exactly that. Clearing feedback writes NULL back.
+ *
+ * Hand-written rather than emitted by `npm run generate-postgres-migration`.
+ * The generator diffs the entire entity set against the live database, and
+ * several models in this worktree are being changed in parallel, so a
+ * generated file would have swept up unrelated in-flight columns. The
+ * statement below is exactly what the generator produces for a nullable
+ * ShortText column (ColumnLength.ShortText = 100) — compare "analysisTldr" in
+ * 1786600000000-AddInvestigationAnalysisTldr for the varchar shape.
+ */
+export class AddAIChatMessageFeedback1787200000000
+  implements MigrationInterface
+{
+  public name: string = "AddAIChatMessageFeedback1787200000000";
+
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(
+      `ALTER TABLE "AIConversationMessage" ADD "userFeedback" character varying(100)`,
+    );
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(
+      `ALTER TABLE "AIConversationMessage" DROP COLUMN "userFeedback"`,
+    );
+  }
+}

package/Server/Infrastructure/Postgres/SchemaMigrations/1787300000000-AddEpisodeMemberNotifyIndexes.ts

@@ -0,0 +1,59 @@
+import { MigrationInterface, QueryRunner } from "typeorm";
+
+/*
+ * Partial indexes behind the two episode-member "owner notified" sweeps:
+ *
+ *   AlertEpisodeOwner:SendAlertAddedEmail    -> AlertEpisodeMember
+ *   IncidentEpisodeOwner:SendIncidentAddedEmail -> IncidentEpisodeMember
+ *
+ * Both run EVERY_MINUTE and select the rows whose notify flag is still
+ * false. The flag is written false on insert and flipped true by the sweep
+ * itself, so in steady state nearly every row is true and each tick was a
+ * full sequential scan that returned nothing — 2 x 1,440 = 2,880 zero-row
+ * scans a day across the two tables.
+ *
+ * PARTIAL rather than a plain index on the column: a full index would be the
+ * size of the table to serve a predicate matching a handful of rows, and
+ * every notification write would maintain it. Restricted to
+ * `flag = false AND "deletedAt" IS NULL` it stays about the size of the
+ * sweep's own working set. The soft-delete clause is not decoration: the
+ * sweep always carries it (TypeORM adds it for the delete-date column), and
+ * nothing ever flips the flag on a soft-deleted row, so without it those
+ * rows would sit in the index forever.
+ *
+ * Hand-written rather than emitted by `npm run generate-postgres-migration`,
+ * which needs a live Postgres to diff against and none was reachable. Both
+ * statements are exactly what the generator emits for these decorators:
+ * TypeORM's DefaultNamingStrategy names an index
+ * `"IDX_" + sha1(table_sortedColumns_where).slice(0, 26)` — note the WHERE
+ * text is part of the hashed key, so the names below are tied to the exact
+ * predicate string in the models — and PostgresQueryRunner.createIndexSql
+ * emits `CREATE INDEX <name> ON <table> (<columns>) WHERE <where>`. Compare
+ * "IDX_91e00aa0bbe2c0cdcfbf884a2d" on "NetworkEndpoint" in
+ * 1784757142154-AddNetworkSiteHierarchyAndEndpoints for the partial-index
+ * shape. The accompanying test recomputes both names from TypeORM's own
+ * naming strategy so the Schema Drift job finds nothing left to generate.
+ */
+export class AddEpisodeMemberNotifyIndexes1787300000000
+  implements MigrationInterface
+{
+  public name: string = "AddEpisodeMemberNotifyIndexes1787300000000";
+
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(
+      `CREATE INDEX "IDX_76aa0238e90e515992b65bc0a6" ON "AlertEpisodeMember" ("isOwnerNotifiedOfAlertAdded") WHERE "isOwnerNotifiedOfAlertAdded" = false AND "deletedAt" IS NULL`,
+    );
+    await queryRunner.query(
+      `CREATE INDEX "IDX_79e3a24f776e52d635dbeb9c27" ON "IncidentEpisodeMember" ("isOwnerNotifiedOfIncidentAdded") WHERE "isOwnerNotifiedOfIncidentAdded" = false AND "deletedAt" IS NULL`,
+    );
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(
+      `DROP INDEX "public"."IDX_79e3a24f776e52d635dbeb9c27"`,
+    );
+    await queryRunner.query(
+      `DROP INDEX "public"."IDX_76aa0238e90e515992b65bc0a6"`,
+    );
+  }
+}

package/Server/Infrastructure/Postgres/SchemaMigrations/Index.ts

@@ -515,6 +515,18 @@ import { AddDataSourceTable1786303472848 } from "./1786303472848-AddDataSourceTa
 import { MigrationName1786446545142 } from "./1786446545142-MigrationName";
 import { AddMonitorDependency1786449497966 } from "./1786449497966-AddMonitorDependency";
 import { AddDeletedProjectTable1786461136170 } from "./1786461136170-AddDeletedProjectTable";
+import { MigrationName1786551733814 } from "./1786551733814-MigrationName";
+import { AddInventoryItemArchiveAndCustomFields1786900000000 } from "./1786900000000-AddInventoryItemArchiveAndCustomFields";
+import { RenameTelemetryEntityToInventoryItem1786800000000 } from "./1786800000000-RenameTelemetryEntityToInventoryItem";
+import { AddWorkflowLogStepTrace1786559879134 } from "./1786559879134-AddWorkflowLogStepTrace";
+import { AddMonitoringMethodToNetworkDevice1786625176831 } from "./1786625176831-AddMonitoringMethodToNetworkDevice";
+import { AddNetworkDeviceLink1786634985763 } from "./1786634985763-AddNetworkDeviceLink";
+import { AddNetworkDeviceLinkRule1786639512056 } from "./1786639512056-AddNetworkDeviceLinkRule";
+import { AddNetworkTopologySuppression1786639972982 } from "./1786639972982-AddNetworkTopologySuppression";
+import { AddOnCallNotificationFallbackColumns1787000000000 } from "./1787000000000-AddOnCallNotificationFallbackColumns";
+import { AddAIConversationPageContext1787100000000 } from "./1787100000000-AddAIConversationPageContext";
+import { AddAIChatMessageFeedback1787200000000 } from "./1787200000000-AddAIChatMessageFeedback";
+import { AddEpisodeMemberNotifyIndexes1787300000000 } from "./1787300000000-AddEpisodeMemberNotifyIndexes";
 
 export default [
   InitialMigration,
@@ -1034,4 +1046,16 @@ export default [
   MigrationName1786446545142,
   AddMonitorDependency1786449497966,
   AddDeletedProjectTable1786461136170,
+  MigrationName1786551733814,
+  AddWorkflowLogStepTrace1786559879134,
+  AddMonitoringMethodToNetworkDevice1786625176831,
+  AddNetworkDeviceLink1786634985763,
+  AddNetworkDeviceLinkRule1786639512056,
+  AddNetworkTopologySuppression1786639972982,
+  RenameTelemetryEntityToInventoryItem1786800000000,
+  AddInventoryItemArchiveAndCustomFields1786900000000,
+  AddOnCallNotificationFallbackColumns1787000000000,
+  AddAIConversationPageContext1787100000000,
+  AddAIChatMessageFeedback1787200000000,
+  AddEpisodeMemberNotifyIndexes1787300000000,
 ];

package/Server/Infrastructure/Queue.ts

@@ -561,29 +561,94 @@ export default class Queue {
   }
 
   /*
-   * Backlog = jobs still waiting for a worker (waiting + delayed), for
-   * autoscaling signals. ACTIVE jobs are deliberately excluded: they are
-   * already being served, so to a scaler they are capacity, not demand.
-   * Telemetry jobs in particular sit in the active state for the fan-in
-   * writer's full flush window (up to TELEMETRY_FANIN_MAX_WAIT_MS) while
-   * awaiting their ClickHouse ack, so a fleet scaled on an active-inclusive
-   * count converges on pod count = job rate x ack latency — every new pod
-   * just parks more jobs without shortening the flush window. Scaling on
-   * waiting + delayed only breaks that feedback loop. getQueueSize above
-   * keeps the active-inclusive sum for ingest backpressure checks, where
-   * in-flight work genuinely counts against capacity.
+   * How many job schedulers (BullMQ's repeatable/cron registrations) this
+   * queue holds. This is a ZCARD of the repeat zset, so it is O(1) and safe
+   * to call on every scrape of the scaling metric — unlike getJobSchedulers(),
+   * which pages the whole set.
+   *
+   * Guarded because the count is only ever used to *correct* the backlog
+   * number: a BullMQ build without this method (it is newer than the
+   * repeatable API it replaces) must degrade to the uncorrected
+   * waiting + delayed sum rather than throw and take the whole scaling
+   * metric offline with it.
+   */
+  private static async getJobSchedulerCount(queue: BullQueue): Promise<number> {
+    if (typeof queue.getJobSchedulersCount !== "function") {
+      return 0;
+    }
+
+    return (await queue.getJobSchedulersCount()) || 0;
+  }
+
+  /*
+   * Backlog = jobs still waiting for a worker, for autoscaling signals.
+   *
+   * ACTIVE jobs are deliberately excluded: they are already being served, so
+   * to a scaler they are capacity, not demand. Telemetry jobs in particular
+   * sit in the active state for the fan-in writer's full flush window (up to
+   * TELEMETRY_FANIN_MAX_WAIT_MS) while awaiting their ClickHouse ack, so a
+   * fleet scaled on an active-inclusive count converges on pod count =
+   * job rate x ack latency — every new pod just parks more jobs without
+   * shortening the flush window. Scaling on waiting + delayed only breaks
+   * that feedback loop. getQueueSize above keeps the active-inclusive sum for
+   * ingest backpressure checks, where in-flight work genuinely counts against
+   * capacity.
+   *
+   * The job SCHEDULERS are then subtracted back off — but ONLY from the
+   * delayed side, which is the whole subtlety here. Every RunCron()
+   * registration parks exactly one entry in the DELAYED set: the placeholder
+   * holding its next fire time. That placeholder is not work anybody can
+   * drain, and getDelayedCount() counts it. Left in, the placeholders put a
+   * hard floor under the metric equal to the number of registered crons (127
+   * on the Worker queue today), which sits above every sensible scale-down
+   * target, so the fleet could never scale in no matter how idle it was.
+   *
+   * The subtraction is deliberately NOT `max(0, waiting + delayed - n)`,
+   * because a scheduler entry stops being a parked placeholder the moment it
+   * comes due. BullMQ's promoteDelayedJobs ZREMs the placeholder out of
+   * `delayed` and LPUSHes it onto `wait`, and the SUCCESSOR placeholder is
+   * not created until a worker actually moves that job to active
+   * (Worker.nextJobFromJobData -> upsertJobScheduler). So for the whole
+   * promotion window there are k due cron jobs sitting in `waiting` with
+   * delayed = n - k, while the repeat zset — and therefore the scheduler
+   * count — still reads n.
+   *
+   * Those k jobs are real, drainable demand: they are queued work waiting for
+   * a worker. Subtracting the full scheduler count off the combined sum would
+   * cancel them out exactly, so the metric would read 0. That is not a
+   * rounding error, it is the dangerous direction: at the top of a minute a
+   * large batch of EVERY_MINUTE crons is promoted at once, and an idle-looking
+   * zero would tell KEDA to scale in precisely when the work arrived.
+   * Discounting the placeholders only where they actually live — the delayed
+   * set — leaves `waiting` untouched and reports that burst at face value.
+   *
+   * When delayed >= schedulerCount (the steady state, every placeholder
+   * parked) the two forms agree exactly, so this costs nothing in the common
+   * case.
+   *
+   * The inner Math.max(0, ...) clamp still matters: the three counts are read
+   * concurrently and are not a consistent snapshot, so delayed can be read
+   * mid-promotion and come back below the scheduler count. Without the clamp
+   * that shortfall would be subtracted out of `waiting`, re-introducing the
+   * masking this form exists to avoid, and could drive the gauge negative —
+   * which KEDA reads as a garbage value.
    */
   @CaptureSpan()
   public static async getQueueBacklogSize(
     queueName: QueueName,
   ): Promise<number> {
     const queue: BullQueue = this.getQueue(queueName);
-    const [waitingCount, delayedCount]: [number, number] = await Promise.all([
+    const [waitingCount, delayedCount, schedulerCount]: [
+      number,
+      number,
+      number,
+    ] = await Promise.all([
       queue.getWaitingCount(),
       queue.getDelayedCount(),
+      this.getJobSchedulerCount(queue),
     ]);
 
-    return waitingCount + delayedCount;
+    return waitingCount + Math.max(0, delayedCount - schedulerCount);
   }
 
   @CaptureSpan()

package/Server/Middleware/MasterAdminAuthorization.ts

@@ -59,13 +59,18 @@ export default class MasterAdminAuthorization {
    * Same as isAuthorizedMasterAdminMiddleware, but ALSO accepts the instance-wide
    * master API key (Admin Dashboard → Settings → API Key) supplied in the
    * `apikey` header. The master key has root/master-admin access, so this lets
-   * automated callers reach the master-admin OneUptime Health endpoints with the
-   * key instead of a logged-in master-admin session.
+   * automated callers reach master-admin endpoints with the key instead of a
+   * logged-in master-admin session.
    *
-   * Deliberately scoped to the read-only health / diagnostics routes only. It is
-   * NOT used on higher-risk master-admin actions (e.g. the read/write query
-   * console or broadcast email), which stay on the JWT-only middleware above so
-   * that a leaked static key cannot trigger them headlessly.
+   * Deliberately scoped to READ-ONLY routes: the OneUptime Health / diagnostics
+   * endpoints, and the cross-tenant read of one user's projects
+   * (POST /user/:userId/projects). It is NOT used on master-admin routes that
+   * change anything — the read/write query console, broadcast email, removing a
+   * user from a project, setting a password — which stay on the JWT-only
+   * middleware above so that a leaked static key cannot trigger them headlessly.
+   *
+   * Read-only is the line to hold when adding a route here. A key that can read
+   * discloses; a key that can write takes over.
    */
   public static async isAuthorizedMasterAdminOrMasterApiKeyMiddleware(
     req: ExpressRequest,