@oneuptime/common 11.3.25 → 11.3.28

package/Server/Services/Index.ts

@@ -12,6 +12,7 @@ import BillingService from "./BillingService";
 import CallLogService from "./CallLogService";
 import CallService from "./CallService";
 import DataMigrationService from "./DataMigrationService";
+import MigrationFailureService from "./MigrationFailureService";
 import DomainService from "./DomainService";
 import EmailLogService from "./EmailLogService";
 import EmailVerificationTokenService from "./EmailVerificationTokenService";
@@ -273,6 +274,7 @@ const services: Array<BaseService> = [
   CallService,
 
   DataMigrationService,
+  MigrationFailureService,
   DomainService,
 
   EmailLogService,

package/Server/Services/LlmProviderService.ts

@@ -6,6 +6,7 @@ import ObjectID from "../../Types/ObjectID";
 import UpdateBy from "../Types/Database/UpdateBy";
 import QueryHelper from "../Types/Database/QueryHelper";
 import LIMIT_MAX from "../../Types/Database/LimitMax";
+import SortOrder from "../../Types/BaseDatabase/SortOrder";
 import CaptureSpan from "../Utils/Telemetry/CaptureSpan";
 
 export class Service extends DatabaseService<Model> {
@@ -153,6 +154,115 @@ export class Service extends DatabaseService<Model> {
 
     return null;
   }
+
+  /*
+   * Resolve the provider to use for a chat turn. When the user has explicitly
+   * chosen a provider (llmProviderId), use it — but only if it is actually
+   * usable by this project: either a global provider, or one owned by the
+   * project. Otherwise fall back to the project default / global provider.
+   * A stale id (e.g. the chosen provider was deleted) also falls back, so a
+   * conversation never breaks because its provider went away.
+   */
+  @CaptureSpan()
+  public async getProviderForChat(data: {
+    projectId: ObjectID;
+    llmProviderId?: ObjectID | undefined;
+  }): Promise<Model | null> {
+    if (data.llmProviderId) {
+      const provider: Model | null = await this.findOneBy({
+        query: {
+          _id: data.llmProviderId.toString(),
+        },
+        select: {
+          _id: true,
+          name: true,
+          llmType: true,
+          apiKey: true,
+          baseUrl: true,
+          modelName: true,
+          isGlobalLlm: true,
+          projectId: true,
+          costPerMillionTokensInUSDCents: true,
+        },
+        props: {
+          isRoot: true,
+        },
+      });
+
+      if (provider) {
+        const isGlobal: boolean = provider.isGlobalLlm === true;
+        const belongsToProject: boolean =
+          provider.projectId?.toString() === data.projectId.toString();
+
+        if (isGlobal || belongsToProject) {
+          return provider;
+        }
+      }
+      // Fall through to default resolution when the id is invalid/inaccessible.
+    }
+
+    return this.getLLMProviderForProject(data.projectId);
+  }
+
+  /*
+   * The providers a project member can pick from in the chat provider switcher:
+   * every provider configured for the project plus every global provider.
+   * Secrets (apiKey) are never selected here — this only feeds the picker UI.
+   */
+  @CaptureSpan()
+  public async getSelectableProvidersForProject(
+    projectId: ObjectID,
+  ): Promise<Array<Model>> {
+    const [projectProviders, globalProviders]: [Array<Model>, Array<Model>] =
+      await Promise.all([
+        this.findBy({
+          query: {
+            projectId: projectId,
+          },
+          select: {
+            _id: true,
+            name: true,
+            llmType: true,
+            modelName: true,
+            isDefault: true,
+            isGlobalLlm: true,
+          },
+          sort: {
+            isDefault: SortOrder.Descending,
+            name: SortOrder.Ascending,
+          },
+          skip: 0,
+          limit: LIMIT_MAX,
+          props: {
+            isRoot: true,
+          },
+        }),
+        this.findBy({
+          query: {
+            projectId: QueryHelper.isNull(),
+            isGlobalLlm: true,
+          },
+          select: {
+            _id: true,
+            name: true,
+            llmType: true,
+            modelName: true,
+            isDefault: true,
+            isGlobalLlm: true,
+          },
+          sort: {
+            name: SortOrder.Ascending,
+          },
+          skip: 0,
+          limit: LIMIT_MAX,
+          props: {
+            isRoot: true,
+          },
+        }),
+      ]);
+
+    return [...projectProviders, ...globalProviders];
+  }
 }
 
 export default new Service();

package/Server/Services/MigrationFailureService.ts

@@ -0,0 +1,9 @@
+import DatabaseService from "./DatabaseService";
+import Model from "../../Models/DatabaseModels/MigrationFailure";
+
+export class Service extends DatabaseService<Model> {
+  public constructor() {
+    super(Model);
+  }
+}
+export default new Service();

package/Server/Services/MonitorProbeService.ts

@@ -310,8 +310,14 @@ export class Service extends DatabaseService<MonitorProbe> {
     _onCreate: OnCreate<MonitorProbe>,
     createdItem: MonitorProbe,
   ): Promise<MonitorProbe> {
-    if (createdItem.probeId) {
-      await MonitorService.refreshProbeStatus(createdItem.probeId);
+    /*
+     * Refresh only the monitor this probe was just added to. Do NOT call
+     * refreshProbeStatus(probeId) here: a global/shared probe can be attached
+     * to thousands of monitors, and refreshing every one of them on a single
+     * create storms the database and can 500 the request.
+     */
+    if (createdItem.monitorId) {
+      await this.refreshMonitorStatusSafely(createdItem.monitorId);
     }
 
     return Promise.resolve(createdItem);
@@ -321,7 +327,7 @@ export class Service extends DatabaseService<MonitorProbe> {
     onUpdate: OnUpdate<MonitorProbe>,
     updatedItemIds: ObjectID[],
   ): Promise<OnUpdate<MonitorProbe>> {
-    // if isEnabled is updated, refresh the probe status
+    // if isEnabled is updated, refresh the status of the affected monitors.
     if (onUpdate.updateBy.data.isEnabled !== undefined) {
       const monitorProbes: Array<MonitorProbe> = await this.findBy({
         query: {
@@ -329,8 +335,6 @@ export class Service extends DatabaseService<MonitorProbe> {
         },
         select: {
           monitorId: true,
-          probeId: true,
-          nextPingAt: true,
         },
         limit: LIMIT_PER_PROJECT,
         skip: 0,
@@ -340,16 +344,38 @@ export class Service extends DatabaseService<MonitorProbe> {
       });
 
       for (const monitorProbe of monitorProbes) {
-        if (!monitorProbe.probeId) {
+        if (!monitorProbe.monitorId) {
           continue;
         }
 
-        await MonitorService.refreshProbeStatus(monitorProbe.probeId);
+        /*
+         * Refresh only the monitor whose probe row changed. Do NOT call
+         * refreshProbeStatus(probeId) here: a global/shared probe can be
+         * attached to thousands of monitors, and refreshing all of them on a
+         * single enable/disable toggle storms the database and 500s the
+         * request (the row itself is already saved by this point).
+         */
+        await this.refreshMonitorStatusSafely(monitorProbe.monitorId);
       }
     }
 
     return onUpdate;
   }
+
+  /*
+   * Refresh a single monitor's aggregate probe status without letting a
+   * failure bubble up. These refreshes run in post-commit hooks
+   * (onCreateSuccess / onUpdateSuccess) after the MonitorProbe row is already
+   * persisted, so a status-refresh error must not turn a successful save into
+   * a 500 for the user — log it instead.
+   */
+  private async refreshMonitorStatusSafely(monitorId: ObjectID): Promise<void> {
+    try {
+      await MonitorService.refreshMonitorProbeStatus(monitorId);
+    } catch (err) {
+      logger.error(err);
+    }
+  }
 }
 
 export default new Service();

package/Server/Services/MonitorService.ts

@@ -1620,14 +1620,26 @@ ${createdItem.description?.trim() || "No description provided."}
 
     /*
      * Each monitor appears at most once for a given probeId (composite
-     * unique on MonitorProbe), so concurrent refreshes operate on
-     * disjoint rows and are safe to run in parallel.
+     * unique on MonitorProbe), so concurrent refreshes operate on disjoint
+     * rows and are safe to run in parallel. A global/shared probe can be
+     * attached to thousands of monitors, though, so refresh in bounded
+     * batches instead of firing every refresh at once — an unbounded
+     * Promise.all here would exhaust the database connection pool.
      */
-    await Promise.all(
-      monitorProbes.map((monitorProbe: MonitorProbe) => {
-        return this.refreshMonitorProbeStatus(monitorProbe.monitorId!);
-      }),
-    );
+    const refreshConcurrency: number = 50;
+
+    for (let i: number = 0; i < monitorProbes.length; i += refreshConcurrency) {
+      const batch: Array<MonitorProbe> = monitorProbes.slice(
+        i,
+        i + refreshConcurrency,
+      );
+
+      await Promise.all(
+        batch.map((monitorProbe: MonitorProbe) => {
+          return this.refreshMonitorProbeStatus(monitorProbe.monitorId!);
+        }),
+      );
+    }
   }
 
   @CaptureSpan()

package/Server/Services/OnCallDutyPolicyEscalationRuleScheduleService.ts

@@ -22,6 +22,9 @@ import PushNotificationMessage from "../../Types/PushNotification/PushNotificati
 import PushNotificationUtil from "../Utils/PushNotificationUtil";
 import { createWhatsAppMessageFromTemplate } from "../Utils/WhatsAppTemplateUtil";
 import { WhatsAppMessagePayload } from "../../Types/WhatsApp/WhatsAppMessage";
+import OnCallDutyPolicyTimeLogService from "./OnCallDutyPolicyTimeLogService";
+import OneUptimeDate from "../../Types/Date";
+import logger, { LogAttributes } from "../Utils/Logger";
 
 export class Service extends DatabaseService<Model> {
   public constructor() {
@@ -86,6 +89,39 @@ export class Service extends DatabaseService<Model> {
       return createdItem;
     }
 
+    /*
+     * Open the roster user's on-call time log for this schedule/rule/policy.
+     * Mirrors onDeleteSuccess's endTimeLogForUser (detaching a schedule closes
+     * the log, so attaching must open it). Without this, attaching a schedule
+     * whose roster user is ALREADY established never opened a log — the roster
+     * refresh only opens one when the current user CHANGES — so the whole stint
+     * went unrecorded in on-call reporting and the later detach's
+     * endTimeLogForUser was a no-op (audit M1). startTimeLogForUser is idempotent
+     * (open-log dedup), so a later natural handoff re-opening the same log is a
+     * no-op.
+     */
+    if (
+      createdModel.onCallDutyPolicy?.id &&
+      createdModel.onCallDutyPolicyEscalationRule?.id &&
+      createdModel.projectId &&
+      createdModel.onCallDutyPolicyScheduleId
+    ) {
+      OnCallDutyPolicyTimeLogService.startTimeLogForUser({
+        projectId: createdModel.projectId,
+        onCallDutyPolicyId: createdModel.onCallDutyPolicy.id,
+        onCallDutyPolicyEscalationRuleId:
+          createdModel.onCallDutyPolicyEscalationRule.id,
+        userId: userOnSchedule,
+        onCallDutyPolicyScheduleId: createdModel.onCallDutyPolicyScheduleId,
+        startsAt: OneUptimeDate.getCurrentDate(),
+      }).catch((error: Error) => {
+        logger.error(error, {
+          projectId: createdModel.projectId?.toString(),
+          userId: userOnSchedule?.toString(),
+        } as LogAttributes);
+      });
+    }
+
     const scheduleName: string =
       createdModel.onCallDutyPolicySchedule?.name || "No name provided";
 
@@ -289,6 +325,35 @@ export class Service extends DatabaseService<Model> {
         continue;
       }
 
+      /*
+       * Close the roster user's open on-call time log for this schedule/rule/
+       * policy. Roster handoffs open a time log keyed by schedule + rule + policy
+       * (OnCallDutyPolicyScheduleService.startTimeLogForUser); the user-link and
+       * team-link delete handlers close theirs, but detaching a SCHEDULE from the
+       * rule left the log open forever, so the roster user kept showing as
+       * on-call in reporting (audit F18).
+       */
+      if (
+        deletedItem.onCallDutyPolicy?.id &&
+        deletedItem.onCallDutyPolicyEscalationRule?.id &&
+        deletedItem.projectId
+      ) {
+        OnCallDutyPolicyTimeLogService.endTimeLogForUser({
+          projectId: deletedItem.projectId,
+          onCallDutyPolicyId: deletedItem.onCallDutyPolicy.id,
+          onCallDutyPolicyEscalationRuleId:
+            deletedItem.onCallDutyPolicyEscalationRule.id,
+          userId: userOnSchedule,
+          onCallDutyPolicyScheduleId: deletedItem.onCallDutyPolicyScheduleId!,
+          endsAt: OneUptimeDate.getCurrentDate(),
+        }).catch((error: Error) => {
+          logger.error(error, {
+            projectId: deletedItem.projectId?.toString(),
+            userId: userOnSchedule?.toString(),
+          } as LogAttributes);
+        });
+      }
+
       const sendEmailToUserId: ObjectID | undefined | null = userOnSchedule;
 
       if (!sendEmailToUserId) {

package/Server/Services/OnCallDutyPolicyEscalationRuleService.ts

@@ -10,6 +10,7 @@ import OnCallDutyPolicyEscalationRuleScheduleService from "./OnCallDutyPolicyEsc
 import OnCallDutyPolicyEscalationRuleTeamService from "./OnCallDutyPolicyEscalationRuleTeamService";
 import OnCallDutyPolicyEscalationRuleUserService from "./OnCallDutyPolicyEscalationRuleUserService";
 import OnCallDutyPolicyExecutionLogService from "./OnCallDutyPolicyExecutionLogService";
+import OnCallDutyPolicyExecutionLog from "../../Models/DatabaseModels/OnCallDutyPolicyExecutionLog";
 import OnCallDutyPolicyExecutionLogTimelineService from "./OnCallDutyPolicyExecutionLogTimelineService";
 import OnCallDutyPolicyScheduleService from "./OnCallDutyPolicyScheduleService";
 import TeamMemberService from "./TeamMemberService";
@@ -315,31 +316,66 @@ export class Service extends DatabaseService<Model> {
     );
 
     type StartUserNotificationRuleExecutionFunction = (
-      userId: ObjectID,
+      originalUserId: ObjectID,
+      alertSentToUserId: ObjectID,
       teamId: ObjectID | null,
       scheduleId: ObjectID | null,
     ) => Promise<void>;
 
-    const startUserNotificationRuleExecution: StartUserNotificationRuleExecutionFunction =
-      async (
-        userId: ObjectID,
-        teamId: ObjectID | null,
-        scheduleId: ObjectID | null,
-      ): Promise<void> => {
-        // This is where user is notified.
-
-        // get route alert to user id.
-        let routeAlertToUserId: ObjectID | null = null;
+    /*
+     * Resolve the FINAL alert recipient for a target, applying a user override
+     * (getRouteAlertToUserId) exactly ONCE per paging decision.
+     *
+     * Schedule targets are already override-resolved: getCurrentUserIdInSchedule
+     * applies overrides via UserOverrideUtil. Applying getRouteAlertToUserId to
+     * them again re-substituted an already-substituted user (transitive), so the
+     * schedule paging path could page a different person than the roster,
+     * dashboard, handoff notification, and a rule that lists the same user
+     * directly — all of which apply overrides only once (audit F5). So team and
+     * direct-user targets (the RAW roster user) get the single override hop here;
+     * schedule targets do not.
+     */
+    const resolveAlertRecipientUserId: (
+      userId: ObjectID,
+      isFromSchedule: boolean,
+    ) => Promise<ObjectID> = async (
+      userId: ObjectID,
+      isFromSchedule: boolean,
+    ): Promise<ObjectID> => {
+      if (isFromSchedule) {
+        return userId;
+      }
 
-        if (options.onCallPolicyId) {
-          routeAlertToUserId = await this.getRouteAlertToUserId({
+      if (options.onCallPolicyId) {
+        const routeAlertToUserId: ObjectID | null =
+          await this.getRouteAlertToUserId({
             userId,
             onCallDutyPolicyId: options.onCallPolicyId,
             projectId: options.projectId,
           });
+
+        if (routeAlertToUserId) {
+          return routeAlertToUserId;
         }
+      }
+
+      return userId;
+    };
+
+    const startUserNotificationRuleExecution: StartUserNotificationRuleExecutionFunction =
+      async (
+        originalUserId: ObjectID,
+        alertSentToUserId: ObjectID,
+        teamId: ObjectID | null,
+        scheduleId: ObjectID | null,
+      ): Promise<void> => {
+        /*
+         * This is where user is notified. alertSentToUserId is already resolved
+         * (override applied at most once by resolveAlertRecipientUserId).
+         */
 
-        const alertSentToUserId: ObjectID = routeAlertToUserId || userId;
+        const wasOverridden: boolean =
+          alertSentToUserId.toString() !== originalUserId.toString();
 
         logger.debug(
           `Starting notification rule execution for userId: ${alertSentToUserId.toString()}`,
@@ -354,8 +390,8 @@ export class Service extends DatabaseService<Model> {
         log.status = OnCallDutyExecutionLogTimelineStatus.Executing;
         log.alertSentToUserId = alertSentToUserId;
 
-        if (routeAlertToUserId) {
-          log.overridedByUserId = userId;
+        if (wasOverridden) {
+          log.overridedByUserId = originalUserId;
         }
 
         if (teamId) {
@@ -390,12 +426,36 @@ export class Service extends DatabaseService<Model> {
             onCallDutyPolicyExecutionLogTimelineId: log.id!,
             projectId: options.projectId,
             onCallScheduleId: scheduleId || undefined,
-            overridedByUserId: routeAlertToUserId ? userId : undefined,
+            overridedByUserId: wasOverridden ? originalUserId : undefined,
           },
         );
       };
 
+    /*
+     * Dedup on the RESOLVED recipient (post-override), not the pre-route id.
+     * Keying on the pre-route id let two different targets that both reroute to
+     * the same backup each fire a full notification chain for the same person in
+     * one rule (audit F6).
+     */
     const uniqueUserIds: Array<ObjectID> = [];
+    const alreadyNotified: (recipientId: ObjectID) => boolean = (
+      recipientId: ObjectID,
+    ): boolean => {
+      return Boolean(
+        uniqueUserIds.find((userId: ObjectID) => {
+          return recipientId.toString() === userId.toString();
+        }),
+      );
+    };
+
+    /*
+     * M-5: tracks whether any schedule target for this rule momentarily had no
+     * on-call user (a restriction gap / future start / empty layer). If a rule
+     * reaches NO ONE solely because of such a gap, we re-sample the same rule on
+     * the next cron ticks (bounded) instead of permanently skipping it, so a
+     * user coming on-call moments later still gets paged.
+     */
+    let hadScheduleGap: boolean = false;
 
     for (const teamInRule of teamsInRule) {
       const usersInTeam: Array<User> = await TeamMemberService.getUsersInTeam(
@@ -403,14 +463,16 @@ export class Service extends DatabaseService<Model> {
       );
 
       for (const user of usersInTeam) {
-        if (
-          !uniqueUserIds.find((userId: ObjectID) => {
-            return user.id?.toString() === userId.toString();
-          })
-        ) {
-          uniqueUserIds.push(user.id!);
+        const recipientUserId: ObjectID = await resolveAlertRecipientUserId(
+          user.id!,
+          false,
+        );
+
+        if (!alreadyNotified(recipientUserId)) {
+          uniqueUserIds.push(recipientUserId);
           await startUserNotificationRuleExecution(
             user.id!,
+            recipientUserId,
             teamInRule.teamId!,
             null,
           );
@@ -419,7 +481,7 @@ export class Service extends DatabaseService<Model> {
           log.statusMessage =
             "Skipped because notification sent to this user already.";
           log.status = OnCallDutyExecutionLogTimelineStatus.Skipped;
-          log.alertSentToUserId = user.id!;
+          log.alertSentToUserId = recipientUserId;
           log.userBelongsToTeamId = teamInRule.teamId!;
 
           await OnCallDutyPolicyExecutionLogTimelineService.create({
@@ -433,19 +495,25 @@ export class Service extends DatabaseService<Model> {
     }
 
     for (const userRule of usersInRule) {
-      if (
-        !uniqueUserIds.find((userId: ObjectID) => {
-          return userRule.userId?.toString() === userId.toString();
-        })
-      ) {
-        uniqueUserIds.push(userRule.userId!);
-        await startUserNotificationRuleExecution(userRule.userId!, null, null);
+      const recipientUserId: ObjectID = await resolveAlertRecipientUserId(
+        userRule.userId!,
+        false,
+      );
+
+      if (!alreadyNotified(recipientUserId)) {
+        uniqueUserIds.push(recipientUserId);
+        await startUserNotificationRuleExecution(
+          userRule.userId!,
+          recipientUserId,
+          null,
+          null,
+        );
       } else {
         const log: OnCallDutyPolicyExecutionLogTimeline = getNewLog();
         log.statusMessage =
           "Skipped because notification sent to this user already.";
         log.status = OnCallDutyExecutionLogTimelineStatus.Skipped;
-        log.alertSentToUserId = userRule.userId!;
+        log.alertSentToUserId = recipientUserId;
 
         await OnCallDutyPolicyExecutionLogTimelineService.create({
           data: log,
@@ -464,6 +532,8 @@ export class Service extends DatabaseService<Model> {
         );
 
       if (!userIdInSchedule) {
+        hadScheduleGap = true;
+
         const log: OnCallDutyPolicyExecutionLogTimeline = getNewLog();
         log.statusMessage =
           "Skipped because no active users are found in this schedule.";
@@ -480,14 +550,17 @@ export class Service extends DatabaseService<Model> {
         continue;
       }
 
-      if (
-        !uniqueUserIds.find((userId: ObjectID) => {
-          return userIdInSchedule?.toString() === userId.toString();
-        })
-      ) {
-        uniqueUserIds.push(userIdInSchedule);
+      // Schedule users are already override-resolved; no second override hop (F5).
+      const recipientUserId: ObjectID = await resolveAlertRecipientUserId(
+        userIdInSchedule,
+        true,
+      );
+
+      if (!alreadyNotified(recipientUserId)) {
+        uniqueUserIds.push(recipientUserId);
         await startUserNotificationRuleExecution(
           userIdInSchedule,
+          recipientUserId,
           null,
           scheduleRule.onCallDutyPolicyScheduleId!,
         );
@@ -496,7 +569,7 @@ export class Service extends DatabaseService<Model> {
         log.statusMessage =
           "Skipped because notification sent to this user already.";
         log.status = OnCallDutyExecutionLogTimelineStatus.Skipped;
-        log.alertSentToUserId = userIdInSchedule;
+        log.alertSentToUserId = recipientUserId;
         log.onCallDutyScheduleId = scheduleRule.onCallDutyPolicyScheduleId!;
 
         await OnCallDutyPolicyExecutionLogTimelineService.create({
@@ -521,6 +594,77 @@ export class Service extends DatabaseService<Model> {
       });
     }
 
+    /*
+     * M-5: if this rule reached NO ONE and the only reason was a schedule gap
+     * (a schedule target with no on-call user right now), re-sample the SAME
+     * rule on the next cron ticks — bounded — instead of permanently skipping.
+     * ExecutePendingExecutions selects the rule with order =
+     * lastExecutedEscalationRuleOrder + 1, so rewinding the order by one makes
+     * it re-run this rule. No notifications were sent (uniqueUserIds is empty),
+     * so a re-sample cannot double-page anyone.
+     */
+    const maxScheduleGapRetries: number = 5;
+    const scheduleGapRetryIntervalMinutes: number = 1;
+    const reachedNoOne: boolean = uniqueUserIds.length === 0;
+
+    if (reachedNoOne && hadScheduleGap && rule.order) {
+      const executionLog: OnCallDutyPolicyExecutionLog | null =
+        await OnCallDutyPolicyExecutionLogService.findOneById({
+          id: options.onCallPolicyExecutionLogId,
+          select: {
+            scheduleGapRetryCount: true,
+          },
+          props: {
+            isRoot: true,
+          },
+        });
+
+      const retryCount: number = executionLog?.scheduleGapRetryCount || 0;
+
+      if (retryCount < maxScheduleGapRetries) {
+        await OnCallDutyPolicyExecutionLogService.updateOneById({
+          id: options.onCallPolicyExecutionLogId,
+          data: {
+            lastExecutedEscalationRuleOrder: rule.order - 1,
+            executeNextEscalationRuleInMinutes: scheduleGapRetryIntervalMinutes,
+            scheduleGapRetryCount: retryCount + 1,
+          },
+          props: {
+            isRoot: true,
+          },
+        });
+
+        logger.debug(
+          `Rule ${ruleId.toString()} reached no one due to a schedule gap; re-sampling ${retryCount + 1}/${maxScheduleGapRetries}.`,
+          {
+            onCallDutyPolicyEscalationRuleId: ruleId.toString(),
+          } as LogAttributes,
+        );
+      } else {
+        // Retries exhausted — advance normally next tick and reset the counter.
+        await OnCallDutyPolicyExecutionLogService.updateOneById({
+          id: options.onCallPolicyExecutionLogId,
+          data: {
+            scheduleGapRetryCount: 0,
+          },
+          props: {
+            isRoot: true,
+          },
+        });
+      }
+    } else {
+      // Reached someone (or no schedule gap): clear retry state for the next rule.
+      await OnCallDutyPolicyExecutionLogService.updateOneById({
+        id: options.onCallPolicyExecutionLogId,
+        data: {
+          scheduleGapRetryCount: 0,
+        },
+        props: {
+          isRoot: true,
+        },
+      });
+    }
+
     logger.debug(`Completed rule execution for ruleId: ${ruleId.toString()}`, {
       projectId: options.projectId.toString(),
       onCallDutyPolicyEscalationRuleId: ruleId.toString(),
@@ -912,8 +1056,8 @@ export class Service extends DatabaseService<Model> {
         // moving down.
 
         for (const resource of resources) {
-          if (resource.order! <= newOrder) {
-            // increment order.
+          if (resource.order! > currentOrder && resource.order! <= newOrder) {
+            // decrement order to fill the gap left by the moved rule.
             await this.updateOneBy({
               query: {
                 _id: resource._id!,