@oneuptime/common 11.3.26 → 11.3.28

package/Server/Infrastructure/PostgresDatabase.ts

@@ -1,5 +1,6 @@
 import logger from "../Utils/Logger";
 import DatabaseDataSourceOptions from "./Postgres/DataSourceOptions";
+import { recordSchemaMigrationFailureBestEffort } from "../Utils/Database/MigrationFailureLog";
 import Sleep from "../../Types/Sleep";
 import { DataSource, DataSourceOptions, QueryRunner } from "typeorm";
 import { createDatabase, dropDatabase } from "typeorm-extension";
@@ -95,6 +96,16 @@ export default class Database {
     } catch (err) {
       logger.error("Postgres Database Connection Failed");
       logger.error(err);
+
+      /*
+       * When this process runs schema migrations on boot (migrationsRun=true),
+       * connect() also fails if a migration threw. Record which migration
+       * failed and why — on a throwaway connection, since the DataSource above
+       * is unusable — so the admin health page can explain the pending schema.
+       * Best-effort and self-contained: it never throws and never masks `err`.
+       */
+      await recordSchemaMigrationFailureBestEffort(dataSourceOptions, err);
+
       throw err;
     }
   }

package/Server/Services/AIService.ts

@@ -11,7 +11,10 @@ import LLMService, {
   LLMMessage,
   LLMToolCall,
   LLMToolDefinition,
+  LLMUsage,
 } from "../Utils/LLM/LLMService";
+import LlmType from "../../Types/LLM/LlmType";
+import { Span, trace } from "@opentelemetry/api";
 import LlmProvider from "../../Models/DatabaseModels/LlmProvider";
 import LlmLog from "../../Models/DatabaseModels/LlmLog";
 import LlmLogStatus from "../../Types/LlmLogStatus";
@@ -129,9 +132,17 @@ export class Service extends BaseService {
       logEntry.aiRunId = request.aiRunId;
     }
 
-    // Check if billing should apply
+    /*
+     * Check if billing should apply. Only bill for the global (OneUptime-hosted)
+     * provider, and only when it actually has a per-token cost. A free global
+     * provider (costPerMillionTokensInUSDCents = 0, the default) consumes no
+     * balance, so it must not require or block on one either — otherwise a $0
+     * provider would still fail with "Insufficient AI balance".
+     */
     const shouldBill: boolean =
-      IsBillingEnabled && (llmProvider.isGlobalLlm || false);
+      IsBillingEnabled &&
+      (llmProvider.isGlobalLlm || false) &&
+      (llmProvider.costPerMillionTokensInUSDCents || 0) > 0;
 
     // Check balance if billing enabled and using global provider
     if (shouldBill) {
@@ -234,6 +245,18 @@ export class Service extends BaseService {
         }
       }
 
+      /*
+       * Emit gen_ai.* semantic-convention attributes on the active span so
+       * OneUptime's own AI usage is a first-class LLM span in OneUptime's own
+       * telemetry (dogfooding — LlmSpanUtil detects these). Never fails the call.
+       */
+      this.setGenAiSpanAttributes({
+        llmType: llmProvider.llmType,
+        modelName: llmConfig.modelName,
+        usage: response.usage,
+        costInUSDCents: logEntry.costInUSDCents,
+      });
+
       // Save log entry
       const savedLog: LlmLog = await LlmLogService.create({
         data: logEntry,
@@ -261,6 +284,55 @@ export class Service extends BaseService {
       throw error;
     }
   }
+
+  /*
+   * Set gen_ai.* attributes (OpenTelemetry GenAI semantic conventions) on the
+   * currently-active span. The @CaptureSpan()-wrapped caller owns that span, so
+   * LlmSpanUtil recognizes these calls as first-class LLM spans.
+   */
+  private setGenAiSpanAttributes(data: {
+    llmType: LlmType;
+    modelName?: string | undefined;
+    usage?: LLMUsage | undefined;
+    costInUSDCents?: number | undefined;
+  }): void {
+    try {
+      const span: Span | undefined = trace.getActiveSpan();
+      if (!span) {
+        return;
+      }
+
+      span.setAttribute("gen_ai.system", data.llmType.toString());
+      span.setAttribute("gen_ai.provider.name", data.llmType.toString());
+      span.setAttribute("gen_ai.operation.name", "chat");
+
+      if (data.modelName) {
+        span.setAttribute("gen_ai.request.model", data.modelName);
+        span.setAttribute("gen_ai.response.model", data.modelName);
+      }
+
+      if (data.usage) {
+        span.setAttribute(
+          "gen_ai.usage.input_tokens",
+          data.usage.promptTokens || 0,
+        );
+        span.setAttribute(
+          "gen_ai.usage.output_tokens",
+          data.usage.completionTokens || 0,
+        );
+        span.setAttribute(
+          "gen_ai.usage.total_tokens",
+          data.usage.totalTokens || 0,
+        );
+      }
+
+      if (data.costInUSDCents) {
+        span.setAttribute("gen_ai.usage.cost_usd", data.costInUSDCents / 100);
+      }
+    } catch {
+      // Telemetry must never fail the LLM call.
+    }
+  }
 }
 
 export default new Service();

package/Server/Services/AnalyticsDatabaseService.ts

@@ -1899,7 +1899,16 @@ export default class AnalyticsDatabaseService<
     return data;
   }
 
-  protected async getException(error: Exception): Promise<void> {
+  /*
+   * Rethrow hook for the catch blocks below. MUST stay synchronous and
+   * `never`-returning: the call sites use `throw this.getException(error)`,
+   * so if this were `async` it would return a Promise that `throw` then
+   * throws verbatim (never awaited) — the real exception is lost, callers up
+   * the stack catch a bare Promise (surfacing as "[object Promise]"), and the
+   * un-awaited rejection becomes an unhandled rejection. Throwing directly
+   * propagates the original exception on the synchronous throw path.
+   */
+  protected getException(error: Exception): never {
     throw error;
   }
 

package/Server/Services/DatabaseService.ts

@@ -443,7 +443,16 @@ class DatabaseService<TBaseModel extends BaseModel> extends BaseService {
     return Promise.resolve(error);
   }
 
-  protected async getException(error: Exception): Promise<void> {
+  /*
+   * Rethrow hook for the catch blocks below. MUST stay synchronous and
+   * `never`-returning: the call sites use `throw this.getException(error)`,
+   * so if this were `async` it would return a Promise that `throw` then
+   * throws verbatim (never awaited) — the real exception is lost, callers up
+   * the stack catch a bare Promise (surfacing as "[object Promise]"), and the
+   * un-awaited rejection becomes an unhandled rejection. Throwing directly
+   * propagates the original exception on the synchronous throw path.
+   */
+  protected getException(error: Exception): never {
     throw error;
   }
 

package/Server/Services/IncidentService.ts

@@ -3479,7 +3479,7 @@ ${incidentSeverity.name}
     const response: LLMCompletionResponse = await LLMService.getCompletion({
       llmProviderConfig: llmConfig,
       messages: aiContext.messages,
-      temperature: 0.7,
+      temperature: 0.2,
     });
 
     return response.content;

package/Server/Services/IncomingCallPolicyEscalationRuleService.ts

@@ -135,6 +135,79 @@ export class Service extends DatabaseService<IncomingCallPolicyEscalationRule> {
   protected override async onBeforeUpdate(
     updateBy: UpdateBy<IncomingCallPolicyEscalationRule>,
   ): Promise<OnUpdate<IncomingCallPolicyEscalationRule>> {
+    /*
+     * Enforce user/schedule mutual exclusivity on update (parity with onBeforeCreate).
+     * Only runs when the update actually touches one of the routing-target fields, so
+     * internal updates (status/order via isRoot) are unaffected.
+     */
+    const data: UpdateBy<IncomingCallPolicyEscalationRule>["data"] =
+      updateBy.data;
+    const isTouchingTarget: boolean =
+      data.userId !== undefined ||
+      data.onCallDutyPolicyScheduleId !== undefined;
+
+    if (isTouchingTarget && updateBy.query._id) {
+      const settingUser: boolean = Boolean(data.userId);
+      const settingSchedule: boolean = Boolean(data.onCallDutyPolicyScheduleId);
+
+      if (settingUser && settingSchedule) {
+        throw new BadDataException(
+          "Only one of User or On-Call Schedule can be specified, not both",
+        );
+      }
+
+      // Setting one target clears the other so a rule can never hold both.
+      const nullableData: {
+        userId?: ObjectID | null;
+        onCallDutyPolicyScheduleId?: ObjectID | null;
+      } = data as {
+        userId?: ObjectID | null;
+        onCallDutyPolicyScheduleId?: ObjectID | null;
+      };
+      if (settingUser) {
+        nullableData.onCallDutyPolicyScheduleId = null;
+      }
+      if (settingSchedule) {
+        nullableData.userId = null;
+      }
+
+      const existing: IncomingCallPolicyEscalationRule | null =
+        await this.findOneBy({
+          query: {
+            _id: updateBy.query._id!,
+          },
+          select: {
+            userId: true,
+            onCallDutyPolicyScheduleId: true,
+          },
+          props: {
+            isRoot: true,
+          },
+        });
+
+      /*
+       * Whether each target will be present AFTER this update, accounting for
+       * fields left untouched (keep existing) and the opposite-field clearing
+       * done above.
+       */
+      const willHaveUser: boolean = settingUser
+        ? true
+        : data.userId === undefined
+          ? Boolean(existing?.userId)
+          : false;
+      const willHaveSchedule: boolean = settingSchedule
+        ? true
+        : data.onCallDutyPolicyScheduleId === undefined
+          ? Boolean(existing?.onCallDutyPolicyScheduleId)
+          : false;
+
+      if (!willHaveUser && !willHaveSchedule) {
+        throw new BadDataException(
+          "Either a User or an On-Call Schedule must be specified for the escalation rule",
+        );
+      }
+    }
+
     if (updateBy.data.order && !updateBy.props.isRoot && updateBy.query._id) {
       const resource: IncomingCallPolicyEscalationRule | null =
         await this.findOneBy({
@@ -194,8 +267,8 @@ export class Service extends DatabaseService<IncomingCallPolicyEscalationRule> {
       if (newOrder > currentOrder) {
         // moving down.
         for (const resource of resources) {
-          if (resource.order! <= newOrder) {
-            // increment order.
+          if (resource.order! > currentOrder && resource.order! <= newOrder) {
+            // decrement order to fill the gap left by the moved rule.
             await this.updateOneBy({
               query: {
                 _id: resource._id!,

package/Server/Services/IncomingCallPolicyService.ts

@@ -2,7 +2,10 @@ import DatabaseService from "./DatabaseService";
 import IncomingCallPolicy from "../../Models/DatabaseModels/IncomingCallPolicy";
 import IncomingCallPolicyLabelRuleEngineService from "./IncomingCallPolicyLabelRuleEngineService";
 import IncomingCallPolicyOwnerRuleEngineService from "./IncomingCallPolicyOwnerRuleEngineService";
-import { OnCreate } from "../Types/Database/Hooks";
+import { OnCreate, OnDelete } from "../Types/Database/Hooks";
+import DeleteBy from "../Types/Database/DeleteBy";
+import LIMIT_MAX from "../../Types/Database/LimitMax";
+import releaseIncomingCallPhoneNumber from "../Utils/IncomingCallPhoneNumber";
 import CaptureSpan from "../Utils/Telemetry/CaptureSpan";
 import logger, { LogAttributes } from "../Utils/Logger";
 
@@ -11,6 +14,44 @@ export class Service extends DatabaseService<IncomingCallPolicy> {
     super(IncomingCallPolicy);
   }
 
+  @CaptureSpan()
+  protected override async onBeforeDelete(
+    deleteBy: DeleteBy<IncomingCallPolicy>,
+  ): Promise<OnDelete<IncomingCallPolicy>> {
+    /*
+     * Release any provisioned numbers before the policy rows are removed so we
+     * don't leave paid, orphaned numbers on the provider that can never be
+     * released again (the policy — and its SID — would be gone).
+     */
+    const policies: Array<IncomingCallPolicy> = await this.findBy({
+      query: deleteBy.query,
+      select: {
+        _id: true,
+        callProviderPhoneNumberId: true,
+        projectCallSMSConfigId: true,
+      },
+      limit: LIMIT_MAX,
+      skip: 0,
+      props: {
+        isRoot: true,
+      },
+    });
+
+    for (const policy of policies) {
+      if (policy.callProviderPhoneNumberId && policy.projectCallSMSConfigId) {
+        await releaseIncomingCallPhoneNumber({
+          projectCallSMSConfigId: policy.projectCallSMSConfigId,
+          callProviderPhoneNumberId: policy.callProviderPhoneNumberId,
+        });
+      }
+    }
+
+    return {
+      deleteBy,
+      carryForward: null,
+    };
+  }
+
   @CaptureSpan()
   protected override async onCreateSuccess(
     _onCreate: OnCreate<IncomingCallPolicy>,

package/Server/Services/Index.ts

@@ -12,6 +12,7 @@ import BillingService from "./BillingService";
 import CallLogService from "./CallLogService";
 import CallService from "./CallService";
 import DataMigrationService from "./DataMigrationService";
+import MigrationFailureService from "./MigrationFailureService";
 import DomainService from "./DomainService";
 import EmailLogService from "./EmailLogService";
 import EmailVerificationTokenService from "./EmailVerificationTokenService";
@@ -273,6 +274,7 @@ const services: Array<BaseService> = [
   CallService,
 
   DataMigrationService,
+  MigrationFailureService,
   DomainService,
 
   EmailLogService,

package/Server/Services/MigrationFailureService.ts

@@ -0,0 +1,9 @@
+import DatabaseService from "./DatabaseService";
+import Model from "../../Models/DatabaseModels/MigrationFailure";
+
+export class Service extends DatabaseService<Model> {
+  public constructor() {
+    super(Model);
+  }
+}
+export default new Service();

package/Server/Services/MonitorProbeService.ts

@@ -310,8 +310,14 @@ export class Service extends DatabaseService<MonitorProbe> {
     _onCreate: OnCreate<MonitorProbe>,
     createdItem: MonitorProbe,
   ): Promise<MonitorProbe> {
-    if (createdItem.probeId) {
-      await MonitorService.refreshProbeStatus(createdItem.probeId);
+    /*
+     * Refresh only the monitor this probe was just added to. Do NOT call
+     * refreshProbeStatus(probeId) here: a global/shared probe can be attached
+     * to thousands of monitors, and refreshing every one of them on a single
+     * create storms the database and can 500 the request.
+     */
+    if (createdItem.monitorId) {
+      await this.refreshMonitorStatusSafely(createdItem.monitorId);
     }
 
     return Promise.resolve(createdItem);
@@ -321,7 +327,7 @@ export class Service extends DatabaseService<MonitorProbe> {
     onUpdate: OnUpdate<MonitorProbe>,
     updatedItemIds: ObjectID[],
   ): Promise<OnUpdate<MonitorProbe>> {
-    // if isEnabled is updated, refresh the probe status
+    // if isEnabled is updated, refresh the status of the affected monitors.
     if (onUpdate.updateBy.data.isEnabled !== undefined) {
       const monitorProbes: Array<MonitorProbe> = await this.findBy({
         query: {
@@ -329,8 +335,6 @@ export class Service extends DatabaseService<MonitorProbe> {
         },
         select: {
           monitorId: true,
-          probeId: true,
-          nextPingAt: true,
         },
         limit: LIMIT_PER_PROJECT,
         skip: 0,
@@ -340,16 +344,38 @@ export class Service extends DatabaseService<MonitorProbe> {
       });
 
       for (const monitorProbe of monitorProbes) {
-        if (!monitorProbe.probeId) {
+        if (!monitorProbe.monitorId) {
           continue;
         }
 
-        await MonitorService.refreshProbeStatus(monitorProbe.probeId);
+        /*
+         * Refresh only the monitor whose probe row changed. Do NOT call
+         * refreshProbeStatus(probeId) here: a global/shared probe can be
+         * attached to thousands of monitors, and refreshing all of them on a
+         * single enable/disable toggle storms the database and 500s the
+         * request (the row itself is already saved by this point).
+         */
+        await this.refreshMonitorStatusSafely(monitorProbe.monitorId);
       }
     }
 
     return onUpdate;
   }
+
+  /*
+   * Refresh a single monitor's aggregate probe status without letting a
+   * failure bubble up. These refreshes run in post-commit hooks
+   * (onCreateSuccess / onUpdateSuccess) after the MonitorProbe row is already
+   * persisted, so a status-refresh error must not turn a successful save into
+   * a 500 for the user — log it instead.
+   */
+  private async refreshMonitorStatusSafely(monitorId: ObjectID): Promise<void> {
+    try {
+      await MonitorService.refreshMonitorProbeStatus(monitorId);
+    } catch (err) {
+      logger.error(err);
+    }
+  }
 }
 
 export default new Service();

package/Server/Services/MonitorService.ts

@@ -1620,14 +1620,26 @@ ${createdItem.description?.trim() || "No description provided."}
 
     /*
      * Each monitor appears at most once for a given probeId (composite
-     * unique on MonitorProbe), so concurrent refreshes operate on
-     * disjoint rows and are safe to run in parallel.
+     * unique on MonitorProbe), so concurrent refreshes operate on disjoint
+     * rows and are safe to run in parallel. A global/shared probe can be
+     * attached to thousands of monitors, though, so refresh in bounded
+     * batches instead of firing every refresh at once — an unbounded
+     * Promise.all here would exhaust the database connection pool.
      */
-    await Promise.all(
-      monitorProbes.map((monitorProbe: MonitorProbe) => {
-        return this.refreshMonitorProbeStatus(monitorProbe.monitorId!);
-      }),
-    );
+    const refreshConcurrency: number = 50;
+
+    for (let i: number = 0; i < monitorProbes.length; i += refreshConcurrency) {
+      const batch: Array<MonitorProbe> = monitorProbes.slice(
+        i,
+        i + refreshConcurrency,
+      );
+
+      await Promise.all(
+        batch.map((monitorProbe: MonitorProbe) => {
+          return this.refreshMonitorProbeStatus(monitorProbe.monitorId!);
+        }),
+      );
+    }
   }
 
   @CaptureSpan()

package/Server/Services/OnCallDutyPolicyEscalationRuleScheduleService.ts

@@ -22,6 +22,9 @@ import PushNotificationMessage from "../../Types/PushNotification/PushNotificati
 import PushNotificationUtil from "../Utils/PushNotificationUtil";
 import { createWhatsAppMessageFromTemplate } from "../Utils/WhatsAppTemplateUtil";
 import { WhatsAppMessagePayload } from "../../Types/WhatsApp/WhatsAppMessage";
+import OnCallDutyPolicyTimeLogService from "./OnCallDutyPolicyTimeLogService";
+import OneUptimeDate from "../../Types/Date";
+import logger, { LogAttributes } from "../Utils/Logger";
 
 export class Service extends DatabaseService<Model> {
   public constructor() {
@@ -86,6 +89,39 @@ export class Service extends DatabaseService<Model> {
       return createdItem;
     }
 
+    /*
+     * Open the roster user's on-call time log for this schedule/rule/policy.
+     * Mirrors onDeleteSuccess's endTimeLogForUser (detaching a schedule closes
+     * the log, so attaching must open it). Without this, attaching a schedule
+     * whose roster user is ALREADY established never opened a log — the roster
+     * refresh only opens one when the current user CHANGES — so the whole stint
+     * went unrecorded in on-call reporting and the later detach's
+     * endTimeLogForUser was a no-op (audit M1). startTimeLogForUser is idempotent
+     * (open-log dedup), so a later natural handoff re-opening the same log is a
+     * no-op.
+     */
+    if (
+      createdModel.onCallDutyPolicy?.id &&
+      createdModel.onCallDutyPolicyEscalationRule?.id &&
+      createdModel.projectId &&
+      createdModel.onCallDutyPolicyScheduleId
+    ) {
+      OnCallDutyPolicyTimeLogService.startTimeLogForUser({
+        projectId: createdModel.projectId,
+        onCallDutyPolicyId: createdModel.onCallDutyPolicy.id,
+        onCallDutyPolicyEscalationRuleId:
+          createdModel.onCallDutyPolicyEscalationRule.id,
+        userId: userOnSchedule,
+        onCallDutyPolicyScheduleId: createdModel.onCallDutyPolicyScheduleId,
+        startsAt: OneUptimeDate.getCurrentDate(),
+      }).catch((error: Error) => {
+        logger.error(error, {
+          projectId: createdModel.projectId?.toString(),
+          userId: userOnSchedule?.toString(),
+        } as LogAttributes);
+      });
+    }
+
     const scheduleName: string =
       createdModel.onCallDutyPolicySchedule?.name || "No name provided";
 
@@ -289,6 +325,35 @@ export class Service extends DatabaseService<Model> {
         continue;
       }
 
+      /*
+       * Close the roster user's open on-call time log for this schedule/rule/
+       * policy. Roster handoffs open a time log keyed by schedule + rule + policy
+       * (OnCallDutyPolicyScheduleService.startTimeLogForUser); the user-link and
+       * team-link delete handlers close theirs, but detaching a SCHEDULE from the
+       * rule left the log open forever, so the roster user kept showing as
+       * on-call in reporting (audit F18).
+       */
+      if (
+        deletedItem.onCallDutyPolicy?.id &&
+        deletedItem.onCallDutyPolicyEscalationRule?.id &&
+        deletedItem.projectId
+      ) {
+        OnCallDutyPolicyTimeLogService.endTimeLogForUser({
+          projectId: deletedItem.projectId,
+          onCallDutyPolicyId: deletedItem.onCallDutyPolicy.id,
+          onCallDutyPolicyEscalationRuleId:
+            deletedItem.onCallDutyPolicyEscalationRule.id,
+          userId: userOnSchedule,
+          onCallDutyPolicyScheduleId: deletedItem.onCallDutyPolicyScheduleId!,
+          endsAt: OneUptimeDate.getCurrentDate(),
+        }).catch((error: Error) => {
+          logger.error(error, {
+            projectId: deletedItem.projectId?.toString(),
+            userId: userOnSchedule?.toString(),
+          } as LogAttributes);
+        });
+      }
+
       const sendEmailToUserId: ObjectID | undefined | null = userOnSchedule;
 
       if (!sendEmailToUserId) {