@oneuptime/common 12.0.29 → 12.0.31

package/Server/Services/TelemetrySourceMapService.ts

@@ -6,12 +6,20 @@ import BadDataException from "../../Types/Exception/BadDataException";
 import ObjectID from "../../Types/ObjectID";
 import SortOrder from "../../Types/BaseDatabase/SortOrder";
 import LIMIT_MAX from "../../Types/Database/LimitMax";
+import PositiveNumber from "../../Types/PositiveNumber";
 import QueryHelper from "../Types/Database/QueryHelper";
 import CaptureSpan from "../Utils/Telemetry/CaptureSpan";
+import logger from "../Utils/Logger";
 import SourceMapResolver, {
+  MAX_FRAMES_TO_RESOLVE,
   MAX_SOURCE_MAP_SIZE_IN_BYTES,
   SourceMapBundle,
 } from "../Utils/Telemetry/SourceMapResolver";
+import {
+  SourceMapMaxBytesPerResolve,
+  SourceMapMaxMapsPerRelease,
+  SourceMapRetentionInDays,
+} from "../EnvironmentConfig";
 import {
   MinifiedStackFrame,
   ResolvedStackFrame,
@@ -23,17 +31,24 @@ export { MAX_SOURCE_MAP_SIZE_IN_BYTES };
 /*
  * How long uploaded maps are kept. Old releases age out automatically —
  * a map is only useful while exceptions from its release are still within
- * telemetry retention, and 90 days comfortably exceeds the default.
+ * telemetry retention, and the default comfortably exceeds it.
+ * SOURCE_MAP_RETENTION_DAYS overrides it.
  */
-export const SOURCE_MAP_RETENTION_DAYS: number = 90;
+export const SOURCE_MAP_RETENTION_DAYS: number = SourceMapRetentionInDays;
 
 /*
- * How many maps one (service, release) pair may hold. Enforced by the
- * upload endpoint (existing bundles + new bundles must fit) and used as
- * the resolver's read limit, so nothing stored is ever silently ignored.
- * A build rarely emits more than a few dozen chunks with maps.
+ * How many distinct bundles one (service, release) pair may hold, from
+ * SOURCE_MAP_MAX_MAPS_PER_RELEASE.
+ *
+ * This is a WRITE gate and nothing else: the upload endpoint checks that
+ * existing bundles + new bundles fit, and rejects with a 400 naming the
+ * limit. It used to double as the resolver's read limit, which coupled two
+ * different quantities — distinct bundle paths on the write side, rows on
+ * the read side — and made raising it dangerous. resolveFramesForService now
+ * bounds itself by BYTES loaded (SourceMapMaxBytesPerResolve), so a release
+ * that fits this ceiling always resolves in full.
  */
-export const MAX_SOURCE_MAPS_PER_RELEASE: number = 100;
+export const MAX_SOURCE_MAPS_PER_RELEASE: number = SourceMapMaxMapsPerRelease;
 
 export class Service extends DatabaseService<Model> {
   public constructor() {
@@ -74,11 +89,116 @@ export class Service extends DatabaseService<Model> {
       throw new BadDataException("Service version is required.");
     }
 
+    await this.assertReleaseHasRoomFor(createBy);
+
     createBy.data.sizeInBytes = Buffer.byteLength(content, "utf8");
 
     return { createBy, carryForward: null };
   }
 
+  /**
+   * Enforce the per-release ceiling on the create path itself.
+   *
+   * The upload endpoint checks the whole batch up front, which is where the
+   * useful error message lives — but TelemetrySourceMap is a full CRUD
+   * resource, so POST /api/telemetry-source-map reaches create() without ever
+   * passing that check. Without this the configured ceiling was advisory on
+   * that path, and a release could quietly grow past what the operator set.
+   *
+   * Two steps, so the common case stays cheap: an indexed COUNT on every
+   * insert, and the exact distinct-bundle answer only once that COUNT says
+   * the release is at the boundary. A release is written once per CI run, and
+   * on the upload path the batch check has already passed, so the second step
+   * is effectively unreachable there.
+   */
+  @CaptureSpan()
+  private async assertReleaseHasRoomFor(
+    createBy: CreateBy<Model>,
+  ): Promise<void> {
+    const projectId: ObjectID | undefined =
+      createBy.data.projectId || createBy.props.tenantId || undefined;
+    const serviceId: ObjectID | undefined =
+      createBy.data.serviceId || undefined;
+    const serviceVersion: string | undefined = createBy.data.serviceVersion;
+
+    /*
+     * Not a silent pass: a create missing any of these cannot be scoped to a
+     * release, and the tenant column / required-column checks reject it on
+     * their own with a better message than a count could give.
+     */
+    if (!projectId || !serviceId || !serviceVersion) {
+      return;
+    }
+
+    /*
+     * Cheap check first, and it is the one that runs in practice: a COUNT of
+     * ROWS on the (projectId, serviceId, serviceVersion) index. Below the
+     * ceiling there is nothing to decide, whatever the rows contain.
+     */
+    const existingRowCount: PositiveNumber = await this.countBy({
+      query: {
+        projectId: projectId,
+        serviceId: serviceId,
+        serviceVersion: serviceVersion,
+      },
+      skip: 0,
+      limit: LIMIT_MAX,
+      props: {
+        isRoot: true,
+      },
+    });
+
+    if (existingRowCount.toNumber() < MAX_SOURCE_MAPS_PER_RELEASE) {
+      return;
+    }
+
+    /*
+     * At or above the ceiling on rows — but rows are not the unit the ceiling
+     * is expressed in. The model deliberately allows more than one row per
+     * bundle (a racing double upload must not fail CI), so rejecting on the
+     * row count would fail a legitimate re-upload of a full release purely
+     * because a duplicate row exists. Pay for the exact answer only here,
+     * where the fast path has already established this is the boundary.
+     */
+    const storedBundlePaths: Array<string> =
+      await this.getStoredBundlePathsForRelease({
+        projectId: projectId,
+        serviceId: serviceId,
+        serviceVersion: serviceVersion,
+      });
+
+    const distinctBundlePaths: Set<string> = new Set(
+      storedBundlePaths.map((bundlePath: string) => {
+        return SourceMapResolver.normalizePath(bundlePath);
+      }),
+    );
+
+    /*
+     * Normalized, which makes this backstop slightly MORE permissive than the
+     * upload endpoint's raw-path gate. That is the right direction for a
+     * backstop: it must never be the thing that fails a request the primary
+     * gate already approved.
+     */
+    const incomingBundlePath: string = SourceMapResolver.normalizePath(
+      createBy.data.bundlePath || "",
+    );
+
+    /*
+     * Adding a row for a bundle the release already holds does not grow the
+     * release, so it stays allowed even at the ceiling. Only a NEW distinct
+     * bundle is refused.
+     */
+    if (distinctBundlePaths.has(incomingBundlePath)) {
+      return;
+    }
+
+    if (distinctBundlePaths.size >= MAX_SOURCE_MAPS_PER_RELEASE) {
+      throw new BadDataException(
+        `This release already holds ${distinctBundlePaths.size} source maps, which is the limit of ${MAX_SOURCE_MAPS_PER_RELEASE} per release. Delete unused maps, use a new serviceVersion, or raise SOURCE_MAP_MAX_MAPS_PER_RELEASE.`,
+      );
+    }
+  }
+
   /**
    * Bundle paths currently stored for a (project, service, release) tuple.
    * Used by the upload endpoint to enforce the per-release ceiling without
@@ -182,15 +302,26 @@ export class Service extends DatabaseService<Model> {
         }),
         resolvedCount: 0,
         sourceMapCount: 0,
+        sourceMapsSkippedForSize: 0,
       };
     }
 
     /*
-     * Two-step fetch: list bundle paths first (cheap), then pull content
-     * only for bundles that at least one frame can match. Maps can be tens
-     * of megabytes each and a release may store up to
-     * MAX_SOURCE_MAPS_PER_RELEASE of them — loading them all for every
-     * exception view would be a memory hazard for no benefit.
+     * Two-step fetch: list bundle paths first (cheap — no content), then pull
+     * content only for the bundles a frame can match, and only as many of
+     * those as the byte budget allows. Maps can be tens of megabytes each, so
+     * loading a whole release for every exception view would be a memory
+     * hazard for no benefit.
+     *
+     * The listing reads with LIMIT_MAX rather than the per-release ceiling.
+     * The ceiling counts distinct bundle PATHS, while a read limit counts
+     * ROWS, and duplicate rows for one bundle are deliberately allowed (see
+     * the model: a racing double upload must not fail CI). With the two
+     * numbers equal, one duplicate row was enough to push a distinct bundle
+     * out of resolution — it stored fine, showed in the dashboard, and
+     * silently never resolved. Reading everything and deduping afterwards
+     * removes that coupling; memory is bounded below by bytes, not by how
+     * many rows a content-free listing returns.
      */
     const sourceMapRows: Array<Model> = await this.findBy({
       query: {
@@ -202,11 +333,12 @@ export class Service extends DatabaseService<Model> {
         _id: true,
         bundlePath: true,
         createdAt: true,
+        sizeInBytes: true,
       },
       sort: {
         createdAt: SortOrder.Descending,
       },
-      limit: MAX_SOURCE_MAPS_PER_RELEASE,
+      limit: LIMIT_MAX,
       skip: 0,
       props: {
         isRoot: true,
@@ -237,16 +369,121 @@ export class Service extends DatabaseService<Model> {
       dedupedRows.push(row);
     }
 
-    const matchedRows: Array<Model> = dedupedRows.filter((row: Model) => {
-      return data.frames.some((frame: MinifiedStackFrame) => {
-        return (
-          SourceMapResolver.getMatchScore(
-            frame.fileName,
-            row.bundlePath || "",
-          ) > 0
+    /*
+     * Only the frames resolveFrames will actually attempt can pull a map in.
+     * Matching against every frame would let a caller-supplied array drive an
+     * O(rows x frames) scan and select maps that could never be used anyway —
+     * the frames array is parsed from client-supplied stack trace text, so it
+     * is capped rather than trusted (the API rejects absurd lengths outright).
+     */
+    const framesForMatching: Array<MinifiedStackFrame> = data.frames.slice(
+      0,
+      MAX_FRAMES_TO_RESOLVE,
+    );
+
+    /*
+     * Score every candidate against those frames. The score is how many
+     * trailing path segments the frame and the bundle share, so a frame
+     * "static/main.a8f1.js" prefers the bundle that agrees on "static/" over
+     * one that only agrees on the file name. That ordering is what makes the
+     * byte budget below defensible: when not everything fits, what gets
+     * dropped is the weakest match, not an arbitrary row.
+     */
+    const scoredRows: Array<{
+      row: Model;
+      score: number;
+      sizeInBytes: number;
+    }> = [];
+
+    for (const row of dedupedRows) {
+      let bestScore: number = 0;
+
+      for (const frame of framesForMatching) {
+        const score: number = SourceMapResolver.getMatchScore(
+          frame.fileName,
+          row.bundlePath || "",
         );
-      });
-    });
+
+        if (score > bestScore) {
+          bestScore = score;
+        }
+      }
+
+      if (bestScore > 0) {
+        scoredRows.push({
+          row: row,
+          score: bestScore,
+          /*
+           * onBeforeCreate stamps sizeInBytes on every insert, so a missing
+           * value means a row written straight to the database. Charge it the
+           * per-map ceiling rather than nothing: an unknown size must not be
+           * the way to slip past a budget meant to bound memory.
+           */
+          sizeInBytes:
+            typeof row.sizeInBytes === "number" &&
+            Number.isFinite(row.sizeInBytes) &&
+            row.sizeInBytes > 0
+              ? row.sizeInBytes
+              : MAX_SOURCE_MAP_SIZE_IN_BYTES,
+        });
+      }
+    }
+
+    scoredRows.sort(
+      (
+        a: { row: Model; score: number; sizeInBytes: number },
+        b: { row: Model; score: number; sizeInBytes: number },
+      ) => {
+        if (b.score !== a.score) {
+          return b.score - a.score;
+        }
+
+        // Same match quality: prefer the newer upload, as the dedupe does.
+        const aTime: number = a.row.createdAt
+          ? new Date(a.row.createdAt).getTime()
+          : 0;
+        const bTime: number = b.row.createdAt
+          ? new Date(b.row.createdAt).getTime()
+          : 0;
+
+        return bTime - aTime;
+      },
+    );
+
+    /*
+     * The byte budget. This is the bound that used to be implied by the
+     * per-release count and is a far tighter one, because it measures the
+     * thing that actually consumes memory. Lower-scoring maps that still fit
+     * are kept (hence continue, not break) so a single oversized map does not
+     * cost the rest of the stack trace its symbols.
+     *
+     * The best-scoring map is always attempted even if it alone exceeds the
+     * budget: a budget configured below the per-map ceiling should degrade to
+     * "one map at a time", not to "nothing ever resolves". Peak is therefore
+     * max(SourceMapMaxBytesPerResolve, MAX_SOURCE_MAP_SIZE_IN_BYTES).
+     */
+    const matchedRows: Array<Model> = [];
+    let budgetUsedInBytes: number = 0;
+    let sourceMapsSkippedForSize: number = 0;
+
+    for (const scored of scoredRows) {
+      const wouldExceedBudget: boolean =
+        budgetUsedInBytes + scored.sizeInBytes > SourceMapMaxBytesPerResolve;
+
+      if (wouldExceedBudget && matchedRows.length > 0) {
+        sourceMapsSkippedForSize++;
+        continue;
+      }
+
+      budgetUsedInBytes += scored.sizeInBytes;
+      matchedRows.push(scored.row);
+    }
+
+    if (sourceMapsSkippedForSize > 0) {
+      logger.warn(
+        `Source map resolution skipped ${sourceMapsSkippedForSize} matching map(s) for service ${data.serviceId.toString()} release ${data.serviceVersion}: loading them would exceed SOURCE_MAP_MAX_BYTES_PER_RESOLVE (${SourceMapMaxBytesPerResolve} bytes).`,
+      );
+    }
 
     const bundles: Array<SourceMapBundle> = [];
 
@@ -269,7 +506,7 @@ export class Service extends DatabaseService<Model> {
           bundlePath: true,
           content: true,
         },
-        limit: MAX_SOURCE_MAPS_PER_RELEASE,
+        limit: matchedRows.length,
         skip: 0,
         props: {
           isRoot: true,
@@ -300,6 +537,12 @@ export class Service extends DatabaseService<Model> {
        * distinguishable from "no maps uploaded".
        */
       sourceMapCount: dedupedRows.length,
+      /*
+       * Non-zero means symbols are missing because of the byte budget, not
+       * because the maps were never uploaded — the one distinction a user
+       * staring at a half-resolved stack trace cannot make for themselves.
+       */
+      sourceMapsSkippedForSize: sourceMapsSkippedForSize,
     };
   }
 }

package/Server/Services/UserNotificationSettingService.ts

@@ -68,7 +68,13 @@ export class Service extends DatabaseService<UserNotificationSetting> {
     smsMessage: SMSMessage;
     callRequestMessage: CallRequestMessage;
     pushNotificationMessage: PushNotificationMessage;
-    whatsAppMessage: WhatsAppMessagePayload;
+    /*
+     * Optional: WhatsApp only delivers Meta-approved template payloads, so a
+     * caller whose event type has no registered template leaves this out and
+     * the channel is skipped (the body of this method already guards for it)
+     * instead of failing at the notification service.
+     */
+    whatsAppMessage?: WhatsAppMessagePayload | undefined;
     telegramMessage?: TelegramMessagePayload | undefined;
     incidentId?: ObjectID | undefined;
     alertId?: ObjectID | undefined;
@@ -846,6 +852,38 @@ export class Service extends DatabaseService<UserNotificationSetting> {
       projectId,
       NotificationSettingEventType.SEND_WHEN_USER_IS_NO_LONGER_ACTIVE_ON_ON_CALL_ROSTER,
     );
+
+    await this.addShiftReminderNotificationSettings(userId, projectId);
+  }
+
+  /*
+   * The two shift-reminder events ("before my shift starts", "my upcoming
+   * shift is reassigned"). Email AND push on by default: a reminder that
+   * only lands in a mailbox is easy to miss at 05:45, and both are the
+   * user's own lead times, not a page. Idempotent — the
+   * AddShiftReminderNotificationSettingsForUsers data migration calls this
+   * for every existing member, and sendUserNotification sends nothing
+   * without a row, so this is what makes the reminder worker's output
+   * non-zero for users who joined before the events existed.
+   */
+  @CaptureSpan()
+  public async addShiftReminderNotificationSettings(
+    userId: ObjectID,
+    projectId: ObjectID,
+  ): Promise<void> {
+    await this.addNotificationSettingIfNotExists(
+      userId,
+      projectId,
+      NotificationSettingEventType.SEND_BEFORE_USER_ON_CALL_SHIFT_STARTS,
+      { alertByPush: true },
+    );
+
+    await this.addNotificationSettingIfNotExists(
+      userId,
+      projectId,
+      NotificationSettingEventType.SEND_WHEN_USER_ON_CALL_SHIFT_IS_REASSIGNED,
+      { alertByPush: true },
+    );
   }
 
   private async addAlertNotificationSettings(
@@ -940,6 +978,7 @@ export class Service extends DatabaseService<UserNotificationSetting> {
     userId: ObjectID,
     projectId: ObjectID,
     eventType: NotificationSettingEventType,
+    options?: { alertByPush?: boolean | undefined },
   ): Promise<void> {
     const existingNotification: PositiveNumber = await this.countBy({
       query: {
@@ -959,6 +998,10 @@ export class Service extends DatabaseService<UserNotificationSetting> {
       item.eventType = eventType;
       item.alertByEmail = true;
 
+      if (options?.alertByPush) {
+        item.alertByPush = true;
+      }
+
       await this.create({
         data: item,
         props: {

package/Server/Services/UserOnCallCalendarFeedService.ts

@@ -0,0 +1,159 @@
+import CreateBy from "../Types/Database/CreateBy";
+import UpdateBy from "../Types/Database/UpdateBy";
+import { OnCreate, OnUpdate } from "../Types/Database/Hooks";
+import DatabaseService from "./DatabaseService";
+import CalendarFeedToken, {
+  CalendarFeedRotation,
+  MintedCalendarFeedToken,
+} from "../Utils/OnCall/CalendarFeedToken";
+import CaptureSpan from "../Utils/Telemetry/CaptureSpan";
+import Model from "../../Models/DatabaseModels/UserOnCallCalendarFeed";
+import BadDataException from "../../Types/Exception/BadDataException";
+import NotAuthorizedException from "../../Types/Exception/NotAuthorizedException";
+import NotFoundException from "../../Types/Exception/NotFoundException";
+import CalendarFeedWindow from "../../Types/OnCallDutyPolicy/CalendarFeedWindow";
+import ObjectID from "../../Types/ObjectID";
+import OneUptimeDate from "../../Types/Date";
+
+/*
+ * The personal calendar feed row: one per (project, user).
+ *
+ * WHAT THE HOOKS GUARANTEE
+ *
+ *   - Only root creates one. The table create list is already empty, but the
+ *     hook says it out loud: the plaintext token is minted here or by the
+ *     calendar API, never by a request body.
+ *   - A created row always carries a consistent token set (see
+ *     CalendarFeedToken.applyTokenColumnsOnCreate): there is no path to a row
+ *     whose hash does not match its token.
+ *   - pastDays / futureDays are clamped to the CalendarFeedWindow bounds on
+ *     every write, so a feed can never be asked to render a year.
+ *
+ * Rotation lives here too (rotateTokenById) so the grace-hash bookkeeping is
+ * written once and the API, the member-leave cleanup and the tests share it.
+ * Cache purging is the caller's job - this service knows nothing about the
+ * feed cache.
+ */
+export class Service extends DatabaseService<Model> {
+  public constructor() {
+    super(Model);
+  }
+
+  @CaptureSpan()
+  protected override async onBeforeCreate(
+    createBy: CreateBy<Model>,
+  ): Promise<OnCreate<Model>> {
+    if (!createBy.props.isRoot) {
+      throw new NotAuthorizedException(
+        "Calendar feeds are generated by the calendar API, not created directly.",
+      );
+    }
+
+    const projectId: ObjectID | undefined =
+      createBy.data.projectId || createBy.props.tenantId;
+
+    if (!projectId) {
+      throw new BadDataException("projectId is required");
+    }
+
+    if (!createBy.data.userId) {
+      throw new BadDataException("userId is required");
+    }
+
+    createBy.data.projectId = projectId;
+
+    // Root only reaches this line, so a pre-minted token may be trusted.
+    CalendarFeedToken.applyTokenColumnsOnCreate(createBy.data, {
+      trustSuppliedToken: true,
+    });
+
+    createBy.data.pastDays = CalendarFeedWindow.clampPastDays(
+      createBy.data.pastDays,
+    );
+    createBy.data.futureDays = CalendarFeedWindow.clampFutureDays(
+      createBy.data.futureDays,
+    );
+
+    return { createBy, carryForward: null };
+  }
+
+  @CaptureSpan()
+  protected override async onBeforeUpdate(
+    updateBy: UpdateBy<Model>,
+  ): Promise<OnUpdate<Model>> {
+    if (updateBy.data.pastDays !== undefined) {
+      updateBy.data.pastDays = CalendarFeedWindow.clampPastDays(
+        updateBy.data.pastDays,
+      );
+    }
+
+    if (updateBy.data.futureDays !== undefined) {
+      updateBy.data.futureDays = CalendarFeedWindow.clampFutureDays(
+        updateBy.data.futureDays,
+      );
+    }
+
+    return { updateBy, carryForward: null };
+  }
+
+  /**
+   * Create the feed row for a user in a project and hand the plaintext token
+   * back ONCE, so the caller can build the subscription URL without a
+   * decrypt round-trip. Root only; the caller is responsible for holding the
+   * per-(project, user) lock the API uses around mint-or-rotate.
+   */
+  @CaptureSpan()
+  public async createForUser(data: {
+    projectId: ObjectID;
+    userId: ObjectID;
+  }): Promise<{ feed: Model; minted: MintedCalendarFeedToken }> {
+    const minted: MintedCalendarFeedToken = CalendarFeedToken.mintSet();
+
+    const feed: Model = new Model();
+    feed.projectId = data.projectId;
+    feed.userId = data.userId;
+    feed.token = minted.token;
+    feed.rotatedAt = OneUptimeDate.getCurrentDate();
+
+    const created: Model = await this.create({
+      data: feed,
+      props: { isRoot: true },
+    });
+
+    return { feed: created, minted };
+  }
+
+  /**
+   * Rotate the token of an existing feed: mint a new one, move the current
+   * hash into the 30-day grace slot, stamp rotatedAt. Returns the new
+   * plaintext token ONCE. Root write; the caller purges caches.
+   */
+  @CaptureSpan()
+  public async rotateTokenById(data: {
+    id: ObjectID;
+  }): Promise<CalendarFeedRotation> {
+    const existing: Model | null = await this.findOneBy({
+      query: { _id: data.id },
+      select: { _id: true, tokenHash: true },
+      props: { isRoot: true, ignoreHooks: true },
+    });
+
+    if (!existing) {
+      throw new NotFoundException("Calendar feed not found.");
+    }
+
+    const rotation: CalendarFeedRotation = CalendarFeedToken.buildRotation({
+      currentTokenHash: existing.tokenHash,
+    });
+
+    await this.updateOneById({
+      id: data.id,
+      data: CalendarFeedToken.toRotationUpdateData(rotation),
+      props: { isRoot: true },
+    });
+
+    return rotation;
+  }
+}
+
+export default new Service();