@oneuptime/common 12.0.14 → 12.0.16

package/Server/Services/ScheduledMaintenanceStateTimelineService.ts

@@ -1,6 +1,7 @@
 import CreateBy from "../Types/Database/CreateBy";
 import DeleteBy from "../Types/Database/DeleteBy";
-import { OnCreate, OnDelete } from "../Types/Database/Hooks";
+import UpdateBy from "../Types/Database/UpdateBy";
+import { OnCreate, OnDelete, OnUpdate } from "../Types/Database/Hooks";
 import QueryHelper from "../Types/Database/QueryHelper";
 import DatabaseService from "./DatabaseService";
 import MonitorService from "./MonitorService";
@@ -9,6 +10,7 @@ import MonitorStatusTimelineService from "./MonitorStatusTimelineService";
 import ScheduledMaintenancePublicNoteService from "./ScheduledMaintenancePublicNoteService";
 import ScheduledMaintenanceService from "./ScheduledMaintenanceService";
 import ScheduledMaintenanceStateService from "./ScheduledMaintenanceStateService";
+import ScheduledMaintenanceMeasurementValueService from "./ScheduledMaintenanceMeasurementValueService";
 import SortOrder from "../../Types/BaseDatabase/SortOrder";
 import OneUptimeDate from "../../Types/Date";
 import BadDataException from "../../Types/Exception/BadDataException";
@@ -655,9 +657,173 @@ export class Service extends DatabaseService<ScheduledMaintenanceStateTimeline>
       });
     }
 
+    /*
+     * A new timeline row moves every anchor that resolves against it, so the
+     * derived measurements have to be recomputed. Fire-and-forget: the state
+     * change itself is the user-visible write and must not fail because a
+     * derived number could not be recalculated.
+     */
+    ScheduledMaintenanceMeasurementValueService.recomputeForScheduledMaintenance(
+      {
+        scheduledMaintenanceId: createdItem.scheduledMaintenanceId,
+      },
+    ).catch((error: Error) => {
+      logger.error(
+        `Error while recomputing scheduled maintenance measurements:`,
+        {
+          projectId: createdItem.projectId?.toString(),
+          scheduledMaintenanceId:
+            createdItem.scheduledMaintenanceId?.toString(),
+        } as LogAttributes,
+      );
+      logger.error(error, {
+        projectId: createdItem.projectId?.toString(),
+        scheduledMaintenanceId: createdItem.scheduledMaintenanceId?.toString(),
+      } as LogAttributes);
+    });
+
     return createdItem;
   }
 
+  @CaptureSpan()
+  protected override async onBeforeUpdate(
+    updateBy: UpdateBy<ScheduledMaintenanceStateTimeline>,
+  ): Promise<OnUpdate<ScheduledMaintenanceStateTimeline>> {
+    /*
+     * Resolved before the update runs, because the update may narrow or move
+     * the rows the query matches -- and because a row can be repointed at a
+     * different event, in which case the event it LEFT also has to be
+     * recomputed. onUpdateSuccess unions this with the after-state.
+     */
+    const scheduledMaintenanceIds: Array<ObjectID> =
+      await this.getScheduledMaintenanceIdsForTimelineQuery(updateBy);
+
+    return {
+      updateBy,
+      carryForward: scheduledMaintenanceIds,
+    };
+  }
+
+  @CaptureSpan()
+  protected override async onUpdateSuccess(
+    onUpdate: OnUpdate<ScheduledMaintenanceStateTimeline>,
+    updatedItemIds: ObjectID[],
+  ): Promise<OnUpdate<ScheduledMaintenanceStateTimeline>> {
+    const scheduledMaintenanceIds: Array<ObjectID> = [
+      ...(Array.isArray(onUpdate.carryForward)
+        ? (onUpdate.carryForward as Array<ObjectID>)
+        : []),
+    ];
+
+    if (updatedItemIds.length > 0) {
+      const updatedTimelines: Array<ScheduledMaintenanceStateTimeline> =
+        await this.findBy({
+          query: {
+            _id: QueryHelper.any(updatedItemIds),
+          },
+          select: {
+            scheduledMaintenanceId: true,
+          },
+          skip: 0,
+          limit: LIMIT_PER_PROJECT,
+          props: {
+            isRoot: true,
+            ignoreHooks: true,
+          },
+        });
+
+      scheduledMaintenanceIds.push(
+        ...this.getScheduledMaintenanceIdsFromTimelines(updatedTimelines),
+      );
+    }
+
+    this.recomputeMeasurementsForScheduledMaintenanceIds(
+      scheduledMaintenanceIds,
+    );
+
+    return onUpdate;
+  }
+
+  private async getScheduledMaintenanceIdsForTimelineQuery(
+    updateBy: UpdateBy<ScheduledMaintenanceStateTimeline>,
+  ): Promise<Array<ObjectID>> {
+    const timelines: Array<ScheduledMaintenanceStateTimeline> =
+      await this.findBy({
+        query: updateBy.query,
+        select: {
+          scheduledMaintenanceId: true,
+        },
+        skip: updateBy.skip,
+        limit: updateBy.limit,
+        props: {
+          isRoot: true,
+          ignoreHooks: true,
+        },
+      });
+
+    return this.getScheduledMaintenanceIdsFromTimelines(timelines);
+  }
+
+  private getScheduledMaintenanceIdsFromTimelines(
+    timelines: Array<ScheduledMaintenanceStateTimeline>,
+  ): Array<ObjectID> {
+    const scheduledMaintenanceIds: Map<string, ObjectID> = new Map<
+      string,
+      ObjectID
+    >();
+
+    for (const timeline of timelines) {
+      if (timeline.scheduledMaintenanceId) {
+        scheduledMaintenanceIds.set(
+          timeline.scheduledMaintenanceId.toString(),
+          timeline.scheduledMaintenanceId,
+        );
+      }
+    }
+
+    return Array.from(scheduledMaintenanceIds.values());
+  }
+
+  /*
+   * `startsAt` on a timeline row is editable so an operator can correct a
+   * wrong timestamp. Every measurement anchored on that row is derived from
+   * it, so the stored values must move with the correction -- otherwise the
+   * page shows a number computed from a timestamp that no longer exists.
+   */
+  private recomputeMeasurementsForScheduledMaintenanceIds(
+    scheduledMaintenanceIds: Array<ObjectID>,
+  ): void {
+    const uniqueScheduledMaintenanceIds: Map<string, ObjectID> = new Map<
+      string,
+      ObjectID
+    >();
+
+    for (const scheduledMaintenanceId of scheduledMaintenanceIds) {
+      uniqueScheduledMaintenanceIds.set(
+        scheduledMaintenanceId.toString(),
+        scheduledMaintenanceId,
+      );
+    }
+
+    for (const scheduledMaintenanceId of uniqueScheduledMaintenanceIds.values()) {
+      ScheduledMaintenanceMeasurementValueService.recomputeForScheduledMaintenance(
+        {
+          scheduledMaintenanceId: scheduledMaintenanceId,
+        },
+      ).catch((error: Error) => {
+        logger.error(
+          `Error while recomputing scheduled maintenance measurements:`,
+          {
+            scheduledMaintenanceId: scheduledMaintenanceId.toString(),
+          } as LogAttributes,
+        );
+        logger.error(error, {
+          scheduledMaintenanceId: scheduledMaintenanceId.toString(),
+        } as LogAttributes);
+      });
+    }
+  }
+
   @CaptureSpan()
   public async enableActiveMonitoringForMonitors(
     scheduledMaintenanceEvent: ScheduledMaintenance,
@@ -979,6 +1145,27 @@ export class Service extends DatabaseService<ScheduledMaintenanceStateTimeline>
           },
         });
       }
+
+      /*
+       * A deleted timeline row removes the point some anchors resolved to.
+       * The recompute is a total function of what is left, so it converges
+       * without any per-case repair -- but it has to be triggered.
+       */
+      ScheduledMaintenanceMeasurementValueService.recomputeForScheduledMaintenance(
+        {
+          scheduledMaintenanceId: scheduledMaintenanceId,
+        },
+      ).catch((error: Error) => {
+        logger.error(
+          `Error while recomputing scheduled maintenance measurements:`,
+          {
+            scheduledMaintenanceId: scheduledMaintenanceId?.toString(),
+          } as LogAttributes,
+        );
+        logger.error(error, {
+          scheduledMaintenanceId: scheduledMaintenanceId?.toString(),
+        } as LogAttributes);
+      });
     }
 
     return onDelete;

package/Server/Services/SecurityEventService.ts

@@ -0,0 +1,104 @@
+import ClickhouseDatabase from "../Infrastructure/ClickhouseDatabase";
+import AnalyticsDatabaseService, {
+  DbJSONResponse,
+  Results,
+} from "./AnalyticsDatabaseService";
+import SecurityEvent from "../../Models/AnalyticsModels/SecurityEvent";
+import TableColumnType from "../../Types/AnalyticsDatabase/TableColumnType";
+import ObjectID from "../../Types/ObjectID";
+import { JSONObject } from "../../Types/JSON";
+import { SQL, Statement } from "../Utils/AnalyticsDatabase/Statement";
+import { getQuerySettings } from "../Utils/AnalyticsDatabase/QuerySettingsHelper";
+
+export interface DetectionMatchGroup {
+  groupValue: string;
+  matchCount: number;
+  sampleMessage: string;
+  sampleObservables: Array<string>;
+}
+
+export class SecurityEventService extends AnalyticsDatabaseService<SecurityEvent> {
+  public constructor(clickhouseDatabase?: ClickhouseDatabase | undefined) {
+    super({ modelType: SecurityEvent, database: clickhouseDatabase });
+  }
+
+  /*
+   * Detection-engine query: count matching events in a window, grouped by
+   * an optional field expression. `whereFragment` and
+   * `groupByExpression` are Statements built by SigmaClickhouseCompiler —
+   * every rule-controlled value inside them is a bound parameter.
+   */
+  public async findDetectionMatches(data: {
+    projectId: ObjectID;
+    startTime: Date;
+    endTime: Date;
+    whereFragment: Statement;
+    groupByExpression: Statement | null;
+    maxGroups: number;
+  }): Promise<Array<DetectionMatchGroup>> {
+    if (!this.database) {
+      this.useDefaultDatabase();
+    }
+
+    const databaseName: string =
+      this.database!.getDatasourceOptions().database!;
+
+    const statement: Statement = SQL`SELECT `;
+
+    if (data.groupByExpression) {
+      statement.append(data.groupByExpression);
+    } else {
+      statement.append("''");
+    }
+
+    statement.append(
+      ` AS groupValue, count() AS matchCount, any(message) AS sampleMessage, arrayDistinct(arrayFlatten(groupArray(20)(observables))) AS sampleObservables FROM ${databaseName}.${this.model.tableName} WHERE `,
+    );
+
+    statement.append(
+      SQL`projectId = ${{
+        type: TableColumnType.ObjectID,
+        value: data.projectId,
+      }} AND time >= ${{
+        type: TableColumnType.DateTime64,
+        value: data.startTime,
+      }} AND time < ${{
+        type: TableColumnType.DateTime64,
+        value: data.endTime,
+      }} AND (`,
+    );
+
+    statement.append(data.whereFragment);
+    statement.append(") GROUP BY groupValue ORDER BY matchCount DESC LIMIT ");
+    statement.append(
+      SQL`${{
+        type: TableColumnType.Number,
+        value: data.maxGroups,
+      }}`,
+    );
+    statement.append(getQuerySettings({ maxExecutionTimeInSeconds: 60 }));
+
+    const result: Results = await this.executeQuery(statement);
+
+    const response: DbJSONResponse = await result.json<{
+      data?: Array<JSONObject>;
+    }>();
+
+    return (response.data || []).map((row: JSONObject): DetectionMatchGroup => {
+      return {
+        groupValue: String(row["groupValue"] ?? ""),
+        matchCount: Number(row["matchCount"] || 0),
+        sampleMessage: String(row["sampleMessage"] ?? ""),
+        sampleObservables: Array.isArray(row["sampleObservables"])
+          ? (row["sampleObservables"] as Array<unknown>).map(
+              (value: unknown): string => {
+                return String(value);
+              },
+            )
+          : [],
+      };
+    });
+  }
+}
+
+export default new SecurityEventService();

package/Server/Services/SpanCountBaselineService.ts

@@ -0,0 +1,173 @@
+import AnalyticsDatabaseService from "./AnalyticsDatabaseService";
+import ClickhouseDatabase from "../Infrastructure/ClickhouseDatabase";
+import SpanCountBaseline from "../../Models/AnalyticsModels/SpanCountBaseline";
+import { MetricBaselineService } from "./MetricBaselineService";
+import CountAnomaly, {
+  CountBaselineStats,
+  CountBaselineSummary,
+} from "../Utils/Monitor/Criteria/CountAnomaly";
+import logger, { LogAttributes } from "../Utils/Logger";
+import ObjectID from "../../Types/ObjectID";
+
+/**
+ * Read-side service for the `SpanCountBaseline` MV target table — the
+ * span-volume peer of `MetricBaselineService`.
+ *
+ * The table is declared by the {@link SpanCountBaseline} model and
+ * created by the boot-time analytics schema-sync (table + MV, both
+ * idempotent). Rows are populated by the attached MV; this service only
+ * reads.
+ *
+ * `getBaseline(...)` fetches the per-minute span counts of one
+ * hour-of-week cell over the rolling window and computes the stats in
+ * app code via {@link CountAnomaly.computeStats} — the input is at most
+ * `60 × ceil(windowDays / 7)` rows, and keeping the math in TS makes it
+ * unit-testable. Window/minSamples defaults and caps are shared with
+ * MetricBaselineService so every anomaly criterion behaves alike.
+ */
+export class SpanCountBaselineService extends AnalyticsDatabaseService<SpanCountBaseline> {
+  public constructor(clickhouseDatabase?: ClickhouseDatabase | undefined) {
+    super({ modelType: SpanCountBaseline, database: clickhouseDatabase });
+  }
+
+  /**
+   * Fetch the rolling-window baseline of spans-per-minute for one
+   * (project, [services], [statuses], hour-of-week) cell. Returns null
+   * when no data exists.
+   *
+   * `telemetryServiceIds` empty/omitted aggregates across every service
+   * in the project; `spanStatusCodes` empty/omitted aggregates across
+   * every span status — matching how the Traces monitor query itself
+   * broadens when those filters are unset.
+   */
+  public async getBaseline(input: {
+    projectId: ObjectID | string;
+    telemetryServiceIds?: Array<ObjectID | string> | undefined;
+    spanStatusCodes?: Array<number> | undefined;
+    hourOfWeek: number;
+    windowDays?: number | undefined;
+    minSamples?: number | undefined;
+  }): Promise<CountBaselineSummary | null> {
+    const windowDays: number = Math.min(
+      input.windowDays || MetricBaselineService.DEFAULT_WINDOW_DAYS,
+      MetricBaselineService.MAX_WINDOW_DAYS,
+    );
+    const minSamples: number =
+      input.minSamples || MetricBaselineService.DEFAULT_MIN_SAMPLES;
+
+    const projectIdStr: string = this.escapeString(
+      input.projectId instanceof ObjectID
+        ? input.projectId.toString()
+        : input.projectId,
+    );
+    const hour: number = Math.max(
+      0,
+      Math.min(167, Math.floor(input.hourOfWeek)),
+    );
+
+    const serviceIds: Array<string> = (input.telemetryServiceIds || []).map(
+      (id: ObjectID | string) => {
+        return this.escapeString(id instanceof ObjectID ? id.toString() : id);
+      },
+    );
+    const serviceIdClause: string =
+      serviceIds.length > 0
+        ? `AND primaryEntityId IN (${serviceIds
+            .map((id: string) => {
+              return `'${id}'`;
+            })
+            .join(", ")})`
+        : "";
+
+    /*
+     * Persisted monitor steps round-trip through JSON, so status codes
+     * can arrive as numeric strings — coerce before filtering rather
+     * than silently dropping the criterion's scope.
+     */
+    const statusCodes: Array<number> = (input.spanStatusCodes || [])
+      .map((code: number) => {
+        return Number(code);
+      })
+      .filter((code: number) => {
+        return Number.isFinite(code);
+      });
+    const statusCodeClause: string =
+      statusCodes.length > 0
+        ? `AND statusCode IN (${statusCodes
+            .map((code: number) => {
+              return String(Math.floor(code));
+            })
+            .join(", ")})`
+        : "";
+
+    /*
+     * One row per non-empty minute cell; each row's merged count is a
+     * raw baseline sample ("spans in that minute"). GROUP BY day +
+     * minuteOfHour also folds across services/statuses when the IN
+     * clauses are broad, so a multi-service monitor baselines its
+     * combined volume — matching what its observed spanCount measures.
+     */
+    const sql: string = `
+      SELECT
+        countMerge(spanCountState) AS c
+      FROM SpanCountBaseline
+      WHERE projectId = '${projectIdStr}'
+        ${serviceIdClause}
+        ${statusCodeClause}
+        AND hourOfWeek = ${hour}
+        AND day >= today() - INTERVAL ${windowDays} DAY
+      GROUP BY day, minuteOfHour
+    `;
+
+    const resultSet: {
+      json: () => Promise<{ data: Array<{ c: number | string }> }>;
+    } = (await this.executeQuery(sql)) as unknown as {
+      json: () => Promise<{ data: Array<{ c: number | string }> }>;
+    };
+
+    const parsed: { data: Array<{ c: number | string }> } =
+      await resultSet.json();
+
+    const counts: Array<number> = parsed.data.map(
+      (row: { c: number | string }) => {
+        return this.toNumber(row.c);
+      },
+    );
+
+    const stats: CountBaselineStats | null = CountAnomaly.computeStats(counts);
+    if (!stats) {
+      return null;
+    }
+
+    const summary: CountBaselineSummary = {
+      ...stats,
+      isReliable: stats.sampleCount >= minSamples,
+      windowDays,
+      hourOfWeek: hour,
+    };
+
+    logger.debug("SpanCountBaselineService.getBaseline", {
+      projectId: projectIdStr,
+      hourOfWeek: hour,
+      sampleCount: summary.sampleCount,
+      isReliable: summary.isReliable,
+    } as LogAttributes);
+
+    return summary;
+  }
+
+  private toNumber(v: number | string | undefined): number {
+    if (typeof v === "number") {
+      return v;
+    }
+    const n: number = Number(v);
+    return Number.isFinite(n) ? n : 0;
+  }
+
+  private escapeString(v: string): string {
+    // ClickHouse string-literal escape for backslash and single quote.
+    return v.replace(/\\/g, "\\\\").replace(/'/g, "\\'");
+  }
+}
+
+export default new SpanCountBaselineService();

package/Server/Services/StatusPageService.ts

@@ -764,9 +764,10 @@ export class Service extends DatabaseService<StatusPage> {
        * excluded any event that spanned the ENTIRE window but closed after it, so a total
        * outage could render as a silent 100% uptime.
        *
-       * `greaterThanEqualToOrNull` emits `(endsAt >= :date or endsAt IS NULL)`. Note that
-       * `greaterThanOrNull` emits exactly the same `>=` SQL despite its name, so the explicit
-       * name is used here to keep the intent readable.
+       * `greaterThanEqualToOrNull` emits `(endsAt >= :date or endsAt IS NULL)` - the inclusive
+       * `>=` is deliberate so a row that ends exactly at `startDate` still counts as overlapping.
+       * `greaterThanOrNull` would emit a strict `>` and drop that boundary row, so it is NOT
+       * interchangeable here.
        *
        * Supported by the (monitorId, startsAt) and (endsAt) indexes.
        */

package/Server/Services/TelemetryUsageBillingService.ts

@@ -16,6 +16,7 @@ import ProjectService from "./ProjectService";
 import Project from "../../Models/DatabaseModels/Project";
 import SpanService from "./SpanService";
 import LogService from "./LogService";
+import SecurityEventService from "./SecurityEventService";
 import MetricService from "./MetricService";
 import ExceptionInstanceService from "./ExceptionInstanceService";
 import ProfileService from "./ProfileService";
@@ -47,6 +48,7 @@ import {
   AverageProfileRowSizeInBytes,
   AverageSessionReplaySessionSizeInBytes,
   AverageProfileSampleRowSizeInBytes,
+  AverageSecurityEventRowSizeInBytes,
   IsBillingEnabled,
 } from "../EnvironmentConfig";
 import CaptureSpan from "../Utils/Telemetry/CaptureSpan";
@@ -235,6 +237,16 @@ export class Service extends DatabaseService<Model> {
           }),
           averageRowSizeInBytes,
         );
+      } else if (data.productType === ProductType.SecurityEvents) {
+        addUsage(
+          await SecurityEventService.groupTelemetryUsageByService({
+            projectId: data.projectId,
+            timestampColumnName: "time",
+            startDate: startOfDay,
+            endDate: endOfDay,
+          }),
+          averageRowSizeInBytes,
+        );
       } else if (data.productType === ProductType.SessionReplay) {
         /*
          * One table only, and a bespoke aggregation - see
@@ -551,7 +563,8 @@ export class Service extends DatabaseService<Model> {
       data.productType !== ProductType.Metrics &&
       data.productType !== ProductType.Logs &&
       data.productType !== ProductType.Profiles &&
-      data.productType !== ProductType.SessionReplay
+      data.productType !== ProductType.SessionReplay &&
+      data.productType !== ProductType.SecurityEvents
     ) {
       throw new BadDataException(
         "This product type is not a telemetry product type.",
@@ -658,7 +671,8 @@ export class Service extends DatabaseService<Model> {
       productType !== ProductType.Logs &&
       productType !== ProductType.Metrics &&
       productType !== ProductType.Profiles &&
-      productType !== ProductType.SessionReplay
+      productType !== ProductType.SessionReplay &&
+      productType !== ProductType.SecurityEvents
     ) {
       return fallbackSize;
     }
@@ -675,6 +689,7 @@ export class Service extends DatabaseService<Model> {
          * Present so the narrowed lookup above stays exhaustive.
          */
         [ProductType.SessionReplay]: AverageSessionReplaySessionSizeInBytes,
+        [ProductType.SecurityEvents]: AverageSecurityEventRowSizeInBytes,
       }[productType] ?? fallbackSize;
 
     if (!Number.isFinite(value) || value <= 0) {

package/Server/Services/UserEmailService.ts

@@ -117,10 +117,10 @@ export class Service extends DatabaseService<Model> {
        * create response for it to confirm the address the way a user would.
        *
        * It is gated on an environment variable that is unset in every shipped
-       * config and set true ONLY in the CI e2e job (docker-compose passes it
-       * through, .github/workflows/test-release.yaml sets it for the SaaS run).
-       * In production the flag is absent, this branch never runs, and the code
-       * never leaves the mail path.
+       * config and set true ONLY in the CI e2e jobs (docker-compose passes it
+       * through; both SaaS e2e jobs set it — test-release.yaml for master and
+       * release.yml for the release branch). In production the flag is absent,
+       * this branch never runs, and the code never leaves the mail path.
        */
       if (
         process.env["EXPOSE_VERIFICATION_CODE_IN_API_RESPONSE_FOR_E2E"] ===