@oneuptime/common 10.0.54 → 10.0.56

package/Server/Services/ExceptionAggregationService.ts

@@ -0,0 +1,335 @@
+import { SQL, Statement } from "../Utils/AnalyticsDatabase/Statement";
+import ExceptionInstanceService from "./ExceptionInstanceService";
+import TableColumnType from "../../Types/AnalyticsDatabase/TableColumnType";
+import { JSONObject } from "../../Types/JSON";
+import ObjectID from "../../Types/ObjectID";
+import BadDataException from "../../Types/Exception/BadDataException";
+import Includes from "../../Types/BaseDatabase/Includes";
+import AnalyticsTableName from "../../Types/AnalyticsDatabase/AnalyticsTableName";
+import CaptureSpan from "../Utils/Telemetry/CaptureSpan";
+import { DbJSONResponse, Results } from "./AnalyticsDatabaseService";
+
+export interface HistogramBucket {
+  time: string;
+  series: string;
+  count: number;
+}
+
+export interface ExceptionFilters {
+  serviceIds?: Array<ObjectID> | undefined;
+  exceptionTypes?: Array<string> | undefined;
+  environments?: Array<string> | undefined;
+  fingerprints?: Array<string> | undefined;
+  traceIds?: Array<string> | undefined;
+  escaped?: boolean | undefined;
+  messageSearchText?: string | undefined;
+}
+
+export interface HistogramRequest extends ExceptionFilters {
+  projectId: ObjectID;
+  startTime: Date;
+  endTime: Date;
+  bucketSizeInMinutes: number;
+}
+
+export interface FacetValue {
+  value: string;
+  count: number;
+}
+
+export interface FacetRequest extends ExceptionFilters {
+  projectId: ObjectID;
+  startTime: Date;
+  endTime: Date;
+  facetKey: string;
+  limit?: number | undefined;
+}
+
+export class ExceptionAggregationService {
+  private static readonly DEFAULT_FACET_LIMIT: number = 500;
+  private static readonly TABLE_NAME: string =
+    AnalyticsTableName.ExceptionInstance;
+  private static readonly TOP_LEVEL_COLUMNS: Set<string> = new Set([
+    "serviceId",
+    "exceptionType",
+    "environment",
+    "fingerprint",
+    "traceId",
+    "spanId",
+    "escaped",
+    "release",
+  ]);
+  private static readonly ATTRIBUTE_KEY_PATTERN: RegExp = /^[a-zA-Z0-9._:/-]+$/;
+  private static readonly MAX_FACET_KEY_LENGTH: number = 256;
+
+  @CaptureSpan()
+  public static async getHistogram(
+    request: HistogramRequest,
+  ): Promise<Array<HistogramBucket>> {
+    const statement: Statement =
+      ExceptionAggregationService.buildHistogramStatement(request);
+
+    const dbResult: Results =
+      await ExceptionInstanceService.executeQuery(statement);
+    const response: DbJSONResponse = await dbResult.json<{
+      data?: Array<JSONObject>;
+    }>();
+
+    const rows: Array<JSONObject> = response.data || [];
+
+    return rows.map((row: JSONObject): HistogramBucket => {
+      return {
+        time: String(row["bucket"] || ""),
+        series: ExceptionAggregationService.mapEscapedToSeries(row["escaped"]),
+        count: Number(row["cnt"] || 0),
+      };
+    });
+  }
+
+  @CaptureSpan()
+  public static async getFacetValues(
+    request: FacetRequest,
+  ): Promise<Array<FacetValue>> {
+    const statement: Statement =
+      ExceptionAggregationService.buildFacetStatement(request);
+
+    const dbResult: Results =
+      await ExceptionInstanceService.executeQuery(statement);
+    const response: DbJSONResponse = await dbResult.json<{
+      data?: Array<JSONObject>;
+    }>();
+
+    const rows: Array<JSONObject> = response.data || [];
+
+    return rows
+      .map((row: JSONObject): FacetValue => {
+        return {
+          value: String(row["val"] || ""),
+          count: Number(row["cnt"] || 0),
+        };
+      })
+      .filter((facet: FacetValue): boolean => {
+        return facet.value.length > 0;
+      });
+  }
+
+  private static mapEscapedToSeries(raw: unknown): string {
+    // ClickHouse returns booleans as 0/1
+    if (raw === true || raw === 1 || raw === "1" || raw === "true") {
+      return "unhandled";
+    }
+    return "handled";
+  }
+
+  private static buildHistogramStatement(request: HistogramRequest): Statement {
+    const intervalSeconds: number = request.bucketSizeInMinutes * 60;
+
+    const statement: Statement = SQL`
+      SELECT
+        toStartOfInterval(time, INTERVAL ${{
+          type: TableColumnType.Number,
+          value: intervalSeconds,
+        }} SECOND) AS bucket,
+        escaped,
+        count() AS cnt
+      FROM ${ExceptionAggregationService.TABLE_NAME}
+      WHERE projectId = ${{
+        type: TableColumnType.ObjectID,
+        value: request.projectId,
+      }}
+        AND time >= ${{
+          type: TableColumnType.Date,
+          value: request.startTime,
+        }}
+        AND time <= ${{
+          type: TableColumnType.Date,
+          value: request.endTime,
+        }}
+    `;
+
+    ExceptionAggregationService.appendCommonFilters(statement, request);
+
+    statement.append(" GROUP BY bucket, escaped ORDER BY bucket ASC");
+
+    /*
+     * Defense in depth: cap histogram runtime below nginx's 60s
+     * proxy_read_timeout. 'break' returns partial aggregated results
+     * rather than throwing, which is acceptable for a density viz.
+     */
+    statement.append(
+      " SETTINGS max_execution_time = 45, timeout_overflow_mode = 'break'",
+    );
+
+    return statement;
+  }
+
+  private static buildFacetStatement(request: FacetRequest): Statement {
+    const limit: number =
+      request.limit ?? ExceptionAggregationService.DEFAULT_FACET_LIMIT;
+
+    ExceptionAggregationService.validateFacetKey(request.facetKey);
+
+    const isTopLevelColumn: boolean =
+      ExceptionAggregationService.isTopLevelColumn(request.facetKey);
+
+    const statement: Statement = new Statement();
+
+    if (isTopLevelColumn) {
+      statement.append(
+        SQL`SELECT toString(${request.facetKey}) AS val, count() AS cnt FROM ${ExceptionAggregationService.TABLE_NAME}`,
+      );
+    } else {
+      statement.append(
+        SQL`SELECT JSONExtractRaw(attributes, ${{
+          type: TableColumnType.Text,
+          value: request.facetKey,
+        }}) AS val, count() AS cnt FROM ${ExceptionAggregationService.TABLE_NAME}`,
+      );
+    }
+
+    statement.append(
+      SQL` WHERE projectId = ${{
+        type: TableColumnType.ObjectID,
+        value: request.projectId,
+      }} AND time >= ${{
+        type: TableColumnType.Date,
+        value: request.startTime,
+      }} AND time <= ${{
+        type: TableColumnType.Date,
+        value: request.endTime,
+      }}`,
+    );
+
+    if (!isTopLevelColumn) {
+      statement.append(
+        SQL` AND JSONHas(attributes, ${{
+          type: TableColumnType.Text,
+          value: request.facetKey,
+        }}) = 1`,
+      );
+    }
+
+    ExceptionAggregationService.appendCommonFilters(statement, request);
+
+    statement.append(
+      SQL` GROUP BY val ORDER BY cnt DESC LIMIT ${{
+        type: TableColumnType.Number,
+        value: limit,
+      }}`,
+    );
+
+    /*
+     * Defense in depth: cap individual facet query runtime below nginx's
+     * 60s proxy_read_timeout so a slow facet never starves the endpoint.
+     */
+    statement.append(
+      " SETTINGS max_execution_time = 45, timeout_overflow_mode = 'break'",
+    );
+
+    return statement;
+  }
+
+  private static appendCommonFilters(
+    statement: Statement,
+    request: ExceptionFilters,
+  ): void {
+    if (request.serviceIds && request.serviceIds.length > 0) {
+      statement.append(
+        SQL` AND serviceId IN (${{
+          type: TableColumnType.ObjectID,
+          value: new Includes(
+            request.serviceIds.map((id: ObjectID) => {
+              return id.toString();
+            }),
+          ),
+        }})`,
+      );
+    }
+
+    if (request.exceptionTypes && request.exceptionTypes.length > 0) {
+      statement.append(
+        SQL` AND exceptionType IN (${{
+          type: TableColumnType.Text,
+          value: new Includes(request.exceptionTypes),
+        }})`,
+      );
+    }
+
+    if (request.environments && request.environments.length > 0) {
+      statement.append(
+        SQL` AND environment IN (${{
+          type: TableColumnType.Text,
+          value: new Includes(request.environments),
+        }})`,
+      );
+    }
+
+    if (request.fingerprints && request.fingerprints.length > 0) {
+      statement.append(
+        SQL` AND fingerprint IN (${{
+          type: TableColumnType.Text,
+          value: new Includes(request.fingerprints),
+        }})`,
+      );
+    }
+
+    if (request.traceIds && request.traceIds.length > 0) {
+      statement.append(
+        SQL` AND traceId IN (${{
+          type: TableColumnType.Text,
+          value: new Includes(request.traceIds),
+        }})`,
+      );
+    }
+
+    if (request.escaped !== undefined) {
+      statement.append(
+        SQL` AND escaped = ${{
+          type: TableColumnType.Boolean,
+          value: request.escaped,
+        }}`,
+      );
+    }
+
+    if (
+      request.messageSearchText &&
+      request.messageSearchText.trim().length > 0
+    ) {
+      statement.append(
+        SQL` AND message ILIKE ${{
+          type: TableColumnType.Text,
+          value: `%${request.messageSearchText.trim()}%`,
+        }}`,
+      );
+    }
+  }
+
+  private static isTopLevelColumn(key: string): boolean {
+    return ExceptionAggregationService.TOP_LEVEL_COLUMNS.has(key);
+  }
+
+  private static validateFacetKey(
+    facetKey: unknown,
+  ): asserts facetKey is string {
+    if (typeof facetKey !== "string") {
+      throw new BadDataException("Invalid facetKey");
+    }
+
+    if (
+      facetKey.length === 0 ||
+      facetKey.length > ExceptionAggregationService.MAX_FACET_KEY_LENGTH
+    ) {
+      throw new BadDataException("Invalid facetKey");
+    }
+
+    if (ExceptionAggregationService.isTopLevelColumn(facetKey)) {
+      return;
+    }
+
+    if (!ExceptionAggregationService.ATTRIBUTE_KEY_PATTERN.test(facetKey)) {
+      throw new BadDataException("Invalid facetKey");
+    }
+  }
+}
+
+export default ExceptionAggregationService;

package/Server/Services/Index.ts

@@ -34,6 +34,7 @@ import IncidentStateTimelineService from "./IncidentStateTimelineService";
 //Labels.
 import LabelService from "./LabelService";
 import KubernetesClusterService from "./KubernetesClusterService";
+import DockerHostService from "./DockerHostService";
 import LlmProviderService from "./LlmProviderService";
 import LogService from "./LogService";
 import MailService from "./MailService";
@@ -258,6 +259,7 @@ const services: Array<BaseService> = [
 
   LabelService,
   KubernetesClusterService,
+  DockerHostService,
   LlmProviderService,
 
   MailService,

package/Server/Services/LogAggregationService.ts

@@ -174,6 +174,15 @@ export class LogAggregationService {
 
     statement.append(" GROUP BY bucket, severityText ORDER BY bucket ASC");
 
+    /*
+     * Defense in depth: cap histogram runtime below nginx's 60s
+     * proxy_read_timeout. 'break' returns partial aggregated results
+     * rather than throwing, which is acceptable for a density viz.
+     */
+    statement.append(
+      " SETTINGS max_execution_time = 45, timeout_overflow_mode = 'break'",
+    );
+
     return statement;
   }
 
@@ -233,6 +242,14 @@ export class LogAggregationService {
       }}`,
     );
 
+    /*
+     * Defense in depth: cap individual facet query runtime below nginx's
+     * 60s proxy_read_timeout so a slow facet never starves the endpoint.
+     */
+    statement.append(
+      " SETTINGS max_execution_time = 45, timeout_overflow_mode = 'break'",
+    );
+
     return statement;
   }
 

package/Server/Services/MonitorProbeService.ts

@@ -94,13 +94,15 @@ export class Service extends DatabaseService<MonitorProbe> {
     const claimedIds: Array<ObjectID> = await this.executeTransaction(
       async (transactionalEntityManager: EntityManager) => {
         /*
-         * First, select and lock the monitor probes that need to be processed
+         * Select and lock the monitor probes that need to be processed,
+         * including the monitoringInterval so we can compute the real nextPingAt
+         * in a single UPDATE (avoiding a second round-trip).
          * FOR UPDATE SKIP LOCKED ensures that:
          * 1. Rows are locked for this transaction
          * 2. Rows already locked by other transactions are skipped
          */
         const selectQuery: string = `
-        SELECT mp."_id"
+        SELECT mp."_id", m."monitoringInterval"
         FROM "MonitorProbe" mp
         INNER JOIN "Monitor" m ON mp."monitorId" = m."_id"
         INNER JOIN "Project" p ON mp."projectId" = p."_id"
@@ -122,41 +124,60 @@ export class Service extends DatabaseService<MonitorProbe> {
         FOR UPDATE OF mp SKIP LOCKED
       `;
 
-        const selectedRows: Array<{ _id: string }> =
-          await transactionalEntityManager.query(selectQuery, [
-            data.probeId.toString(),
-            currentDate,
-            data.limit,
-          ]);
+        const selectedRows: Array<{
+          _id: string;
+          monitoringInterval: string | null;
+        }> = await transactionalEntityManager.query(selectQuery, [
+          data.probeId.toString(),
+          currentDate,
+          data.limit,
+        ]);
 
         if (selectedRows.length === 0) {
           return [];
         }
 
-        const ids: Array<string> = selectedRows.map((row: { _id: string }) => {
-          return row._id;
-        });
-
-        /*
-         * Update the claimed monitors to set nextPingAt to 1 minute from now
-         * This is a temporary value; the actual nextPingAt will be calculated
-         * based on the monitor's interval after the probe fetches the full details
-         */
-        const tempNextPingAt: Date = OneUptimeDate.addRemoveMinutes(
+        // Compute the real nextPingAt per monitor and batch-update in one query
+        const defaultNextPing: Date = OneUptimeDate.addRemoveMinutes(
           currentDate,
           1,
         );
 
+        const ids: Array<string> = [];
+        const nextPingDates: Array<Date> = [];
+        const caseFragments: Array<string> = [];
+
+        for (let i: number = 0; i < selectedRows.length; i++) {
+          const row: { _id: string; monitoringInterval: string | null } =
+            selectedRows[i]!;
+          ids.push(row._id);
+
+          let nextPing: Date = defaultNextPing;
+          if (row.monitoringInterval) {
+            try {
+              nextPing = CronTab.getNextExecutionTime(row.monitoringInterval);
+            } catch {
+              // fall back to default 1 minute
+            }
+          }
+
+          nextPingDates.push(nextPing);
+          caseFragments.push(
+            `WHEN '${row._id}'::uuid THEN $${i + 3}::timestamptz`,
+          );
+        }
+
         const updateQuery: string = `
         UPDATE "MonitorProbe"
-        SET "lastPingAt" = $1, "nextPingAt" = $2
-        WHERE "_id" = ANY($3::uuid[])
+        SET "lastPingAt" = $1,
+            "nextPingAt" = CASE "_id" ${caseFragments.join(" ")} END
+        WHERE "_id" = ANY($2::uuid[])
       `;
 
         await transactionalEntityManager.query(updateQuery, [
           currentDate,
-          tempNextPingAt,
           ids,
+          ...nextPingDates,
         ]);
 
         return ids.map((id: string) => {

package/Server/Services/MonitorService.ts

@@ -71,7 +71,6 @@ import ExceptionMessages from "../../Types/Exception/ExceptionMessages";
 import Project from "../../Models/DatabaseModels/Project";
 import { createWhatsAppMessageFromTemplate } from "../Utils/WhatsAppTemplateUtil";
 import { WhatsAppMessagePayload } from "../../Types/WhatsApp/WhatsAppMessage";
-import MetricService from "./MetricService";
 
 export interface MonitorDestinationInfo {
   monitorDestination: string;
@@ -284,27 +283,28 @@ export class Service extends DatabaseService<Model> {
     onDelete: OnDelete<Model>,
     _itemIdsBeforeDelete: ObjectID[],
   ): Promise<OnDelete<Model>> {
+    /*
+     * The monitor has already been deleted from the database at this point.
+     * Any failure in the post-delete side effects below (e.g. billing
+     * reporting) must NOT propagate up to the caller as a 500 — otherwise the
+     * client sees "500 Internal Server Error" even though the delete actually
+     * succeeded. Log and swallow instead.
+     *
+     * Note: we intentionally do NOT delete Metric rows for this monitor here.
+     * The Metric table has a ClickHouse TTL on retentionDate (set at ingest
+     * from GlobalConfig.monitorMetricRetentionInDays) that auto-drops rows.
+     * A synchronous ALTER TABLE … DELETE on every monitor deletion is both
+     * redundant and expensive.
+     */
     if (onDelete.deleteBy.props.tenantId && IsBillingEnabled) {
-      await ActiveMonitoringMeteredPlan.reportQuantityToBillingProvider(
-        onDelete.deleteBy.props.tenantId,
-      );
-    }
-
-    if (onDelete.carryForward && onDelete.carryForward.monitors) {
-      for (const monitor of onDelete.carryForward.monitors as Array<Model>) {
-        if (!monitor.projectId || !monitor.id) {
-          continue;
-        }
-
-        await MetricService.deleteBy({
-          query: {
-            projectId: monitor.projectId,
-            serviceId: monitor.id,
-          },
-          props: {
-            isRoot: true,
-          },
-        });
+      try {
+        await ActiveMonitoringMeteredPlan.reportQuantityToBillingProvider(
+          onDelete.deleteBy.props.tenantId,
+        );
+      } catch (error) {
+        logger.error(
+          `Error while reporting active monitor quantity to billing provider for project ${onDelete.deleteBy.props.tenantId?.toString()}: ${error}`,
+        );
       }
     }