@oneuptime/common 11.5.9 → 11.5.11

package/Server/Services/AnalyticsDatabaseService.ts

@@ -29,7 +29,10 @@ import Select from "../Types/AnalyticsDatabase/Select";
 import UpdateBy from "../Types/AnalyticsDatabase/UpdateBy";
 import { SQL, Statement } from "../Utils/AnalyticsDatabase/Statement";
 import StatementGenerator from "../Utils/AnalyticsDatabase/StatementGenerator";
-import { getQuerySettings } from "../Utils/AnalyticsDatabase/QuerySettingsHelper";
+import {
+  getQuerySettings,
+  TimeoutOverflowMode,
+} from "../Utils/AnalyticsDatabase/QuerySettingsHelper";
 import {
   getStorageTableName,
   onClusterClause,
@@ -892,8 +895,47 @@ export default class AnalyticsDatabaseService<
         aggregatedItems.push(aggregatedModel);
       }
 
+      /*
+       * Top-K statements (see MetricService) carry the pre-trim group
+       * count as a constant `__total_groups` column on every row; lift
+       * it off the first row into result metadata. The per-row copy
+       * loop above only projects timestamp/value/group-by columns, so
+       * the helper column never reaches the rows themselves.
+       */
+      let totalGroups: number | undefined = undefined;
+      const firstItem: JSONObject | undefined = items[0];
+      if (firstItem && firstItem["__total_groups"] !== undefined) {
+        const parsedTotalGroups: number = Number(firstItem["__total_groups"]);
+        if (Number.isFinite(parsedTotalGroups)) {
+          totalGroups = parsedTotalGroups;
+        }
+      }
+
+      /*
+       * Truncation detection. Row-count == LIMIT is a heuristic (an
+       * exactly-full window reads as truncated), but a false positive
+       * only over-warns; the silent-data-loss case it exists to catch
+       * (groups × buckets > limit dropping the oldest buckets) is
+       * always flagged. Top-K truncation is exact: the ranking phase
+       * counted every matching group.
+       */
+      const appliedLimit: number = Number(aggregateBy.limit);
+      let truncated: boolean =
+        Number.isFinite(appliedLimit) &&
+        appliedLimit > 0 &&
+        items.length >= appliedLimit;
+      if (
+        aggregateBy.topK &&
+        totalGroups !== undefined &&
+        totalGroups > aggregateBy.topK.count
+      ) {
+        truncated = true;
+      }
+
       return {
         data: aggregatedItems,
+        ...(totalGroups !== undefined ? { totalGroups } : {}),
+        truncated,
       };
     } catch (error) {
       await this.onFindError(error as Exception);
@@ -1194,6 +1236,24 @@ export default class AnalyticsDatabaseService<
     return statement;
   }
 
+  /**
+   * The timeout_overflow_mode an aggregate statement should carry.
+   * Callers that render charts keep the 'break' default (partial
+   * buckets are acceptable there); callers that ALERT on the result —
+   * the metric-monitor worker — pass 'throw' so a timed-out evaluation
+   * fails loudly instead of silently scoring partial data.
+   *
+   * Allow-listed, never passed through: aggregateBy is deserialized
+   * wholesale from API clients and this setting is emitted into SQL as
+   * a trusted literal (see QuerySettingsHelper), so anything other than
+   * the exact string "throw" degrades to the 'break' default.
+   */
+  protected getTimeoutOverflowMode(
+    aggregateBy: AggregateBy<TBaseModel>,
+  ): TimeoutOverflowMode {
+    return aggregateBy.timeoutOverflowMode === "throw" ? "throw" : "break";
+  }
+
   public toAggregateStatement(aggregateBy: AggregateBy<TBaseModel>): {
     statement: Statement;
     columns: Array<string>;
@@ -1306,6 +1366,13 @@ export default class AnalyticsDatabaseService<
     /*
      * Aggregation read-path settings.
      *
+     * - max_execution_time=45: cap aggregate runtime below the
+     *   ClickHouse client's 58s request_timeout, same as the count/find
+     *   statements — a wide-window aggregate over a large table would
+     *   otherwise run until the HTTP client disconnects. The overflow
+     *   mode defaults to 'break' (partial buckets are acceptable for
+     *   chart rendering) but alerting callers opt into 'throw' via
+     *   aggregateBy.timeoutOverflowMode — see getTimeoutOverflowMode.
      * - optimize_aggregation_in_order: when GROUP BY is a prefix of the
      *   sort key (we always group by a time bucket and the time column
      *   is at the tail of every analytics primary key), ClickHouse can
@@ -1322,6 +1389,8 @@ export default class AnalyticsDatabaseService<
      */
     statement.append(
       getQuerySettings({
+        maxExecutionTimeInSeconds: 45,
+        timeoutOverflowMode: this.getTimeoutOverflowMode(aggregateBy),
         additionalSettings: {
           optimize_aggregation_in_order: 1,
           optimize_move_to_prewhere: 1,

package/Server/Services/IncidentService.ts

@@ -56,6 +56,7 @@ import ServiceType from "../../Types/Telemetry/ServiceType";
 import OneUptimeDate from "../../Types/Date";
 import TelemetryUtil from "../Utils/Telemetry/Telemetry";
 import logger, { LogAttributes } from "../Utils/Logger";
+import ProductAnalytics from "../Utils/ProductAnalytics";
 import Semaphore, { SemaphoreMutex } from "../Infrastructure/Semaphore";
 import IncidentFeedService from "./IncidentFeedService";
 import IncidentSlaService from "./IncidentSlaService";
@@ -907,6 +908,18 @@ export class Service extends DatabaseService<Model> {
       throw new BadDataException("id is required");
     }
 
+    /*
+     * Activation event for marketing funnels. Only fires for human-declared
+     * incidents (captureForUser skips when there is no user).
+     */
+    ProductAnalytics.captureForUser({
+      userId: onCreate.createBy.props.userId || createdItem.createdByUserId,
+      event: "server/incident_created",
+      properties: {
+        project_id: createdItem.projectId.toString(),
+      },
+    });
+
     // Get incident data for feed creation
     const incident: Model | null = await this.findOneById({
       id: createdItem.id,

package/Server/Services/Index.ts

@@ -70,6 +70,9 @@ import MailService from "./MailService";
 import MetricService from "./MetricService";
 import MetricItemAggMV1mService from "./MetricItemAggMV1mService";
 import MetricItemAggMV1mByHostV2Service from "./MetricItemAggMV1mByHostV2Service";
+import MetricItemAggMV1mByServiceService from "./MetricItemAggMV1mByServiceService";
+import MetricItemAggMV1mByK8sClusterService from "./MetricItemAggMV1mByK8sClusterService";
+import MetricItemAggMV1mByContainerService from "./MetricItemAggMV1mByContainerService";
 import MutableMetricService from "./MutableMetricService";
 import MetricBaselineService from "./MetricBaselineService";
 import MonitorCustomFieldService from "./MonitorCustomFieldService";
@@ -123,6 +126,7 @@ import GlobalOidcProjectService from "./GlobalOidcProjectService";
 import PromoCodeService from "./PromoCodeService";
 import EnterpriseLicenseService from "./EnterpriseLicenseService";
 import EnterpriseLicenseInstanceService from "./EnterpriseLicenseInstanceService";
+import MarketingConversionService from "./MarketingConversionService";
 import OpenSourceDeploymentService from "./OpenSourceDeploymentService";
 import ResellerPlanService from "./ResellerPlanService";
 import ResellerService from "./ResellerService";
@@ -253,6 +257,7 @@ import WorkspaceNotificationSummaryService from "./WorkspaceNotificationSummaryS
 import OnCallDutyPolicyUserOverrideService from "./OnCallDutyPolicyUserOverrideService";
 
 import MonitorLogService from "./MonitorLogService";
+import NetworkFlowService from "./NetworkFlowService";
 
 import OnCallDutyPolicyTimeLogService from "./OnCallDutyPolicyTimeLogService";
 import ProjectSCIMLogService from "./ProjectSCIMLogService";
@@ -264,6 +269,7 @@ const services: Array<BaseService> = [
   PromoCodeService,
   EnterpriseLicenseService,
   EnterpriseLicenseInstanceService,
+  MarketingConversionService,
   OpenSourceDeploymentService,
 
   ResellerService,
@@ -532,9 +538,13 @@ export const AnalyticsServices: Array<
    */
   MetricItemAggMV1mService,
   MetricItemAggMV1mByHostV2Service,
+  MetricItemAggMV1mByServiceService,
+  MetricItemAggMV1mByK8sClusterService,
+  MetricItemAggMV1mByContainerService,
   MetricBaselineService,
   ExceptionInstanceService,
   MonitorLogService,
+  NetworkFlowService,
   ProfileService,
   ProfileSampleService,
   AuditLogService,

package/Server/Services/MarketingConversionService.ts

@@ -0,0 +1,10 @@
+import DatabaseService from "./DatabaseService";
+import MarketingConversion from "../../Models/DatabaseModels/MarketingConversion";
+
+export class Service extends DatabaseService<MarketingConversion> {
+  public constructor() {
+    super(MarketingConversion);
+  }
+}
+
+export default new Service();

package/Server/Services/MetricItemAggMV1mByContainerService.ts

@@ -0,0 +1,31 @@
+import AnalyticsDatabaseService from "./AnalyticsDatabaseService";
+import ClickhouseDatabase from "../Infrastructure/ClickhouseDatabase";
+import MetricItemAggMV1mByContainer from "../../Models/AnalyticsModels/MetricItemAggMV1mByContainer";
+
+/**
+ * Read-side service for the `MetricItemAggMV1mByContainer` MV target
+ * table — per-(container entity key, minute) pre-aggregated metric
+ * values for container-scoped chart queries. Sister of
+ * `MetricItemAggMV1mByHostV2Service` (which is keyed by hostEntityKey).
+ *
+ * Today's reads go through
+ * `MetricService.tryBuildEntityAggregateMVStatement` with raw SQL; this
+ * service exists so the model is registered with the analytics framework
+ * and the auto-create flow can manage the table + MV on app startup
+ * (idempotent with the AddMetricEntityMinuteAggregateMaterializedViews
+ * migration, which also backfills from MetricItemV3).
+ *
+ * AggregateFunction columns require `*Merge()` finalizers at read time —
+ * the generic typed `findBy` path is therefore not useful and should not
+ * be relied on against this table.
+ */
+export class MetricItemAggMV1mByContainerService extends AnalyticsDatabaseService<MetricItemAggMV1mByContainer> {
+  public constructor(clickhouseDatabase?: ClickhouseDatabase | undefined) {
+    super({
+      modelType: MetricItemAggMV1mByContainer,
+      database: clickhouseDatabase,
+    });
+  }
+}
+
+export default new MetricItemAggMV1mByContainerService();

package/Server/Services/MetricItemAggMV1mByK8sClusterService.ts

@@ -0,0 +1,31 @@
+import AnalyticsDatabaseService from "./AnalyticsDatabaseService";
+import ClickhouseDatabase from "../Infrastructure/ClickhouseDatabase";
+import MetricItemAggMV1mByK8sCluster from "../../Models/AnalyticsModels/MetricItemAggMV1mByK8sCluster";
+
+/**
+ * Read-side service for the `MetricItemAggMV1mByK8sCluster` MV target
+ * table — per-(k8s cluster entity key, minute) pre-aggregated metric
+ * values for Kubernetes-cluster-detail chart queries. Sister of
+ * `MetricItemAggMV1mByHostV2Service` (which is keyed by hostEntityKey).
+ *
+ * Today's reads go through
+ * `MetricService.tryBuildEntityAggregateMVStatement` with raw SQL; this
+ * service exists so the model is registered with the analytics framework
+ * and the auto-create flow can manage the table + MV on app startup
+ * (idempotent with the AddMetricEntityMinuteAggregateMaterializedViews
+ * migration, which also backfills from MetricItemV3).
+ *
+ * AggregateFunction columns require `*Merge()` finalizers at read time —
+ * the generic typed `findBy` path is therefore not useful and should not
+ * be relied on against this table.
+ */
+export class MetricItemAggMV1mByK8sClusterService extends AnalyticsDatabaseService<MetricItemAggMV1mByK8sCluster> {
+  public constructor(clickhouseDatabase?: ClickhouseDatabase | undefined) {
+    super({
+      modelType: MetricItemAggMV1mByK8sCluster,
+      database: clickhouseDatabase,
+    });
+  }
+}
+
+export default new MetricItemAggMV1mByK8sClusterService();

package/Server/Services/MetricItemAggMV1mByServiceService.ts

@@ -0,0 +1,31 @@
+import AnalyticsDatabaseService from "./AnalyticsDatabaseService";
+import ClickhouseDatabase from "../Infrastructure/ClickhouseDatabase";
+import MetricItemAggMV1mByService from "../../Models/AnalyticsModels/MetricItemAggMV1mByService";
+
+/**
+ * Read-side service for the `MetricItemAggMV1mByService` MV target table —
+ * per-(service entity key, minute) pre-aggregated metric values for
+ * service-detail chart queries. Sister of
+ * `MetricItemAggMV1mByHostV2Service` (which is keyed by hostEntityKey).
+ *
+ * Today's reads go through
+ * `MetricService.tryBuildEntityAggregateMVStatement` with raw SQL; this
+ * service exists so the model is registered with the analytics framework
+ * and the auto-create flow can manage the table + MV on app startup
+ * (idempotent with the AddMetricEntityMinuteAggregateMaterializedViews
+ * migration, which also backfills from MetricItemV3).
+ *
+ * AggregateFunction columns require `*Merge()` finalizers at read time —
+ * the generic typed `findBy` path is therefore not useful and should not
+ * be relied on against this table.
+ */
+export class MetricItemAggMV1mByServiceService extends AnalyticsDatabaseService<MetricItemAggMV1mByService> {
+  public constructor(clickhouseDatabase?: ClickhouseDatabase | undefined) {
+    super({
+      modelType: MetricItemAggMV1mByService,
+      database: clickhouseDatabase,
+    });
+  }
+}
+
+export default new MetricItemAggMV1mByServiceService();