@oneuptime/common 10.8.2 → 11.0.0

package/Models/AnalyticsModels/AnalyticsBaseModel/AnalyticsBaseModel.ts

@@ -58,6 +58,7 @@ export default class AnalyticsBaseModel extends CommonModel {
     enableWorkflowOn?: EnableWorkflowOn | undefined;
     enableRealtimeEventsOn?: EnableRealtimeEventsOn | undefined;
     partitionKey: string;
+    tableSettings?: string | undefined;
     projections?: Array<Projection> | undefined;
     materializedViews?: Array<MaterializedView> | undefined;
     enableMCP?: boolean | undefined;
@@ -92,6 +93,12 @@ export default class AnalyticsBaseModel extends CommonModel {
         description: "ID of this object",
         required: true,
         type: TableColumnType.ObjectID,
+        /*
+         * Ids are UUIDv7 (time-ordered — see ObjectID.generateTimeOrdered),
+         * so consecutive rows share their 48-bit timestamp prefix and ZSTD
+         * compresses them well; random v4 ids were incompressible.
+         */
+        codec: { codec: "ZSTD", level: 1 },
       }),
     );
 
@@ -105,16 +112,6 @@ export default class AnalyticsBaseModel extends CommonModel {
       }),
     );
 
-    columns.push(
-      new AnalyticsTableColumn({
-        key: "updatedAt",
-        title: "Updated",
-        description: "Date and Time when the object was updated.",
-        required: true,
-        type: TableColumnType.Date,
-      }),
-    );
-
     if (!data.primaryKeys || data.primaryKeys.length === 0) {
       throw new BadDataException("Primary keys are required");
     }
@@ -175,6 +172,7 @@ export default class AnalyticsBaseModel extends CommonModel {
     this.isMasterAdminApiDocs = data.isMasterAdminApiDocs || false;
     this.enableRealtimeEventsOn = data.enableRealtimeEventsOn;
     this.partitionKey = data.partitionKey;
+    this.tableSettings = data.tableSettings;
     this.projections = data.projections || [];
     this.materializedViews = data.materializedViews || [];
     this.enableMCP = data.enableMCP || false;
@@ -252,6 +250,14 @@ export default class AnalyticsBaseModel extends CommonModel {
     this._partitionKey = v;
   }
 
+  private _tableSettings: string | undefined = undefined;
+  public get tableSettings(): string | undefined {
+    return this._tableSettings;
+  }
+  public set tableSettings(v: string | undefined) {
+    this._tableSettings = v;
+  }
+
   private _sortKeys: Array<string> = [];
   public get sortKeys(): Array<string> {
     return this._sortKeys;
@@ -468,14 +474,6 @@ export default class AnalyticsBaseModel extends CommonModel {
     this.setColumnValue("createdAt", v);
   }
 
-  public get updatedAt(): Date | undefined {
-    return this.getColumnValue("updatedAt") as Date | undefined;
-  }
-
-  public set updatedAt(v: Date | undefined) {
-    this.setColumnValue("updatedAt", v);
-  }
-
   public getAPIDocumentationPath(): string {
     /*
      * Prefer the public CRUD API slug (e.g. "/logs" -> "logs") so the

package/Models/AnalyticsModels/AuditLog.ts

@@ -288,7 +288,9 @@ export default class AuditLog extends AnalyticsBaseModel {
       projections: [],
       sortKeys: ["projectId", "createdAt", "resourceType", "resourceId"],
       primaryKeys: ["projectId", "createdAt"],
-      partitionKey: "sipHash64(projectId) % 16",
+      partitionKey: "toYYYYMM(createdAt)",
+      tableSettings:
+        "ttl_only_drop_parts = 1, non_replicated_deduplication_window = 10000",
       ttlExpression: "retentionDate DELETE",
       /*
        * `createdAt` already participates in the AuditLog sort key

package/Models/AnalyticsModels/ExceptionInstance.ts

@@ -15,7 +15,7 @@ import { SpanStatus } from "./Span";
 import ServiceType from "../../Types/Telemetry/ServiceType";
 
 @OperationalResource()
-@OwnedThrough("serviceId", Service, { includeProjectScope: true })
+@OwnedThrough("primaryEntityId", Service, { includeProjectScope: true })
 export default class ExceptionInstance extends AnalyticsBaseModel {
   public constructor() {
     const projectIdColumn: AnalyticsTableColumn = new AnalyticsTableColumn({
@@ -48,75 +48,78 @@ export default class ExceptionInstance extends AnalyticsBaseModel {
       },
     });
 
-    const serviceIdColumn: AnalyticsTableColumn = new AnalyticsTableColumn({
-      key: "serviceId",
-      title: "Service ID",
-      description:
-        "ID of the resource the exception belongs to (Service / Host / DockerHost / KubernetesCluster / Monitor — disambiguated by serviceType)",
-      required: true,
-      type: TableColumnType.ObjectID,
-      accessControl: {
-        read: [
-          Permission.ProjectOwner,
-          Permission.ProjectAdmin,
-          Permission.ProjectMember,
-          Permission.Viewer,
-          Permission.TelemetryAdmin,
-          Permission.TelemetryMember,
-          Permission.TelemetryViewer,
-          Permission.ReadTelemetryException,
-        ],
-        create: [
-          Permission.ProjectOwner,
-          Permission.ProjectAdmin,
-          Permission.ProjectMember,
-          Permission.TelemetryAdmin,
-          Permission.TelemetryMember,
-          Permission.CreateTelemetryException,
-        ],
-        update: [],
-      },
-    });
-
-    const serviceTypeColumn: AnalyticsTableColumn = new AnalyticsTableColumn({
-      key: "serviceType",
-      isLowCardinality: true,
-      title: "Service Type",
-      description:
-        "Discriminator for serviceId — tells the read side which resource table to dispatch to",
-      required: false,
-      type: TableColumnType.Text,
-      skipIndex: {
-        name: "idx_service_type",
-        type: SkipIndexType.Set,
-        params: [10],
-        granularity: 4,
-      },
-      accessControl: {
-        read: [
-          Permission.ProjectOwner,
-          Permission.ProjectAdmin,
-          Permission.ProjectMember,
-          Permission.Viewer,
-          Permission.TelemetryAdmin,
-          Permission.TelemetryMember,
-          Permission.TelemetryViewer,
-          Permission.ReadTelemetryException,
-        ],
-        create: [
-          Permission.ProjectOwner,
-          Permission.ProjectAdmin,
-          Permission.ProjectMember,
-          Permission.TelemetryAdmin,
-          Permission.TelemetryMember,
-          Permission.CreateTelemetryException,
-        ],
-        update: [],
-      },
-    });
+    const primaryEntityIdColumn: AnalyticsTableColumn =
+      new AnalyticsTableColumn({
+        key: "primaryEntityId",
+        title: "Service ID",
+        description:
+          "ID of the resource the exception belongs to (Service / Host / DockerHost / KubernetesCluster / Monitor — disambiguated by primaryEntityType)",
+        required: true,
+        type: TableColumnType.ObjectID,
+        accessControl: {
+          read: [
+            Permission.ProjectOwner,
+            Permission.ProjectAdmin,
+            Permission.ProjectMember,
+            Permission.Viewer,
+            Permission.TelemetryAdmin,
+            Permission.TelemetryMember,
+            Permission.TelemetryViewer,
+            Permission.ReadTelemetryException,
+          ],
+          create: [
+            Permission.ProjectOwner,
+            Permission.ProjectAdmin,
+            Permission.ProjectMember,
+            Permission.TelemetryAdmin,
+            Permission.TelemetryMember,
+            Permission.CreateTelemetryException,
+          ],
+          update: [],
+        },
+      });
+
+    const primaryEntityTypeColumn: AnalyticsTableColumn =
+      new AnalyticsTableColumn({
+        key: "primaryEntityType",
+        isLowCardinality: true,
+        title: "Service Type",
+        description:
+          "Discriminator for primaryEntityId — tells the read side which resource table to dispatch to",
+        required: false,
+        type: TableColumnType.Text,
+        skipIndex: {
+          name: "idx_service_type",
+          type: SkipIndexType.Set,
+          params: [10],
+          granularity: 4,
+        },
+        accessControl: {
+          read: [
+            Permission.ProjectOwner,
+            Permission.ProjectAdmin,
+            Permission.ProjectMember,
+            Permission.Viewer,
+            Permission.TelemetryAdmin,
+            Permission.TelemetryMember,
+            Permission.TelemetryViewer,
+            Permission.ReadTelemetryException,
+          ],
+          create: [
+            Permission.ProjectOwner,
+            Permission.ProjectAdmin,
+            Permission.ProjectMember,
+            Permission.TelemetryAdmin,
+            Permission.TelemetryMember,
+            Permission.CreateTelemetryException,
+          ],
+          update: [],
+        },
+      });
 
     const timeColumn: AnalyticsTableColumn = new AnalyticsTableColumn({
       key: "time",
+      codec: [{ codec: "DoubleDelta" }, { codec: "ZSTD", level: 1 }],
       title: "Time",
       description: "When was the log created?",
       required: true,
@@ -146,10 +149,11 @@ export default class ExceptionInstance extends AnalyticsBaseModel {
 
     const timeUnixNanoColumn: AnalyticsTableColumn = new AnalyticsTableColumn({
       key: "timeUnixNano",
+      codec: [{ codec: "DoubleDelta" }, { codec: "ZSTD", level: 1 }],
       title: "Time (in Unix Nano)",
       description: "When was the log created?",
       required: true,
-      type: TableColumnType.LongNumber,
+      type: TableColumnType.UInt64,
       accessControl: {
         read: [
           Permission.ProjectOwner,
@@ -336,6 +340,7 @@ export default class ExceptionInstance extends AnalyticsBaseModel {
 
     const traceIdColumn: AnalyticsTableColumn = new AnalyticsTableColumn({
       key: "traceId",
+      codec: { codec: "ZSTD", level: 1 },
       title: "Trace ID",
       description: "ID of the trace",
       required: false,
@@ -371,6 +376,7 @@ export default class ExceptionInstance extends AnalyticsBaseModel {
 
     const spanIdColumn: AnalyticsTableColumn = new AnalyticsTableColumn({
       key: "spanId",
+      codec: { codec: "ZSTD", level: 1 },
       title: "Span ID",
       description: "ID of the span",
       required: false,
@@ -548,6 +554,8 @@ export default class ExceptionInstance extends AnalyticsBaseModel {
 
     const parsedFramesColumn: AnalyticsTableColumn = new AnalyticsTableColumn({
       key: "parsedFrames",
+      // JSON-as-String blob, same body-family treatment as stackTrace.
+      codec: { codec: "ZSTD", level: 3 },
       title: "Parsed Stack Frames",
       description: "Stack trace parsed into structured frames (JSON array)",
       required: false,
@@ -577,6 +585,7 @@ export default class ExceptionInstance extends AnalyticsBaseModel {
 
     const attributesColumn: AnalyticsTableColumn = new AnalyticsTableColumn({
       key: "attributes",
+      codec: { codec: "ZSTD", level: 3 },
       title: "Attributes",
       description: "Attributes",
       required: true,
@@ -605,8 +614,105 @@ export default class ExceptionInstance extends AnalyticsBaseModel {
       },
     });
 
+    const entityKeysColumn: AnalyticsTableColumn = new AnalyticsTableColumn({
+      key: "entityKeys",
+      codec: { codec: "ZSTD", level: 3 },
+      title: "Entity Keys",
+      description:
+        "Stable keys of every OpenTelemetry entity (service, host, k8s.pod, container, ...) this signal belongs to. A superset that includes the primary entity. Enables cross-cutting membership queries via has(entityKeys, :key).",
+      required: true,
+      defaultValue: [],
+      type: TableColumnType.ArrayText,
+      skipIndex: {
+        name: "idx_entity_keys",
+        type: SkipIndexType.BloomFilter,
+        params: [0.01],
+        granularity: 1,
+      },
+      accessControl: {
+        read: [
+          Permission.ProjectOwner,
+          Permission.ProjectAdmin,
+          Permission.ProjectMember,
+          Permission.Viewer,
+          Permission.TelemetryAdmin,
+          Permission.TelemetryMember,
+          Permission.TelemetryViewer,
+          Permission.ReadTelemetryException,
+        ],
+        create: [
+          Permission.ProjectOwner,
+          Permission.ProjectAdmin,
+          Permission.ProjectMember,
+          Permission.TelemetryAdmin,
+          Permission.TelemetryMember,
+          Permission.CreateTelemetryException,
+        ],
+        update: [],
+      },
+    });
+
+    /*
+     * Scalar per-entity-type key columns — denormalized single-value
+     * siblings of `entityKeys`. Each holds the 16-hex key (see
+     * Common/Utils/Telemetry/EntityKey) of the row's entity of that type,
+     * or '' when the resource carries no such entity (non-Nullable String,
+     * so old rows read the type default ''). Unlike the array column, a
+     * scalar equality predicate is usable as an MV/sort key and gets a
+     * cheaper bloom-filter probe; only the high-traffic keys
+     * (service/host/k8s.pod) carry skip indexes. Stamped at ingest by the
+     * same extractor that fills `entityKeys`; never part of identity.
+     */
+    const scalarEntityKeyColumns: Array<AnalyticsTableColumn> = [
+      {
+        key: "serviceEntityKey",
+        title: "Service Entity Key",
+        indexName: "idx_service_entity_key",
+      },
+      {
+        key: "hostEntityKey",
+        title: "Host Entity Key",
+        indexName: "idx_host_entity_key",
+      },
+      {
+        key: "k8sPodEntityKey",
+        title: "Kubernetes Pod Entity Key",
+        indexName: "idx_k8s_pod_entity_key",
+      },
+      { key: "k8sNodeEntityKey", title: "Kubernetes Node Entity Key" },
+      { key: "k8sClusterEntityKey", title: "Kubernetes Cluster Entity Key" },
+      { key: "containerEntityKey", title: "Container Entity Key" },
+    ].map(
+      (def: {
+        key: string;
+        title: string;
+        indexName?: string | undefined;
+      }): AnalyticsTableColumn => {
+        return new AnalyticsTableColumn({
+          key: def.key,
+          title: def.title,
+          description:
+            "Scalar entity key for this entity type (see entityKeys); '' when the resource has no entity of this type.",
+          required: true,
+          defaultValue: "",
+          type: TableColumnType.Text,
+          codec: { codec: "ZSTD", level: 1 },
+          skipIndex: def.indexName
+            ? {
+                name: def.indexName,
+                type: SkipIndexType.BloomFilter,
+                params: [0.01],
+                granularity: 1,
+              }
+            : undefined,
+          accessControl: entityKeysColumn.accessControl,
+        });
+      },
+    );
+
     const retentionDateColumn: AnalyticsTableColumn = new AnalyticsTableColumn({
       key: "retentionDate",
+      codec: [{ codec: "DoubleDelta" }, { codec: "ZSTD", level: 1 }],
       title: "Retention Date",
       description:
         "Date after which this row is eligible for TTL deletion, computed at ingest time as time + service.retainTelemetryDataForDays",
@@ -665,8 +771,8 @@ export default class ExceptionInstance extends AnalyticsBaseModel {
         "Individual exception occurrences captured from your telemetry. Query application errors and their attributes over time.",
       tableColumns: [
         projectIdColumn,
-        serviceIdColumn,
-        serviceTypeColumn,
+        primaryEntityIdColumn,
+        primaryEntityTypeColumn,
         timeColumn,
         timeUnixNanoColumn,
         exceptionTypeColumn,
@@ -682,18 +788,22 @@ export default class ExceptionInstance extends AnalyticsBaseModel {
         environmentColumn,
         parsedFramesColumn,
         attributesColumn,
+        entityKeysColumn,
+        ...scalarEntityKeyColumns,
         retentionDateColumn,
       ],
       projections: [
         {
           name: "proj_exception_group",
           query:
-            "SELECT projectId, serviceId, fingerprint, exceptionType, count() AS cnt, max(time) AS last_seen GROUP BY projectId, serviceId, fingerprint, exceptionType",
+            "SELECT projectId, primaryEntityId, fingerprint, exceptionType, count() AS cnt, max(time) AS last_seen GROUP BY projectId, primaryEntityId, fingerprint, exceptionType",
         },
       ],
-      sortKeys: ["projectId", "time", "serviceId", "fingerprint"],
-      primaryKeys: ["projectId", "time", "serviceId", "fingerprint"],
-      partitionKey: "sipHash64(projectId) % 16",
+      sortKeys: ["projectId", "time", "primaryEntityId", "fingerprint"],
+      primaryKeys: ["projectId", "time", "primaryEntityId", "fingerprint"],
+      partitionKey: "toYYYYMMDD(time)",
+      tableSettings:
+        "ttl_only_drop_parts = 1, non_replicated_deduplication_window = 10000",
       ttlExpression: "retentionDate DELETE",
       defaultSortColumn: "time",
     });
@@ -707,20 +817,20 @@ export default class ExceptionInstance extends AnalyticsBaseModel {
     this.setColumnValue("projectId", v);
   }
 
-  public get serviceId(): ObjectID | undefined {
-    return this.getColumnValue("serviceId") as ObjectID | undefined;
+  public get primaryEntityId(): ObjectID | undefined {
+    return this.getColumnValue("primaryEntityId") as ObjectID | undefined;
   }
 
-  public set serviceId(v: ObjectID | undefined) {
-    this.setColumnValue("serviceId", v);
+  public set primaryEntityId(v: ObjectID | undefined) {
+    this.setColumnValue("primaryEntityId", v);
   }
 
-  public get serviceType(): ServiceType | undefined {
-    return this.getColumnValue("serviceType") as ServiceType | undefined;
+  public get primaryEntityType(): ServiceType | undefined {
+    return this.getColumnValue("primaryEntityType") as ServiceType | undefined;
   }
 
-  public set serviceType(v: ServiceType | undefined) {
-    this.setColumnValue("serviceType", v);
+  public set primaryEntityType(v: ServiceType | undefined) {
+    this.setColumnValue("primaryEntityType", v);
   }
 
   public get time(): Date | undefined {
@@ -803,6 +913,14 @@ export default class ExceptionInstance extends AnalyticsBaseModel {
     this.setColumnValue("attributes", v);
   }
 
+  public get entityKeys(): Array<string> | undefined {
+    return this.getColumnValue("entityKeys") as Array<string> | undefined;
+  }
+
+  public set entityKeys(v: Array<string> | undefined) {
+    this.setColumnValue("entityKeys", v);
+  }
+
   public get spanStatusCode(): SpanStatus | undefined {
     return this.getColumnValue("spanStatusCode") as SpanStatus | undefined;
   }

package/Models/AnalyticsModels/Index.ts

@@ -2,7 +2,7 @@ import AnalyticsBaseModel from "./AnalyticsBaseModel/AnalyticsBaseModel";
 import Log from "./Log";
 import Metric from "./Metric";
 import MetricItemAggMV1m from "./MetricItemAggMV1m";
-import MetricItemAggMV1mByHost from "./MetricItemAggMV1mByHost";
+import MetricItemAggMV1mByHostV2 from "./MetricItemAggMV1mByHostV2";
 import MetricBaselineHourly from "./MetricBaselineHourly";
 import Span from "./Span";
 import ExceptionInstance from "./ExceptionInstance";
@@ -20,7 +20,12 @@ const AnalyticsModels: Array<{ new (): AnalyticsBaseModel }> = [
    * are populated by attached MVs on `Metric` insert. Read-only.
    */
   MetricItemAggMV1m,
-  MetricItemAggMV1mByHost,
+  /*
+   * The hostIdentifier-keyed V1 host rollup was dropped by the
+   * RekeyMetricHostRollupToEntityKey migration; only V2 is registered so
+   * boot-time schema sync doesn't resurrect the dropped V1 table.
+   */
+  MetricItemAggMV1mByHostV2,
   MetricBaselineHourly,
   ExceptionInstance,
   MonitorLog,