@oneuptime/common 10.0.30 → 10.0.33

package/Models/AnalyticsModels/ExceptionInstance.ts

@@ -1,6 +1,7 @@
 import AnalyticsBaseModel from "./AnalyticsBaseModel/AnalyticsBaseModel";
 import Route from "../../Types/API/Route";
 import AnalyticsTableEngine from "../../Types/AnalyticsDatabase/AnalyticsTableEngine";
+import AnalyticsTableName from "../../Types/AnalyticsDatabase/AnalyticsTableName";
 import AnalyticsTableColumn, {
   SkipIndexType,
 } from "../../Types/AnalyticsDatabase/TableColumn";
@@ -63,7 +64,7 @@ export default class ExceptionInstance extends AnalyticsBaseModel {
       title: "Time",
       description: "When was the log created?",
       required: true,
-      type: TableColumnType.Date,
+      type: TableColumnType.DateTime64,
       accessControl: {
         read: [
           Permission.ProjectOwner,
@@ -328,6 +329,12 @@ export default class ExceptionInstance extends AnalyticsBaseModel {
       description: "Name of the span",
       required: false,
       type: TableColumnType.Text,
+      skipIndex: {
+        name: "idx_span_name",
+        type: SkipIndexType.TokenBF,
+        params: [10240, 3, 0],
+        granularity: 4,
+      },
       accessControl: {
         read: [
           Permission.ProjectOwner,
@@ -352,6 +359,12 @@ export default class ExceptionInstance extends AnalyticsBaseModel {
         "Service version / release from service.version resource attribute",
       required: false,
       type: TableColumnType.Text,
+      skipIndex: {
+        name: "idx_release",
+        type: SkipIndexType.Set,
+        params: [100],
+        granularity: 4,
+      },
       accessControl: {
         read: [
           Permission.ProjectOwner,
@@ -376,6 +389,12 @@ export default class ExceptionInstance extends AnalyticsBaseModel {
         "Deployment environment from deployment.environment resource attribute",
       required: false,
       type: TableColumnType.Text,
+      skipIndex: {
+        name: "idx_environment",
+        type: SkipIndexType.Set,
+        params: [10],
+        granularity: 4,
+      },
       accessControl: {
         read: [
           Permission.ProjectOwner,
@@ -422,7 +441,7 @@ export default class ExceptionInstance extends AnalyticsBaseModel {
       description: "Attributes",
       required: true,
       defaultValue: {},
-      type: TableColumnType.JSON,
+      type: TableColumnType.MapStringString,
       accessControl: {
         read: [
           Permission.ProjectOwner,
@@ -451,7 +470,7 @@ export default class ExceptionInstance extends AnalyticsBaseModel {
     });
 
     super({
-      tableName: "ExceptionItem",
+      tableName: AnalyticsTableName.ExceptionInstance,
       tableEngine: AnalyticsTableEngine.MergeTree,
       singularName: "Exception Instance",
       pluralName: "Exception Instances",
@@ -505,7 +524,13 @@ export default class ExceptionInstance extends AnalyticsBaseModel {
         attributesColumn,
         retentionDateColumn,
       ],
-      projections: [],
+      projections: [
+        {
+          name: "proj_exception_group",
+          query:
+            "SELECT projectId, serviceId, fingerprint, exceptionType, count() AS cnt, max(time) AS last_seen GROUP BY projectId, serviceId, fingerprint, exceptionType",
+        },
+      ],
       sortKeys: ["projectId", "time", "serviceId", "fingerprint"],
       primaryKeys: ["projectId", "time", "serviceId", "fingerprint"],
       partitionKey: "sipHash64(projectId) % 16",

package/Models/AnalyticsModels/Log.ts

@@ -1,6 +1,7 @@
 import AnalyticsBaseModel from "./AnalyticsBaseModel/AnalyticsBaseModel";
 import Route from "../../Types/API/Route";
 import AnalyticsTableEngine from "../../Types/AnalyticsDatabase/AnalyticsTableEngine";
+import AnalyticsTableName from "../../Types/AnalyticsDatabase/AnalyticsTableName";
 import AnalyticsTableColumn, {
   SkipIndexType,
 } from "../../Types/AnalyticsDatabase/TableColumn";
@@ -64,7 +65,7 @@ export default class Log extends AnalyticsBaseModel {
       title: "Time",
       description: "When was the log created?",
       required: true,
-      type: TableColumnType.Date,
+      type: TableColumnType.DateTime64,
       accessControl: {
         read: [
           Permission.ProjectOwner,
@@ -165,7 +166,7 @@ export default class Log extends AnalyticsBaseModel {
       description: "Attributes",
       required: true,
       defaultValue: {},
-      type: TableColumnType.JSON,
+      type: TableColumnType.MapStringString,
       accessControl: {
         read: [
           Permission.ProjectOwner,
@@ -298,6 +299,78 @@ export default class Log extends AnalyticsBaseModel {
       },
     });
 
+    const observedTimeUnixNanoColumn: AnalyticsTableColumn =
+      new AnalyticsTableColumn({
+        key: "observedTimeUnixNano",
+        title: "Observed Time (in Unix Nano)",
+        description:
+          "When the log was observed/collected by the telemetry pipeline",
+        required: false,
+        type: TableColumnType.LongNumber,
+        accessControl: {
+          read: [
+            Permission.ProjectOwner,
+            Permission.ProjectAdmin,
+            Permission.ProjectMember,
+            Permission.ReadTelemetryServiceLog,
+          ],
+          create: [
+            Permission.ProjectOwner,
+            Permission.ProjectAdmin,
+            Permission.ProjectMember,
+            Permission.CreateTelemetryServiceLog,
+          ],
+          update: [],
+        },
+      });
+
+    const droppedAttributesCountColumn: AnalyticsTableColumn =
+      new AnalyticsTableColumn({
+        key: "droppedAttributesCount",
+        title: "Dropped Attributes Count",
+        description: "Number of attributes that were dropped during collection",
+        required: false,
+        type: TableColumnType.Number,
+        accessControl: {
+          read: [
+            Permission.ProjectOwner,
+            Permission.ProjectAdmin,
+            Permission.ProjectMember,
+            Permission.ReadTelemetryServiceLog,
+          ],
+          create: [
+            Permission.ProjectOwner,
+            Permission.ProjectAdmin,
+            Permission.ProjectMember,
+            Permission.CreateTelemetryServiceLog,
+          ],
+          update: [],
+        },
+      });
+
+    const flagsColumn: AnalyticsTableColumn = new AnalyticsTableColumn({
+      key: "flags",
+      title: "Flags",
+      description: "Log record flags (e.g., W3C trace flags)",
+      required: false,
+      type: TableColumnType.Number,
+      accessControl: {
+        read: [
+          Permission.ProjectOwner,
+          Permission.ProjectAdmin,
+          Permission.ProjectMember,
+          Permission.ReadTelemetryServiceLog,
+        ],
+        create: [
+          Permission.ProjectOwner,
+          Permission.ProjectAdmin,
+          Permission.ProjectMember,
+          Permission.CreateTelemetryServiceLog,
+        ],
+        update: [],
+      },
+    });
+
     const retentionDateColumn: AnalyticsTableColumn = new AnalyticsTableColumn({
       key: "retentionDate",
       title: "Retention Date",
@@ -309,7 +382,7 @@ export default class Log extends AnalyticsBaseModel {
     });
 
     super({
-      tableName: "LogItem",
+      tableName: AnalyticsTableName.Log,
       tableEngine: AnalyticsTableEngine.MergeTree,
       singularName: "Log",
       accessControl: {
@@ -352,9 +425,18 @@ export default class Log extends AnalyticsBaseModel {
         traceIdColumn,
         spanIdColumn,
         bodyColumn,
+        observedTimeUnixNanoColumn,
+        droppedAttributesCountColumn,
+        flagsColumn,
         retentionDateColumn,
       ],
-      projections: [],
+      projections: [
+        {
+          name: "proj_severity_histogram",
+          query:
+            "SELECT projectId, severityText, toStartOfInterval(time, INTERVAL 1 MINUTE) AS minute, count() AS cnt GROUP BY projectId, severityText, minute",
+        },
+      ],
       sortKeys: ["projectId", "time", "serviceId"],
       primaryKeys: ["projectId", "time", "serviceId"],
       partitionKey: "sipHash64(projectId) % 16",
@@ -450,6 +532,30 @@ export default class Log extends AnalyticsBaseModel {
     this.setColumnValue("spanId", v);
   }
 
+  public get observedTimeUnixNano(): number | undefined {
+    return this.getColumnValue("observedTimeUnixNano") as number | undefined;
+  }
+
+  public set observedTimeUnixNano(v: number | undefined) {
+    this.setColumnValue("observedTimeUnixNano", v);
+  }
+
+  public get droppedAttributesCount(): number | undefined {
+    return this.getColumnValue("droppedAttributesCount") as number | undefined;
+  }
+
+  public set droppedAttributesCount(v: number | undefined) {
+    this.setColumnValue("droppedAttributesCount", v);
+  }
+
+  public get flags(): number | undefined {
+    return this.getColumnValue("flags") as number | undefined;
+  }
+
+  public set flags(v: number | undefined) {
+    this.setColumnValue("flags", v);
+  }
+
   public get retentionDate(): Date | undefined {
     return this.getColumnValue("retentionDate") as Date | undefined;
   }

package/Models/AnalyticsModels/Metric.ts

@@ -1,6 +1,7 @@
 import AnalyticsBaseModel from "./AnalyticsBaseModel/AnalyticsBaseModel";
 import Route from "../../Types/API/Route";
 import AnalyticsTableEngine from "../../Types/AnalyticsDatabase/AnalyticsTableEngine";
+import AnalyticsTableName from "../../Types/AnalyticsDatabase/AnalyticsTableName";
 import AnalyticsTableColumn, {
   SkipIndexType,
 } from "../../Types/AnalyticsDatabase/TableColumn";
@@ -169,6 +170,12 @@ export default class Metric extends AnalyticsBaseModel {
         description: "Metric Point Type of this Metric",
         required: false,
         type: TableColumnType.Text,
+        skipIndex: {
+          name: "idx_metric_point_type",
+          type: SkipIndexType.Set,
+          params: [5],
+          granularity: 4,
+        },
         accessControl: {
           read: [
             Permission.ProjectOwner,
@@ -192,7 +199,7 @@ export default class Metric extends AnalyticsBaseModel {
       title: "Time",
       description: "When did the Metric happen?",
       required: true,
-      type: TableColumnType.Date,
+      type: TableColumnType.DateTime64,
       accessControl: {
         read: [
           Permission.ProjectOwner,
@@ -215,7 +222,7 @@ export default class Metric extends AnalyticsBaseModel {
       title: "Start Time",
       description: "When did the Metric happen?",
       required: false,
-      type: TableColumnType.Date,
+      type: TableColumnType.DateTime64,
       accessControl: {
         read: [
           Permission.ProjectOwner,
@@ -286,7 +293,7 @@ export default class Metric extends AnalyticsBaseModel {
       title: "Attributes",
       description: "Attributes",
       required: true,
-      type: TableColumnType.JSON,
+      type: TableColumnType.MapStringString,
       defaultValue: {},
       accessControl: {
         read: [
@@ -357,7 +364,7 @@ export default class Metric extends AnalyticsBaseModel {
       title: "Count",
       description: "Count",
       required: false,
-      type: TableColumnType.Number,
+      type: TableColumnType.BigNumber,
       accessControl: {
         read: [
           Permission.ProjectOwner,
@@ -473,7 +480,7 @@ export default class Metric extends AnalyticsBaseModel {
       description: "Bucket Counts",
       required: true,
       defaultValue: [],
-      type: TableColumnType.ArrayNumber,
+      type: TableColumnType.ArrayBigNumber,
       accessControl: {
         read: [
           Permission.ProjectOwner,
@@ -498,7 +505,7 @@ export default class Metric extends AnalyticsBaseModel {
         description: "Explicit Bonds",
         required: true,
         defaultValue: [],
-        type: TableColumnType.ArrayNumber,
+        type: TableColumnType.ArrayBigNumber,
         accessControl: {
           read: [
             Permission.ProjectOwner,
@@ -528,7 +535,7 @@ export default class Metric extends AnalyticsBaseModel {
     });
 
     super({
-      tableName: "MetricItem",
+      tableName: AnalyticsTableName.Metric,
       tableEngine: AnalyticsTableEngine.MergeTree,
       singularName: "Metric",
       pluralName: "Metrics",
@@ -583,8 +590,8 @@ export default class Metric extends AnalyticsBaseModel {
         retentionDateColumn,
       ],
       projections: [],
-      sortKeys: ["projectId", "time", "serviceId"],
-      primaryKeys: ["projectId", "time", "serviceId"],
+      sortKeys: ["projectId", "name", "serviceId", "time"],
+      primaryKeys: ["projectId", "name", "serviceId", "time"],
       partitionKey: "sipHash64(projectId) % 16",
       ttlExpression: "retentionDate DELETE",
     });

package/Models/AnalyticsModels/MonitorLog.ts

@@ -1,6 +1,7 @@
 import AnalyticsBaseModel from "./AnalyticsBaseModel/AnalyticsBaseModel";
 import Route from "../../Types/API/Route";
 import AnalyticsTableEngine from "../../Types/AnalyticsDatabase/AnalyticsTableEngine";
+import AnalyticsTableName from "../../Types/AnalyticsDatabase/AnalyticsTableName";
 import AnalyticsTableColumn from "../../Types/AnalyticsDatabase/TableColumn";
 import TableColumnType from "../../Types/AnalyticsDatabase/TableColumnType";
 import { JSONObject } from "../../Types/JSON";
@@ -61,7 +62,7 @@ export default class MonitorLog extends AnalyticsBaseModel {
       title: "Time",
       description: "When was the log created?",
       required: true,
-      type: TableColumnType.Date,
+      type: TableColumnType.DateTime64,
       accessControl: {
         read: [
           Permission.ProjectOwner,
@@ -86,6 +87,7 @@ export default class MonitorLog extends AnalyticsBaseModel {
       required: true,
       defaultValue: {},
       type: TableColumnType.JSON,
+      codec: { codec: "ZSTD", level: 3 },
       accessControl: {
         read: [
           Permission.ProjectOwner,
@@ -114,7 +116,7 @@ export default class MonitorLog extends AnalyticsBaseModel {
     });
 
     super({
-      tableName: "MonitorLog",
+      tableName: AnalyticsTableName.MonitorLog,
       tableEngine: AnalyticsTableEngine.MergeTree,
       singularName: "Monitor Log",
       accessControl: {

package/Models/AnalyticsModels/Span.ts

@@ -1,6 +1,7 @@
 import AnalyticsBaseModel from "./AnalyticsBaseModel/AnalyticsBaseModel";
 import Route from "../../Types/API/Route";
 import AnalyticsTableEngine from "../../Types/AnalyticsDatabase/AnalyticsTableEngine";
+import AnalyticsTableName from "../../Types/AnalyticsDatabase/AnalyticsTableName";
 import AnalyticsTableColumn, {
   SkipIndexType,
 } from "../../Types/AnalyticsDatabase/TableColumn";
@@ -95,7 +96,7 @@ export default class Span extends AnalyticsBaseModel {
       title: "Start Time",
       description: "When did the span start?",
       required: true,
-      type: TableColumnType.Date,
+      type: TableColumnType.DateTime64,
       accessControl: {
         read: [
           Permission.ProjectOwner,
@@ -118,7 +119,7 @@ export default class Span extends AnalyticsBaseModel {
       title: "End Time",
       description: "When did the span end?",
       required: true,
-      type: TableColumnType.Date,
+      type: TableColumnType.DateTime64,
       accessControl: {
         read: [
           Permission.ProjectOwner,
@@ -143,6 +144,7 @@ export default class Span extends AnalyticsBaseModel {
         description: "When did the span start?",
         required: true,
         type: TableColumnType.LongNumber,
+        codec: { codec: "ZSTD", level: 1 },
         accessControl: {
           read: [
             Permission.ProjectOwner,
@@ -167,6 +169,7 @@ export default class Span extends AnalyticsBaseModel {
         description: "How long did the span last?",
         required: true,
         type: TableColumnType.LongNumber,
+        codec: { codec: "ZSTD", level: 1 },
         accessControl: {
           read: [
             Permission.ProjectOwner,
@@ -191,6 +194,7 @@ export default class Span extends AnalyticsBaseModel {
         description: "When did the span end?",
         required: true,
         type: TableColumnType.LongNumber,
+        codec: { codec: "ZSTD", level: 1 },
         accessControl: {
           read: [
             Permission.ProjectOwner,
@@ -214,6 +218,7 @@ export default class Span extends AnalyticsBaseModel {
       description: "ID of the trace",
       required: true,
       type: TableColumnType.Text,
+      codec: { codec: "ZSTD", level: 1 },
       skipIndex: {
         name: "idx_trace_id",
         type: SkipIndexType.BloomFilter,
@@ -243,6 +248,7 @@ export default class Span extends AnalyticsBaseModel {
       description: "ID of the span",
       required: true,
       type: TableColumnType.Text,
+      codec: { codec: "ZSTD", level: 1 },
       skipIndex: {
         name: "idx_span_id",
         type: SkipIndexType.BloomFilter,
@@ -272,6 +278,13 @@ export default class Span extends AnalyticsBaseModel {
       description: "ID of the parent span",
       required: false,
       type: TableColumnType.Text,
+      codec: { codec: "ZSTD", level: 1 },
+      skipIndex: {
+        name: "idx_parent_span_id",
+        type: SkipIndexType.BloomFilter,
+        params: [0.01],
+        granularity: 1,
+      },
       accessControl: {
         read: [
           Permission.ProjectOwner,
@@ -318,7 +331,7 @@ export default class Span extends AnalyticsBaseModel {
       description: "Attributes",
       required: true,
       defaultValue: {},
-      type: TableColumnType.JSON,
+      type: TableColumnType.MapStringString,
       accessControl: {
         read: [
           Permission.ProjectOwner,
@@ -367,6 +380,7 @@ export default class Span extends AnalyticsBaseModel {
       required: true,
       defaultValue: [],
       type: TableColumnType.JSONArray,
+      codec: { codec: "ZSTD", level: 3 },
       accessControl: {
         read: [
           Permission.ProjectOwner,
@@ -389,8 +403,9 @@ export default class Span extends AnalyticsBaseModel {
       title: "Links",
       description: "Span Links",
       required: true,
-      defaultValue: {},
+      defaultValue: [],
       type: TableColumnType.JSON,
+      codec: { codec: "ZSTD", level: 3 },
       accessControl: {
         read: [
           Permission.ProjectOwner,
@@ -443,6 +458,7 @@ export default class Span extends AnalyticsBaseModel {
       description: "Status Message",
       required: false,
       type: TableColumnType.Text,
+      codec: { codec: "ZSTD", level: 1 },
       accessControl: {
         read: [
           Permission.ProjectOwner,
@@ -495,6 +511,43 @@ export default class Span extends AnalyticsBaseModel {
       description: "Kind of the span",
       required: false,
       type: TableColumnType.Text,
+      skipIndex: {
+        name: "idx_kind",
+        type: SkipIndexType.Set,
+        params: [5],
+        granularity: 4,
+      },
+      accessControl: {
+        read: [
+          Permission.ProjectOwner,
+          Permission.ProjectAdmin,
+          Permission.ProjectMember,
+          Permission.ReadTelemetryServiceTraces,
+        ],
+        create: [
+          Permission.ProjectOwner,
+          Permission.ProjectAdmin,
+          Permission.ProjectMember,
+          Permission.CreateTelemetryServiceTraces,
+        ],
+        update: [],
+      },
+    });
+
+    const hasExceptionColumn: AnalyticsTableColumn = new AnalyticsTableColumn({
+      key: "hasException",
+      title: "Has Exception",
+      description:
+        "Whether this span contains an exception event, populated at ingest time for fast error filtering",
+      required: true,
+      defaultValue: false,
+      type: TableColumnType.Boolean,
+      skipIndex: {
+        name: "idx_has_exception",
+        type: SkipIndexType.Set,
+        params: [2],
+        granularity: 4,
+      },
       accessControl: {
         read: [
           Permission.ProjectOwner,
@@ -523,7 +576,7 @@ export default class Span extends AnalyticsBaseModel {
     });
 
     super({
-      tableName: "SpanItem",
+      tableName: AnalyticsTableName.Span,
       tableEngine: AnalyticsTableEngine.MergeTree,
       singularName: "Span",
       pluralName: "Spans",
@@ -574,9 +627,21 @@ export default class Span extends AnalyticsBaseModel {
         statusMessageColumn,
         nameColumn,
         kindColumn,
+        hasExceptionColumn,
         retentionDateColumn,
       ],
-      projections: [],
+      projections: [
+        {
+          name: "proj_agg_by_service",
+          query:
+            "SELECT projectId, serviceId, toStartOfMinute(startTime) AS minute, count() AS cnt, avg(durationUnixNano) AS avg_duration, quantile(0.99)(durationUnixNano) AS p99_duration GROUP BY projectId, serviceId, minute",
+        },
+        {
+          name: "proj_trace_by_id",
+          query:
+            "SELECT projectId, traceId, startTime, serviceId, spanId, parentSpanId, name, durationUnixNano, statusCode, hasException ORDER BY (projectId, traceId, startTime)",
+        },
+      ],
       sortKeys: ["projectId", "startTime", "serviceId", "traceId"],
       primaryKeys: ["projectId", "startTime", "serviceId", "traceId"],
       partitionKey: "sipHash64(projectId) % 16",
@@ -736,6 +801,14 @@ export default class Span extends AnalyticsBaseModel {
     this.setColumnValue("statusMessage", v);
   }
 
+  public get hasException(): boolean | undefined {
+    return this.getColumnValue("hasException") as boolean | undefined;
+  }
+
+  public set hasException(v: boolean | undefined) {
+    this.setColumnValue("hasException", v);
+  }
+
   public get retentionDate(): Date | undefined {
     return this.getColumnValue("retentionDate") as Date | undefined;
   }

package/Models/DatabaseModels/Index.ts

@@ -37,6 +37,10 @@ import IncidentTemplateOwnerUser from "./IncidentTemplateOwnerUser";
 //Labels.
 import Label from "./Label";
 import LogSavedView from "./LogSavedView";
+import LogPipeline from "./LogPipeline";
+import LogPipelineProcessor from "./LogPipelineProcessor";
+import LogDropFilter from "./LogDropFilter";
+import LogScrubRule from "./LogScrubRule";
 // Monitors
 import Monitor from "./Monitor";
 import MonitorCustomField from "./MonitorCustomField";
@@ -252,6 +256,10 @@ const AllModelTypes: Array<{
   ApiKey,
   Label,
   LogSavedView,
+  LogPipeline,
+  LogPipelineProcessor,
+  LogDropFilter,
+  LogScrubRule,
   ApiKeyPermission,
   ProjectSmtpConfig,
   StatusPage,