@oneuptime/common 10.0.31 → 10.0.34

package/Server/Utils/AnalyticsDatabase/StatementGenerator.ts

@@ -8,7 +8,7 @@ import logger from "../Logger";
 import { SQL, Statement } from "./Statement";
 import AnalyticsBaseModel from "../../../Models/AnalyticsModels/AnalyticsBaseModel/AnalyticsBaseModel";
 import CommonModel, {
-  Record,
+  Record as AnalyticsRecord,
   RecordValue,
 } from "../../../Models/AnalyticsModels/AnalyticsBaseModel/CommonModel";
 import AnalyticsTableColumn, {
@@ -85,7 +85,7 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
     return columnNames;
   }
 
-  public getRecordValuesStatement(record: Record): string {
+  public getRecordValuesStatement(record: AnalyticsRecord): string {
     let valueStatement: string = "";
 
     for (const value of record) {
@@ -106,7 +106,7 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
     return valueStatement;
   }
 
-  public getValuesStatement(records: Array<Record>): string {
+  public getValuesStatement(records: Array<AnalyticsRecord>): string {
     let statement: string = "";
     for (const record of records) {
       statement += `(${this.getRecordValuesStatement(record)}), `;
@@ -126,10 +126,10 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
       this.model.getTableColumns(),
     );
 
-    const records: Array<Record> = [];
+    const records: Array<AnalyticsRecord> = [];
 
     for (const item of data.item) {
-      const record: Record = this.getRecord(item);
+      const record: AnalyticsRecord = this.getRecord(item);
       records.push(record);
     }
 
@@ -154,8 +154,8 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
     return statement;
   }
 
-  private getRecord(item: CommonModel): Record {
-    const record: Record = [];
+  private getRecord(item: CommonModel): AnalyticsRecord {
+    const record: AnalyticsRecord = [];
 
     for (const column of item.getTableColumns()) {
       const value: RecordValue | undefined = this.sanitizeValue(
@@ -251,6 +251,34 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
       value = `CAST(${this.escapeStringLiteral(value.toString())} AS Int128)`;
     }
 
+    if (column.type === TableColumnType.BigNumber) {
+      if (typeof value === "string") {
+        value = parseInt(value);
+      }
+    }
+
+    if (column.type === TableColumnType.ArrayBigNumber) {
+      value = `[${(value as Array<number>)
+        .map((v: number) => {
+          if (v && typeof v !== "number") {
+            v = parseFloat(v);
+            return isNaN(v) ? "NULL" : v;
+          }
+          return v;
+        })
+        .join(", ")}]`;
+    }
+
+    if (column.type === TableColumnType.MapStringString) {
+      const mapObj: Record<string, string> = value as Record<string, string>;
+      const entries: Array<string> = Object.entries(mapObj).map(
+        ([k, v]: [string, string]) => {
+          return `${this.escapeStringLiteral(k)}, ${this.escapeStringLiteral(v)}`;
+        },
+      );
+      value = `map(${entries.join(", ")})`;
+    }
+
     return value;
   }
 
@@ -387,6 +415,28 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
         } else {
           whereStatement.append(SQL`AND ${key} IS NULL`);
         }
+      } else if (
+        tableColumn.type === TableColumnType.MapStringString &&
+        typeof value === "object"
+      ) {
+        const mapValue: Record<string, string> = value as Record<
+          string,
+          string
+        >;
+        for (const mapKey in mapValue) {
+          if (mapValue[mapKey] === undefined) {
+            continue;
+          }
+          whereStatement.append(
+            SQL`AND ${key}[${{
+              value: mapKey,
+              type: TableColumnType.Text,
+            }}] = ${{
+              value: mapValue[mapKey] as string,
+              type: TableColumnType.Text,
+            }}`,
+          );
+        }
       } else if (
         (tableColumn.type === TableColumnType.JSON ||
           tableColumn.type === TableColumnType.JSONArray) &&
@@ -453,6 +503,10 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
 
     let first: boolean = true;
     for (const key in groupBy) {
+      if (!this.model.getTableColumn(key)) {
+        throw new BadDataException(`Unknown column: ${key}`);
+      }
+
       if (first) {
         first = false;
       } else {
@@ -468,6 +522,10 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
     const sortStatement: Statement = new Statement();
 
     for (const key in sort) {
+      if (!this.model.getTableColumn(key)) {
+        throw new BadDataException(`Unknown column: ${key}`);
+      }
+
       const value: SortOrder = sort[key]!;
       sortStatement.append(SQL`${key} `).append(
         {
@@ -491,6 +549,10 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
     for (const key in select) {
       const value: any = select[key];
       if (value) {
+        if (!this.model.getTableColumn(key)) {
+          throw new BadDataException(`Unknown column: ${key}`);
+        }
+
         columns.push(key);
         if (first) {
           first = false;
@@ -640,6 +702,13 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
       );
     }
 
+    // Append projections after indexes
+    if (this.model.projections && this.model.projections.length > 0) {
+      for (const projection of this.model.projections) {
+        columns.append(`, PROJECTION ${projection.name} (${projection.query})`);
+      }
+    }
+
     return columns;
   }
 
@@ -649,13 +718,16 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
     return {
       String: TableColumnType.Text,
       Int32: TableColumnType.Number,
-      Int64: TableColumnType.LongNumber,
+      Int64: TableColumnType.BigNumber,
       Int128: TableColumnType.LongNumber,
       Float32: TableColumnType.Decimal,
       Float64: TableColumnType.Decimal,
       DateTime: TableColumnType.Date,
+      "DateTime64(9)": TableColumnType.DateTime64,
       "Array(String)": TableColumnType.ArrayText,
       "Array(Int32)": TableColumnType.ArrayNumber,
+      "Array(Int64)": TableColumnType.ArrayBigNumber,
+      "Map(String, String)": TableColumnType.MapStringString,
       JSON: TableColumnType.JSON, //JSONArray is also JSON
       Bool: TableColumnType.Boolean,
     }[clickhouseType];
@@ -671,11 +743,15 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
       [TableColumnType.IP]: SQL`String`,
       [TableColumnType.Port]: SQL`String`,
       [TableColumnType.Date]: SQL`DateTime`,
+      [TableColumnType.DateTime64]: SQL`DateTime64(9)`,
       [TableColumnType.JSON]: SQL`String`, // we use JSON as a string because ClickHouse has really good JSON support for string types
       [TableColumnType.JSONArray]: SQL`String`, // we use JSON as a string because ClickHouse has really good JSON support for string types
       [TableColumnType.ArrayNumber]: SQL`Array(Int32)`,
+      [TableColumnType.ArrayBigNumber]: SQL`Array(Int64)`,
       [TableColumnType.ArrayText]: SQL`Array(String)`,
       [TableColumnType.LongNumber]: SQL`Int128`,
+      [TableColumnType.BigNumber]: SQL`Int64`,
+      [TableColumnType.MapStringString]: SQL`Map(String, String)`,
     }[type];
 
     if (!statement) {
@@ -700,12 +776,32 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
   }
 
   public toAddColumnStatement(column: AnalyticsTableColumn): Statement {
+    // Build column definition without skip index (indexes must be added separately via ADD INDEX)
+    const columnDef: Statement = new Statement();
+
+    columnDef
+      .append(column.key)
+      .append(SQL` `)
+      .append(
+        column.required
+          ? this.toColumnType(column.type)
+          : SQL`Nullable(`
+              .append(this.toColumnType(column.type))
+              .append(SQL`)`),
+      );
+
+    if (column.codec) {
+      const codecStr: string =
+        column.codec.level !== undefined
+          ? `${column.codec.codec}(${column.codec.level})`
+          : column.codec.codec;
+      columnDef.append(` CODEC(${codecStr})`);
+    }
+
     const statement: Statement = SQL`
             ALTER TABLE ${this.database.getDatasourceOptions().database!}.${
               this.model.tableName
-            } ADD COLUMN IF NOT EXISTS `.append(
-      this.toColumnsCreateStatement([column]),
-    );
+            } ADD COLUMN IF NOT EXISTS `.append(columnDef);
 
     logger.debug(`${this.model.tableName} Add Column Statement`);
     logger.debug(statement);
@@ -713,6 +809,37 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
     return statement;
   }
 
+  public toAddSkipIndexStatement(
+    column: AnalyticsTableColumn,
+  ): Statement | null {
+    if (!column.skipIndex) {
+      return null;
+    }
+
+    const idx: AnalyticsTableColumn["skipIndex"] = column.skipIndex;
+    const paramsStr: string =
+      idx.params && idx.params.length > 0 ? `(${idx.params.join(", ")})` : "";
+
+    const needsAssumeNotNull: boolean =
+      !column.required &&
+      (idx.type === SkipIndexType.TokenBF ||
+        idx.type === SkipIndexType.NgramBF);
+    const columnExpr: string = needsAssumeNotNull
+      ? `assumeNotNull(${column.key})`
+      : column.key;
+
+    const databaseName: string = this.database.getDatasourceOptions().database!;
+    const statement: Statement = new Statement();
+    statement.append(
+      `ALTER TABLE ${databaseName}.${this.model.tableName} ADD INDEX IF NOT EXISTS ${idx.name} ${columnExpr} TYPE ${idx.type}${paramsStr} GRANULARITY ${idx.granularity}`,
+    );
+
+    logger.debug(`${this.model.tableName} Add Skip Index Statement`);
+    logger.debug(statement);
+
+    return statement;
+  }
+
   public toDropColumnStatement(columnName: string): string {
     const statement: string = `ALTER TABLE ${this.database.getDatasourceOptions()
       .database!}.${this.model.tableName} DROP COLUMN IF EXISTS ${columnName}`;

package/Tests/Server/Services/LogAggregationService.test.ts

@@ -2,6 +2,7 @@ import LogAggregationService, {
   FacetRequest,
 } from "../../../Server/Services/LogAggregationService";
 import { Statement } from "../../../Server/Utils/AnalyticsDatabase/Statement";
+import AnalyticsTableName from "../../../Types/AnalyticsDatabase/AnalyticsTableName";
 import ObjectID from "../../../Types/ObjectID";
 import OneUptimeDate from "../../../Types/Date";
 import { describe, expect, test } from "@jest/globals";
@@ -35,7 +36,7 @@ describe("LogAggregationService", () => {
 
     expect(statement.query_params).toStrictEqual({
       p0: "severityText",
-      p1: "LogItem",
+      p1: AnalyticsTableName.Log,
       p2: defaultRequest.projectId.toString(),
       p3: OneUptimeDate.toClickhouseDateTime(defaultRequest.startTime),
       p4: OneUptimeDate.toClickhouseDateTime(defaultRequest.endTime),
@@ -55,7 +56,7 @@ describe("LogAggregationService", () => {
 
     expect(statement.query_params).toStrictEqual({
       p0: facetKey,
-      p1: "LogItem",
+      p1: AnalyticsTableName.Log,
       p2: defaultRequest.projectId.toString(),
       p3: OneUptimeDate.toClickhouseDateTime(defaultRequest.startTime),
       p4: OneUptimeDate.toClickhouseDateTime(defaultRequest.endTime),

package/Types/AnalyticsDatabase/AnalyticsTableName.ts

@@ -0,0 +1,9 @@
+enum AnalyticsTableName {
+  Log = "LogItemV2",
+  Metric = "MetricItemV2",
+  ExceptionInstance = "ExceptionItemV2",
+  Span = "SpanItemV2",
+  MonitorLog = "MonitorLogV2",
+}
+
+export default AnalyticsTableName;

package/Types/AnalyticsDatabase/TableColumnType.ts

@@ -10,8 +10,12 @@ enum ColumnType {
   ArrayNumber = "Array of Numbers",
   ArrayText = "Array of Text",
   LongNumber = "Long Number",
+  BigNumber = "Big Number",
+  DateTime64 = "DateTime64",
   IP = "IP",
   Port = "Port",
+  MapStringString = "Map(String, String)",
+  ArrayBigNumber = "Array of Big Numbers",
 }
 
 export default ColumnType;

package/Types/Date.ts

@@ -1524,4 +1524,26 @@ export default class OneUptimeDate {
     const parsedDate: Date = this.fromString(date);
     return moment(parsedDate).utc().format("YYYY-MM-DD HH:mm:ss");
   }
+
+  public static toClickhouseDateTime64(
+    date: Date | string,
+    nanoTimestamp?: number,
+  ): string {
+    const parsedDate: Date = this.fromString(date);
+    const base: string = moment(parsedDate).utc().format("YYYY-MM-DD HH:mm:ss");
+
+    let nanoFraction: string;
+
+    if (nanoTimestamp !== undefined && nanoTimestamp > 0) {
+      // Extract sub-second nanoseconds from the unix nano timestamp
+      const subSecondNanos: number = nanoTimestamp % 1_000_000_000;
+      nanoFraction = subSecondNanos.toString().padStart(9, "0");
+    } else {
+      // Fall back to milliseconds from the Date object
+      const ms: number = parsedDate.getMilliseconds();
+      nanoFraction = (ms * 1_000_000).toString().padStart(9, "0");
+    }
+
+    return `${base}.${nanoFraction}`;
+  }
 }

package/Types/Log/LogDropFilterAction.ts

@@ -0,0 +1,6 @@
+enum LogDropFilterAction {
+  Drop = "drop",
+  Sample = "sample",
+}
+
+export default LogDropFilterAction;

package/Types/Log/LogPipelineProcessorType.ts

@@ -0,0 +1,44 @@
+enum LogPipelineProcessorType {
+  GrokParser = "GrokParser",
+  AttributeRemapper = "AttributeRemapper",
+  SeverityRemapper = "SeverityRemapper",
+  CategoryProcessor = "CategoryProcessor",
+}
+
+export interface GrokParserConfig {
+  source: string; // field to parse, e.g. "body"
+  pattern: string; // grok pattern
+  targetPrefix?: string; // prefix for extracted attributes
+}
+
+export interface AttributeRemapperConfig {
+  sourceKey: string; // source attribute key
+  targetKey: string; // target attribute key
+  preserveSource?: boolean; // keep original key (default false)
+  overrideOnConflict?: boolean; // overwrite if target exists (default true)
+}
+
+export interface SeverityRemapperConfig {
+  sourceKey: string; // attribute key containing severity info
+  mappings: Array<{
+    matchValue: string; // value to match (case-insensitive)
+    severityText: string; // mapped severity text
+    severityNumber: number; // mapped severity number
+  }>;
+}
+
+export interface CategoryProcessorConfig {
+  targetKey: string; // attribute key to store the category
+  categories: Array<{
+    name: string; // category name/value
+    filterQuery: string; // condition to match
+  }>;
+}
+
+export type LogPipelineProcessorConfig =
+  | GrokParserConfig
+  | AttributeRemapperConfig
+  | SeverityRemapperConfig
+  | CategoryProcessorConfig;
+
+export default LogPipelineProcessorType;

package/Types/Log/LogScrubAction.ts

@@ -0,0 +1,7 @@
+enum LogScrubAction {
+  Mask = "mask",
+  Hash = "hash",
+  Redact = "redact",
+}
+
+export default LogScrubAction;

package/Types/Log/LogScrubPatternType.ts

@@ -0,0 +1,10 @@
+enum LogScrubPatternType {
+  Email = "email",
+  CreditCard = "creditCard",
+  SSN = "ssn",
+  PhoneNumber = "phoneNumber",
+  IPAddress = "ipAddress",
+  Custom = "custom",
+}
+
+export default LogScrubPatternType;

package/Types/Permission.ts

@@ -82,6 +82,30 @@ enum Permission {
   EditTelemetryServiceLog = "EditTelemetryServiceLog",
   ReadTelemetryServiceLog = "ReadTelemetryServiceLog",
 
+  // Log Pipelines
+  CreateProjectLogPipeline = "CreateProjectLogPipeline",
+  DeleteProjectLogPipeline = "DeleteProjectLogPipeline",
+  EditProjectLogPipeline = "EditProjectLogPipeline",
+  ReadProjectLogPipeline = "ReadProjectLogPipeline",
+
+  // Log Pipeline Processors
+  CreateProjectLogPipelineProcessor = "CreateProjectLogPipelineProcessor",
+  DeleteProjectLogPipelineProcessor = "DeleteProjectLogPipelineProcessor",
+  EditProjectLogPipelineProcessor = "EditProjectLogPipelineProcessor",
+  ReadProjectLogPipelineProcessor = "ReadProjectLogPipelineProcessor",
+
+  // Log Drop Filters
+  CreateProjectLogDropFilter = "CreateProjectLogDropFilter",
+  DeleteProjectLogDropFilter = "DeleteProjectLogDropFilter",
+  EditProjectLogDropFilter = "EditProjectLogDropFilter",
+  ReadProjectLogDropFilter = "ReadProjectLogDropFilter",
+
+  // Log Scrub Rules
+  CreateProjectLogScrubRule = "CreateProjectLogScrubRule",
+  DeleteProjectLogScrubRule = "DeleteProjectLogScrubRule",
+  EditProjectLogScrubRule = "EditProjectLogScrubRule",
+  ReadProjectLogScrubRule = "ReadProjectLogScrubRule",
+
   // Exceptions
   CreateTelemetryException = "CreateTelemetryException",
   DeleteTelemetryException = "DeleteTelemetryException",
@@ -3981,6 +4005,156 @@ export class PermissionHelper {
         group: PermissionGroup.Telemetry,
       },
 
+      // Log Pipeline Permissions
+      {
+        permission: Permission.CreateProjectLogPipeline,
+        title: "Create Log Pipeline",
+        description:
+          "This permission can create Log Pipelines in this project.",
+        isAssignableToTenant: true,
+        isAccessControlPermission: false,
+        group: PermissionGroup.Telemetry,
+      },
+      {
+        permission: Permission.DeleteProjectLogPipeline,
+        title: "Delete Log Pipeline",
+        description:
+          "This permission can delete Log Pipelines of this project.",
+        isAssignableToTenant: true,
+        isAccessControlPermission: false,
+        group: PermissionGroup.Telemetry,
+      },
+      {
+        permission: Permission.EditProjectLogPipeline,
+        title: "Edit Log Pipeline",
+        description: "This permission can edit Log Pipelines of this project.",
+        isAssignableToTenant: true,
+        isAccessControlPermission: false,
+        group: PermissionGroup.Telemetry,
+      },
+      {
+        permission: Permission.ReadProjectLogPipeline,
+        title: "Read Log Pipeline",
+        description: "This permission can read Log Pipelines of this project.",
+        isAssignableToTenant: true,
+        isAccessControlPermission: false,
+        group: PermissionGroup.Telemetry,
+      },
+
+      // Log Pipeline Processor Permissions
+      {
+        permission: Permission.CreateProjectLogPipelineProcessor,
+        title: "Create Log Pipeline Processor",
+        description:
+          "This permission can create Log Pipeline Processors in this project.",
+        isAssignableToTenant: true,
+        isAccessControlPermission: false,
+        group: PermissionGroup.Telemetry,
+      },
+      {
+        permission: Permission.DeleteProjectLogPipelineProcessor,
+        title: "Delete Log Pipeline Processor",
+        description:
+          "This permission can delete Log Pipeline Processors of this project.",
+        isAssignableToTenant: true,
+        isAccessControlPermission: false,
+        group: PermissionGroup.Telemetry,
+      },
+      {
+        permission: Permission.EditProjectLogPipelineProcessor,
+        title: "Edit Log Pipeline Processor",
+        description:
+          "This permission can edit Log Pipeline Processors of this project.",
+        isAssignableToTenant: true,
+        isAccessControlPermission: false,
+        group: PermissionGroup.Telemetry,
+      },
+      {
+        permission: Permission.ReadProjectLogPipelineProcessor,
+        title: "Read Log Pipeline Processor",
+        description:
+          "This permission can read Log Pipeline Processors of this project.",
+        isAssignableToTenant: true,
+        isAccessControlPermission: false,
+        group: PermissionGroup.Telemetry,
+      },
+
+      // Log Drop Filter Permissions
+      {
+        permission: Permission.CreateProjectLogDropFilter,
+        title: "Create Log Drop Filter",
+        description:
+          "This permission can create Log Drop Filters in this project.",
+        isAssignableToTenant: true,
+        isAccessControlPermission: false,
+        group: PermissionGroup.Telemetry,
+      },
+      {
+        permission: Permission.DeleteProjectLogDropFilter,
+        title: "Delete Log Drop Filter",
+        description:
+          "This permission can delete Log Drop Filters of this project.",
+        isAssignableToTenant: true,
+        isAccessControlPermission: false,
+        group: PermissionGroup.Telemetry,
+      },
+      {
+        permission: Permission.EditProjectLogDropFilter,
+        title: "Edit Log Drop Filter",
+        description:
+          "This permission can edit Log Drop Filters of this project.",
+        isAssignableToTenant: true,
+        isAccessControlPermission: false,
+        group: PermissionGroup.Telemetry,
+      },
+      {
+        permission: Permission.ReadProjectLogDropFilter,
+        title: "Read Log Drop Filter",
+        description:
+          "This permission can read Log Drop Filters of this project.",
+        isAssignableToTenant: true,
+        isAccessControlPermission: false,
+        group: PermissionGroup.Telemetry,
+      },
+
+      // Log Scrub Rule Permissions
+      {
+        permission: Permission.CreateProjectLogScrubRule,
+        title: "Create Log Scrub Rule",
+        description:
+          "This permission can create Log Scrub Rules in this project.",
+        isAssignableToTenant: true,
+        isAccessControlPermission: false,
+        group: PermissionGroup.Telemetry,
+      },
+      {
+        permission: Permission.DeleteProjectLogScrubRule,
+        title: "Delete Log Scrub Rule",
+        description:
+          "This permission can delete Log Scrub Rules of this project.",
+        isAssignableToTenant: true,
+        isAccessControlPermission: false,
+        group: PermissionGroup.Telemetry,
+      },
+      {
+        permission: Permission.EditProjectLogScrubRule,
+        title: "Edit Log Scrub Rule",
+        description:
+          "This permission can edit Log Scrub Rules of this project.",
+        isAssignableToTenant: true,
+        isAccessControlPermission: false,
+        group: PermissionGroup.Telemetry,
+      },
+      {
+        permission: Permission.ReadProjectLogScrubRule,
+        title: "Read Log Scrub Rule",
+        description:
+          "This permission can read Log Scrub Rules of this project.",
+        isAssignableToTenant: true,
+        isAccessControlPermission: false,
+        group: PermissionGroup.Telemetry,
+      },
+
       {
         permission: Permission.CreateTelemetryException,
         title: "Create Telemetry Service Exception",