@oneuptime/common 10.4.5 → 10.4.6

package/Models/DatabaseModels/Project.ts

@@ -1,4 +1,5 @@
 import MetricDownsamplingRetentionDays from "../../Types/Metrics/MetricDownsamplingRetentionDays";
+import TelemetryRetentionConfig from "../../Types/Telemetry/TelemetryRetentionConfig";
 import Reseller from "./Reseller";
 import ResellerPlan from "./ResellerPlan";
 import User from "./User";
@@ -2028,6 +2029,30 @@ export default class Project extends TenantModel {
   })
   public defaultTelemetryRetentionInDays?: number = undefined;
 
+  @ColumnAccessControl({
+    create: [],
+    read: [
+      Permission.ProjectOwner,
+      Permission.ProjectAdmin,
+      Permission.ProjectMember,
+      Permission.Viewer,
+      Permission.ReadProject,
+    ],
+    update: [Permission.ProjectOwner, Permission.ProjectAdmin],
+  })
+  @TableColumn({
+    type: TableColumnType.JSON,
+    required: false,
+    title: "Telemetry Data Retention Overrides",
+    description:
+      "Project-wide per-pillar retention overrides for telemetry data (logs by severity, traces by status, metrics, profiles). Falls back to defaultTelemetryRetentionInDays when a pillar or bucket is not set.",
+  })
+  @Column({
+    type: ColumnType.JSON,
+    nullable: true,
+  })
+  public telemetryRetentionConfig?: TelemetryRetentionConfig = undefined;
+
   @ColumnAccessControl({
     create: [],
     read: [

package/Models/DatabaseModels/Service.ts

@@ -25,6 +25,7 @@ import ObjectID from "../../Types/ObjectID";
 import Permission from "../../Types/Permission";
 import TechStack from "../../Types/Service/TechStack";
 import MetricDownsamplingRetentionDays from "../../Types/Metrics/MetricDownsamplingRetentionDays";
+import TelemetryRetentionConfig from "../../Types/Telemetry/TelemetryRetentionConfig";
 import {
   Column,
   Entity,
@@ -795,6 +796,47 @@ export default class Service extends BaseModel {
   public metricDownsamplingRetentionDays?: MetricDownsamplingRetentionDays =
     undefined;
 
+  @ColumnAccessControl({
+    create: [
+      Permission.ProjectOwner,
+      Permission.ProjectAdmin,
+      Permission.ProjectMember,
+      Permission.SettingsAdmin,
+      Permission.SettingsMember,
+      Permission.CreateService,
+    ],
+    read: [
+      Permission.ProjectOwner,
+      Permission.ProjectAdmin,
+      Permission.ProjectMember,
+      Permission.Viewer,
+      Permission.SettingsAdmin,
+      Permission.SettingsMember,
+      Permission.SettingsViewer,
+      Permission.ReadService,
+    ],
+    update: [
+      Permission.ProjectOwner,
+      Permission.ProjectAdmin,
+      Permission.ProjectMember,
+      Permission.SettingsAdmin,
+      Permission.SettingsMember,
+      Permission.EditService,
+    ],
+  })
+  @TableColumn({
+    type: TableColumnType.JSON,
+    required: false,
+    title: "Telemetry Data Retention Overrides",
+    description:
+      "Per-pillar retention overrides for this service (logs by severity, traces by status, metrics, profiles). Unset fields inherit the project-level config and umbrella default.",
+  })
+  @Column({
+    type: ColumnType.JSON,
+    nullable: true,
+  })
+  public telemetryRetentionConfig?: TelemetryRetentionConfig = undefined;
+
   @ColumnAccessControl({
     create: [],
     read: [

package/Server/Infrastructure/Postgres/SchemaMigrations/1779199346010-AddTelemetryRetentionConfig.ts

@@ -0,0 +1,25 @@
+import { MigrationInterface, QueryRunner } from "typeorm";
+
+export class AddTelemetryRetentionConfig1779199346010
+  implements MigrationInterface
+{
+  name = "AddTelemetryRetentionConfig1779199346010";
+
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(
+      `ALTER TABLE "Project" ADD "telemetryRetentionConfig" jsonb`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "Service" ADD "telemetryRetentionConfig" jsonb`,
+    );
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(
+      `ALTER TABLE "Service" DROP COLUMN "telemetryRetentionConfig"`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "Project" DROP COLUMN "telemetryRetentionConfig"`,
+    );
+  }
+}

package/Server/Infrastructure/Postgres/SchemaMigrations/Index.ts

@@ -337,6 +337,7 @@ import { MigrateLegacyManagerRolesToAdmin1778900000000 } from "./1778900000000-M
 import { AddOnCallDutyPolicyScheduleOwners1778929624633 } from "./1778929624633-AddOnCallDutyPolicyScheduleOwners";
 import { AddOnCallIncomingCallOwnersAndRules1778931537020 } from "./1778931537020-AddOnCallIncomingCallOwnersAndRules";
 import { IncreaseSmtpUsernameLength1779125489830 } from "./1779125489830-IncreaseSmtpUsernameLength";
+import { AddTelemetryRetentionConfig1779199346010 } from "./1779199346010-AddTelemetryRetentionConfig";
 export default [
   InitialMigration,
   MigrationName1717678334852,
@@ -677,4 +678,5 @@ export default [
   AddOnCallDutyPolicyScheduleOwners1778929624633,
   AddOnCallIncomingCallOwnersAndRules1778931537020,
   IncreaseSmtpUsernameLength1779125489830,
+  AddTelemetryRetentionConfig1779199346010,
 ];

package/Server/Services/OpenTelemetryIngestService.ts

@@ -15,6 +15,7 @@ import { extractOneuptimeLabelNames } from "../Utils/Telemetry/OneuptimeLabel";
 import CaptureSpan from "../Utils/Telemetry/CaptureSpan";
 import SortOrder from "../../Types/BaseDatabase/SortOrder";
 import logger from "../Utils/Logger";
+import TelemetryRetentionConfig from "../../Types/Telemetry/TelemetryRetentionConfig";
 
 export enum OtelAggregationTemporality {
   Cumulative = "AGGREGATION_TEMPORALITY_CUMULATIVE",
@@ -25,26 +26,38 @@ export interface TelemetryServiceMetadata {
   serviceName: string;
   serviceId: ObjectID;
   dataRententionInDays: number;
+  serviceRetentionConfig: TelemetryRetentionConfig | null;
+  serviceUmbrellaInDays: number | null;
+  projectRetentionConfig: TelemetryRetentionConfig | null;
+  projectUmbrellaInDays: number;
+}
+
+interface ProjectRetentionContext {
+  projectRetentionConfig: TelemetryRetentionConfig | null;
+  projectUmbrellaInDays: number;
 }
 
 export default class OTelIngestService {
   @CaptureSpan()
-  private static async getProjectDefaultRetentionInDays(
+  private static async getProjectRetentionContext(
     projectId: ObjectID,
-  ): Promise<number> {
+  ): Promise<ProjectRetentionContext> {
     const project: Project | null = await ProjectService.findOneById({
       id: projectId,
       select: {
         defaultTelemetryRetentionInDays: true,
+        telemetryRetentionConfig: true,
       },
       props: {
         isRoot: true,
       },
     });
 
-    return (
-      project?.defaultTelemetryRetentionInDays || DEFAULT_RETENTION_IN_DAYS
-    );
+    return {
+      projectRetentionConfig: project?.telemetryRetentionConfig ?? null,
+      projectUmbrellaInDays:
+        project?.defaultTelemetryRetentionInDays || DEFAULT_RETENTION_IN_DAYS,
+    };
   }
 
   @CaptureSpan()
@@ -52,17 +65,12 @@ export default class OTelIngestService {
     serviceName: string;
     projectId: ObjectID;
     resourceAttributes?: JSONArray | undefined;
-  }): Promise<{
-    serviceId: ObjectID;
-    dataRententionInDays: number;
-  }> {
-    const result: {
-      serviceId: ObjectID;
-      dataRententionInDays: number;
-    } = await this.findOrCreateTelemetryService({
-      serviceName: data.serviceName,
-      projectId: data.projectId,
-    });
+  }): Promise<TelemetryServiceMetadata> {
+    const result: TelemetryServiceMetadata =
+      await this.findOrCreateTelemetryService({
+        serviceName: data.serviceName,
+        projectId: data.projectId,
+      });
 
     /*
      * Touch `lastSeenAt` on the service. Throttled per-service inside
@@ -119,10 +127,7 @@ export default class OTelIngestService {
   private static async findOrCreateTelemetryService(data: {
     serviceName: string;
     projectId: ObjectID;
-  }): Promise<{
-    serviceId: ObjectID;
-    dataRententionInDays: number;
-  }> {
+  }): Promise<TelemetryServiceMetadata> {
     /*
      * Sort by createdAt ASC for deterministic resolution if the
      * DB ever ends up with duplicates (defense in depth — the
@@ -140,6 +145,7 @@ export default class OTelIngestService {
       select: {
         _id: true,
         retainTelemetryDataForDays: true,
+        telemetryRetentionConfig: true,
       },
       sort: {
         createdAt: SortOrder.Ascending,
@@ -149,10 +155,27 @@ export default class OTelIngestService {
       },
     });
 
-    if (!service) {
-      const projectDefaultRetention: number =
-        await this.getProjectDefaultRetentionInDays(data.projectId);
+    const projectContext: ProjectRetentionContext =
+      await this.getProjectRetentionContext(data.projectId);
 
+    const buildMetadata: (svc: Service) => TelemetryServiceMetadata = (
+      svc: Service,
+    ): TelemetryServiceMetadata => {
+      const serviceUmbrella: number | null =
+        svc.retainTelemetryDataForDays ?? null;
+      return {
+        serviceName: data.serviceName,
+        serviceId: svc.id!,
+        dataRententionInDays:
+          serviceUmbrella || projectContext.projectUmbrellaInDays,
+        serviceRetentionConfig: svc.telemetryRetentionConfig ?? null,
+        serviceUmbrellaInDays: serviceUmbrella,
+        projectRetentionConfig: projectContext.projectRetentionConfig,
+        projectUmbrellaInDays: projectContext.projectUmbrellaInDays,
+      };
+    };
+
+    if (!service) {
       try {
         const newService: Service = new Service();
         newService.projectId = data.projectId;
@@ -166,10 +189,7 @@ export default class OTelIngestService {
           },
         });
 
-        return {
-          serviceId: createdService.id!,
-          dataRententionInDays: projectDefaultRetention,
-        };
+        return buildMetadata(createdService);
       } catch {
         /*
          * Race condition: another request created the service concurrently.
@@ -183,6 +203,7 @@ export default class OTelIngestService {
           select: {
             _id: true,
             retainTelemetryDataForDays: true,
+            telemetryRetentionConfig: true,
           },
           sort: {
             createdAt: SortOrder.Ascending,
@@ -193,12 +214,7 @@ export default class OTelIngestService {
         });
 
         if (existingService) {
-          return {
-            serviceId: existingService.id!,
-            dataRententionInDays:
-              existingService.retainTelemetryDataForDays ||
-              projectDefaultRetention,
-          };
+          return buildMetadata(existingService);
         }
 
         throw new Error(
@@ -207,19 +223,7 @@ export default class OTelIngestService {
       }
     }
 
-    if (service.retainTelemetryDataForDays) {
-      return {
-        serviceId: service.id!,
-        dataRententionInDays: service.retainTelemetryDataForDays,
-      };
-    }
-
-    return {
-      serviceId: service.id!,
-      dataRententionInDays: await this.getProjectDefaultRetentionInDays(
-        data.projectId,
-      ),
-    };
+    return buildMetadata(service);
   }
   @CaptureSpan()
   public static getMetricFromDatapoint(data: {

package/Types/Telemetry/TelemetryRetentionConfig.ts

@@ -0,0 +1,105 @@
+import LogSeverity from "../Log/LogSeverity";
+import { SpanStatus } from "../../Models/AnalyticsModels/Span";
+
+/*
+ * Per-pillar telemetry retention overrides.
+ *
+ * Lives on Project (project-wide defaults) and Service (per-service overrides).
+ * When a row is ingested, retention falls through, narrowest-first:
+ *   1. service[pillar].byX[bucketKey]
+ *   2. service[pillar].default
+ *   3. service.retainTelemetryDataForDays               (umbrella)
+ *   4. project[pillar].byX[bucketKey]
+ *   5. project[pillar].default
+ *   6. project.defaultTelemetryRetentionInDays           (umbrella)
+ *   7. HARDCODED_DEFAULT_TELEMETRY_RETENTION_IN_DAYS
+ */
+export default interface TelemetryRetentionConfig {
+  logs?: {
+    default?: number | null;
+    bySeverity?: Partial<Record<LogSeverity, number | null>>;
+  };
+  traces?: {
+    default?: number | null;
+    byStatus?: Partial<Record<SpanStatus, number | null>>;
+  };
+  metrics?: {
+    default?: number | null;
+  };
+  profiles?: {
+    default?: number | null;
+  };
+}
+
+export type TelemetryPillar = "logs" | "traces" | "metrics" | "profiles";
+
+export const HARDCODED_DEFAULT_TELEMETRY_RETENTION_IN_DAYS: number = 15;
+
+function pickPositive(value: number | null | undefined): number | null {
+  return typeof value === "number" && value > 0 ? value : null;
+}
+
+function getBucketValue(
+  config: TelemetryRetentionConfig | null | undefined,
+  pillar: TelemetryPillar,
+  bucketKey: LogSeverity | SpanStatus | null | undefined,
+): number | null {
+  if (!config || bucketKey === undefined || bucketKey === null) {
+    return null;
+  }
+  if (pillar === "logs") {
+    const bySeverity: Partial<Record<LogSeverity, number | null>> | undefined =
+      config.logs?.bySeverity;
+    return pickPositive(bySeverity?.[bucketKey as LogSeverity]);
+  }
+  if (pillar === "traces") {
+    const byStatus: Partial<Record<SpanStatus, number | null>> | undefined =
+      config.traces?.byStatus;
+    return pickPositive(byStatus?.[bucketKey as SpanStatus]);
+  }
+  return null;
+}
+
+function getPillarDefault(
+  config: TelemetryRetentionConfig | null | undefined,
+  pillar: TelemetryPillar,
+): number | null {
+  if (!config) {
+    return null;
+  }
+  return pickPositive(config[pillar]?.default);
+}
+
+export function resolveTelemetryRetentionInDays(input: {
+  pillar: TelemetryPillar;
+  bucketKey?: LogSeverity | SpanStatus | null;
+  serviceConfig?: TelemetryRetentionConfig | null;
+  serviceUmbrellaInDays?: number | null;
+  projectConfig?: TelemetryRetentionConfig | null;
+  projectUmbrellaInDays?: number | null;
+}): number {
+  const {
+    pillar,
+    bucketKey,
+    serviceConfig,
+    serviceUmbrellaInDays,
+    projectConfig,
+    projectUmbrellaInDays,
+  } = input;
+
+  const candidates: Array<number | null> = [
+    getBucketValue(serviceConfig, pillar, bucketKey),
+    getPillarDefault(serviceConfig, pillar),
+    pickPositive(serviceUmbrellaInDays),
+    getBucketValue(projectConfig, pillar, bucketKey),
+    getPillarDefault(projectConfig, pillar),
+    pickPositive(projectUmbrellaInDays),
+  ];
+
+  for (const value of candidates) {
+    if (value !== null) {
+      return value;
+    }
+  }
+  return HARDCODED_DEFAULT_TELEMETRY_RETENTION_IN_DAYS;
+}