@oneuptime/common 10.7.2 → 10.8.0

package/Server/Services/CloudResourceService.ts

@@ -0,0 +1,342 @@
+import DatabaseService from "./DatabaseService";
+import Model from "../../Models/DatabaseModels/CloudResource";
+import Label from "../../Models/DatabaseModels/Label";
+import CaptureSpan from "../Utils/Telemetry/CaptureSpan";
+import ObjectID from "../../Types/ObjectID";
+import QueryHelper from "../Types/Database/QueryHelper";
+import OneUptimeDate from "../../Types/Date";
+import LIMIT_MAX from "../../Types/Database/LimitMax";
+import GlobalCache from "../Infrastructure/GlobalCache";
+import logger, { LogAttributes } from "../Utils/Logger";
+import crypto from "crypto";
+import { OnCreate } from "../Types/Database/Hooks";
+import CloudResourceLabelRuleEngineService from "./CloudResourceLabelRuleEngineService";
+import CloudResourceOwnerRuleEngineService from "./CloudResourceOwnerRuleEngineService";
+
+const LAST_SEEN_CACHE_NAMESPACE: string = "cloud-resource-last-seen";
+const LAST_SEEN_THROTTLE_SECONDS: number = 60;
+
+const LABELS_APPLIED_CACHE_NAMESPACE: string = "cloud-resource-labels-applied";
+const LABELS_APPLIED_CACHE_TTL_SECONDS: number = 60;
+
+export class Service extends DatabaseService<Model> {
+  public constructor() {
+    super(Model);
+  }
+
+  @CaptureSpan()
+  protected override async onCreateSuccess(
+    _onCreate: OnCreate<Model>,
+    createdItem: Model,
+  ): Promise<Model> {
+    if (createdItem.projectId && createdItem.id) {
+      Promise.resolve()
+        .then(async () => {
+          await CloudResourceLabelRuleEngineService.applyRulesToCloudResource(
+            createdItem,
+          );
+        })
+        .then(async () => {
+          await CloudResourceOwnerRuleEngineService.applyRulesToCloudResource(
+            createdItem,
+          );
+        })
+        .catch((error: Error) => {
+          logger.error(
+            `Error applying cloud resource rules in CloudResourceService.onCreateSuccess: ${error}`,
+            {
+              projectId: createdItem.projectId?.toString(),
+              cloudResourceId: createdItem.id?.toString(),
+            } as LogAttributes,
+          );
+        });
+    }
+    return createdItem;
+  }
+
+  @CaptureSpan()
+  public async findOrCreateByResourceIdentifier(data: {
+    projectId: ObjectID;
+    resourceIdentifier: string;
+    name?: string | undefined;
+    cloudPlatform?: string | undefined;
+    cloudProvider?: string | undefined;
+    cloudRegion?: string | undefined;
+    cloudAccountId?: string | undefined;
+  }): Promise<Model> {
+    const existingResource: Model | null = await this.findOneBy({
+      query: {
+        projectId: data.projectId,
+        resourceIdentifier: QueryHelper.findWithSameText(
+          data.resourceIdentifier,
+        ),
+      },
+      select: {
+        _id: true,
+        projectId: true,
+        resourceIdentifier: true,
+      },
+      props: {
+        isRoot: true,
+      },
+    });
+
+    if (existingResource) {
+      return existingResource;
+    }
+
+    try {
+      const newResource: Model = new Model();
+      newResource.projectId = data.projectId;
+      newResource.name = data.name || data.resourceIdentifier;
+      newResource.resourceIdentifier = data.resourceIdentifier;
+      newResource.otelCollectorStatus = "connected";
+      newResource.lastSeenAt = OneUptimeDate.getCurrentDate();
+      if (data.cloudPlatform) {
+        newResource.cloudPlatform = data.cloudPlatform;
+      }
+      if (data.cloudProvider) {
+        newResource.cloudProvider = data.cloudProvider;
+      }
+      if (data.cloudRegion) {
+        newResource.cloudRegion = data.cloudRegion;
+      }
+      if (data.cloudAccountId) {
+        newResource.cloudAccountId = data.cloudAccountId;
+      }
+
+      const createdResource: Model = await this.create({
+        data: newResource,
+        props: {
+          isRoot: true,
+        },
+      });
+
+      return createdResource;
+    } catch {
+      const reFetchedResource: Model | null = await this.findOneBy({
+        query: {
+          projectId: data.projectId,
+          resourceIdentifier: QueryHelper.findWithSameText(
+            data.resourceIdentifier,
+          ),
+        },
+        select: {
+          _id: true,
+          projectId: true,
+          resourceIdentifier: true,
+        },
+        props: {
+          isRoot: true,
+        },
+      });
+
+      if (reFetchedResource) {
+        return reFetchedResource;
+      }
+
+      throw new Error(
+        "Failed to create or find cloud resource: " + data.resourceIdentifier,
+      );
+    }
+  }
+
+  @CaptureSpan()
+  public async updateLastSeen(
+    cloudResourceId: ObjectID,
+    extra?: {
+      agentVersion?: string | undefined;
+      cloudPlatform?: string | undefined;
+      cloudProvider?: string | undefined;
+      cloudRegion?: string | undefined;
+      cloudAccountId?: string | undefined;
+      runtimeName?: string | undefined;
+      runtimeVersion?: string | undefined;
+    },
+  ): Promise<void> {
+    const cacheKey: string = cloudResourceId.toString();
+    const extrasFingerprint: string = crypto
+      .createHash("sha1")
+      .update(
+        JSON.stringify({
+          agentVersion: extra?.agentVersion ?? null,
+          cloudPlatform: extra?.cloudPlatform ?? null,
+          cloudProvider: extra?.cloudProvider ?? null,
+          cloudRegion: extra?.cloudRegion ?? null,
+          cloudAccountId: extra?.cloudAccountId ?? null,
+          runtimeName: extra?.runtimeName ?? null,
+          runtimeVersion: extra?.runtimeVersion ?? null,
+        }),
+      )
+      .digest("hex");
+
+    const cached: string | null = await GlobalCache.getString(
+      LAST_SEEN_CACHE_NAMESPACE,
+      cacheKey,
+    );
+
+    if (cached === extrasFingerprint) {
+      return; // same data was written recently
+    }
+
+    await GlobalCache.setString(
+      LAST_SEEN_CACHE_NAMESPACE,
+      cacheKey,
+      extrasFingerprint,
+      { expiresInSeconds: LAST_SEEN_THROTTLE_SECONDS },
+    );
+
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    const data: any = {
+      lastSeenAt: OneUptimeDate.getCurrentDate(),
+      otelCollectorStatus: "connected",
+    };
+
+    if (extra?.agentVersion) {
+      data.agentVersion = extra.agentVersion;
+    }
+    if (extra?.cloudPlatform) {
+      data.cloudPlatform = extra.cloudPlatform;
+    }
+    if (extra?.cloudProvider) {
+      data.cloudProvider = extra.cloudProvider;
+    }
+    if (extra?.cloudRegion) {
+      data.cloudRegion = extra.cloudRegion;
+    }
+    if (extra?.cloudAccountId) {
+      data.cloudAccountId = extra.cloudAccountId;
+    }
+    if (extra?.runtimeName) {
+      data.runtimeName = extra.runtimeName;
+    }
+    if (extra?.runtimeVersion) {
+      data.runtimeVersion = extra.runtimeVersion;
+    }
+
+    await this.updateOneById({
+      id: cloudResourceId,
+      data: data,
+      props: {
+        isRoot: true,
+      },
+    });
+  }
+
+  @CaptureSpan()
+  public async attachLabels(data: {
+    cloudResourceId: ObjectID;
+    labelIds: Array<ObjectID>;
+  }): Promise<void> {
+    if (!data.labelIds || data.labelIds.length === 0) {
+      return;
+    }
+
+    const cacheKey: string = data.cloudResourceId.toString();
+    const fingerprint: string = fingerprintLabelIds(data.labelIds);
+    const cached: string | null = await GlobalCache.getString(
+      LABELS_APPLIED_CACHE_NAMESPACE,
+      cacheKey,
+    );
+    if (cached === fingerprint) {
+      return;
+    }
+
+    try {
+      const resourceIdStr: string = data.cloudResourceId.toString();
+      const existingLabels: Array<Label> = await this.getRepository()
+        .createQueryBuilder()
+        .relation(Model, "labels")
+        .of(resourceIdStr)
+        .loadMany();
+
+      const existingIds: Set<string> = new Set();
+      for (const lbl of existingLabels) {
+        const idStr: string | undefined = lbl._id?.toString();
+        if (idStr) {
+          existingIds.add(idStr);
+        }
+      }
+
+      const toAddIds: Array<string> = [];
+      const seen: Set<string> = new Set();
+      for (const id of data.labelIds) {
+        const idStr: string = id.toString();
+        if (existingIds.has(idStr) || seen.has(idStr)) {
+          continue;
+        }
+        seen.add(idStr);
+        toAddIds.push(idStr);
+      }
+
+      if (toAddIds.length > 0) {
+        await this.getRepository()
+          .createQueryBuilder()
+          .relation(Model, "labels")
+          .of(resourceIdStr)
+          .add(toAddIds);
+      }
+
+      await GlobalCache.setString(
+        LABELS_APPLIED_CACHE_NAMESPACE,
+        cacheKey,
+        fingerprint,
+        { expiresInSeconds: LABELS_APPLIED_CACHE_TTL_SECONDS },
+      );
+    } catch (err) {
+      logger.warn(
+        `CloudResourceService.attachLabels failed for resource ${data.cloudResourceId.toString()}: ${
+          err instanceof Error ? err.message : String(err)
+        }`,
+      );
+    }
+  }
+
+  @CaptureSpan()
+  public async markDisconnectedResources(): Promise<void> {
+    const fiveMinutesAgo: Date = OneUptimeDate.addRemoveMinutes(
+      OneUptimeDate.getCurrentDate(),
+      -5,
+    );
+
+    const connectedResources: Array<Model> = await this.findBy({
+      query: {
+        otelCollectorStatus: "connected",
+        lastSeenAt: QueryHelper.lessThan(fiveMinutesAgo),
+      },
+      select: {
+        _id: true,
+      },
+      limit: LIMIT_MAX,
+      skip: 0,
+      props: {
+        isRoot: true,
+      },
+    });
+
+    for (const cloudResource of connectedResources) {
+      if (cloudResource._id) {
+        await this.updateOneById({
+          id: new ObjectID(cloudResource._id.toString()),
+          data: {
+            otelCollectorStatus: "disconnected",
+          },
+          props: {
+            isRoot: true,
+          },
+        });
+      }
+    }
+  }
+}
+
+function fingerprintLabelIds(labelIds: Array<ObjectID>): string {
+  const sorted: Array<string> = labelIds
+    .map((id: ObjectID) => {
+      return id.toString();
+    })
+    .sort();
+  return crypto.createHash("sha1").update(sorted.join(",")).digest("hex");
+}
+
+export default new Service();

package/Server/Services/ExceptionAggregationService.ts

@@ -72,6 +72,9 @@ export class ExceptionAggregationService {
       ["hostId", ServiceType.Host],
       ["dockerHostId", ServiceType.DockerHost],
       ["kubernetesClusterId", ServiceType.KubernetesCluster],
+      ["serverlessFunctionId", ServiceType.ServerlessFunction],
+      ["cloudResourceId", ServiceType.CloudResource],
+      ["rumApplicationId", ServiceType.RealUserMonitor],
     ]);
   private static readonly ATTRIBUTE_KEY_PATTERN: RegExp = /^[a-zA-Z0-9._:/-]+$/;
   private static readonly MAX_FACET_KEY_LENGTH: number = 256;

package/Server/Services/HostService.ts

@@ -195,6 +195,13 @@ export class Service extends DatabaseService<Model> {
       dockerHostId?: ObjectID | undefined;
       kubernetesClusterId?: ObjectID | undefined;
       agentVersion?: string | undefined;
+      deploymentEnvironment?: string | undefined;
+      runtimeName?: string | undefined;
+      runtimeVersion?: string | undefined;
+      cloudProvider?: string | undefined;
+      cloudPlatform?: string | undefined;
+      cloudRegion?: string | undefined;
+      cloudAccountId?: string | undefined;
     },
   ): Promise<void> {
     /*
@@ -267,6 +274,27 @@ export class Service extends DatabaseService<Model> {
     if (extra?.agentVersion) {
       data.agentVersion = extra.agentVersion;
     }
+    if (extra?.deploymentEnvironment) {
+      data.deploymentEnvironment = extra.deploymentEnvironment;
+    }
+    if (extra?.runtimeName) {
+      data.runtimeName = extra.runtimeName;
+    }
+    if (extra?.runtimeVersion) {
+      data.runtimeVersion = extra.runtimeVersion;
+    }
+    if (extra?.cloudProvider) {
+      data.cloudProvider = extra.cloudProvider;
+    }
+    if (extra?.cloudPlatform) {
+      data.cloudPlatform = extra.cloudPlatform;
+    }
+    if (extra?.cloudRegion) {
+      data.cloudRegion = extra.cloudRegion;
+    }
+    if (extra?.cloudAccountId) {
+      data.cloudAccountId = extra.cloudAccountId;
+    }
 
     await this.updateOneById({
       id: hostId,
@@ -291,6 +319,13 @@ export class Service extends DatabaseService<Model> {
     dockerHostId?: ObjectID | undefined;
     kubernetesClusterId?: ObjectID | undefined;
     agentVersion?: string | undefined;
+    deploymentEnvironment?: string | undefined;
+    runtimeName?: string | undefined;
+    runtimeVersion?: string | undefined;
+    cloudProvider?: string | undefined;
+    cloudPlatform?: string | undefined;
+    cloudRegion?: string | undefined;
+    cloudAccountId?: string | undefined;
   }): string {
     const normalized: Record<string, string | number | null> = {
       osType: extra?.osType ?? null,
@@ -306,6 +341,13 @@ export class Service extends DatabaseService<Model> {
       dockerHostId: extra?.dockerHostId?.toString() ?? null,
       kubernetesClusterId: extra?.kubernetesClusterId?.toString() ?? null,
       agentVersion: extra?.agentVersion ?? null,
+      deploymentEnvironment: extra?.deploymentEnvironment ?? null,
+      runtimeName: extra?.runtimeName ?? null,
+      runtimeVersion: extra?.runtimeVersion ?? null,
+      cloudProvider: extra?.cloudProvider ?? null,
+      cloudPlatform: extra?.cloudPlatform ?? null,
+      cloudRegion: extra?.cloudRegion ?? null,
+      cloudAccountId: extra?.cloudAccountId ?? null,
     };
 
     return crypto

package/Server/Services/LogAggregationService.ts

@@ -106,6 +106,9 @@ export class LogAggregationService {
       ["hostId", ServiceType.Host],
       ["dockerHostId", ServiceType.DockerHost],
       ["kubernetesClusterId", ServiceType.KubernetesCluster],
+      ["serverlessFunctionId", ServiceType.ServerlessFunction],
+      ["cloudResourceId", ServiceType.CloudResource],
+      ["rumApplicationId", ServiceType.RealUserMonitor],
     ]);
   private static readonly ATTRIBUTE_KEY_PATTERN: RegExp = /^[a-zA-Z0-9._:/-]+$/;
   private static readonly MAX_FACET_KEY_LENGTH: number = 256;

package/Server/Services/MetricAggregationService.ts

@@ -48,6 +48,9 @@ export class MetricAggregationService {
       ["hostId", ServiceType.Host],
       ["dockerHostId", ServiceType.DockerHost],
       ["kubernetesClusterId", ServiceType.KubernetesCluster],
+      ["serverlessFunctionId", ServiceType.ServerlessFunction],
+      ["cloudResourceId", ServiceType.CloudResource],
+      ["rumApplicationId", ServiceType.RealUserMonitor],
     ]);
   private static readonly ATTRIBUTE_KEY_PATTERN: RegExp = /^[a-zA-Z0-9._:/-]+$/;
   private static readonly MAX_FACET_KEY_LENGTH: number = 256;

package/Server/Services/OpenTelemetryIngestService.ts

@@ -21,9 +21,15 @@ import ServiceType from "../../Types/Telemetry/ServiceType";
 import Host from "../../Models/DatabaseModels/Host";
 import DockerHost from "../../Models/DatabaseModels/DockerHost";
 import KubernetesCluster from "../../Models/DatabaseModels/KubernetesCluster";
+import ServerlessFunction from "../../Models/DatabaseModels/ServerlessFunction";
+import CloudResource from "../../Models/DatabaseModels/CloudResource";
+import RumApplication from "../../Models/DatabaseModels/RumApplication";
 import HostService from "./HostService";
 import DockerHostService from "./DockerHostService";
 import KubernetesClusterService from "./KubernetesClusterService";
+import ServerlessFunctionService from "./ServerlessFunctionService";
+import CloudResourceService from "./CloudResourceService";
+import RumApplicationService from "./RumApplicationService";
 import GlobalCache from "../Infrastructure/GlobalCache";
 
 export enum OtelAggregationTemporality {
@@ -85,6 +91,93 @@ const projectRetentionInProcessCache: Map<string, CachedRetentionContext> =
   new Map();
 
 export default class OTelIngestService {
+  /*
+   * Read a single string-valued OTel resource attribute out of the raw
+   * OTLP attribute array (shape: [{ key, value: { stringValue } }]).
+   * Mirrors OtelIngestBaseService.getStringAttribute — duplicated here
+   * because that one is on the ingest-side base class and this read path
+   * (Service resolution) lives in Common.
+   */
+  private static getResourceStringAttribute(
+    attributes: JSONArray,
+    key: string,
+  ): string | null {
+    for (const attribute of attributes) {
+      if (
+        attribute["key"] === key &&
+        attribute["value"] &&
+        (attribute["value"] as JSONObject)["stringValue"]
+      ) {
+        const value: unknown = (attribute["value"] as JSONObject)["stringValue"];
+        if (typeof value === "string" && value.trim()) {
+          return value.trim();
+        }
+      }
+    }
+    return null;
+  }
+
+  /*
+   * Build the metadata captured onto the Service row from OTel resource
+   * attributes (service.version, deployment.environment, cloud.*, runtime.*).
+   * Returns undefined when no resource attributes are available so the
+   * caller falls back to a plain lastSeenAt touch.
+   */
+  private static buildServiceMetadataFromAttributes(
+    attributes: JSONArray | undefined,
+  ):
+    | {
+        serviceVersion?: string | undefined;
+        deploymentEnvironment?: string | undefined;
+        serviceNamespace?: string | undefined;
+        runtimeName?: string | undefined;
+        runtimeVersion?: string | undefined;
+        cloudProvider?: string | undefined;
+        cloudPlatform?: string | undefined;
+        cloudRegion?: string | undefined;
+        cloudAccountId?: string | undefined;
+      }
+    | undefined {
+    if (!attributes) {
+      return undefined;
+    }
+    return {
+      serviceVersion:
+        this.getResourceStringAttribute(attributes, "service.version") ||
+        undefined,
+      deploymentEnvironment:
+        this.getResourceStringAttribute(
+          attributes,
+          "deployment.environment.name",
+        ) ||
+        this.getResourceStringAttribute(attributes, "deployment.environment") ||
+        undefined,
+      serviceNamespace:
+        this.getResourceStringAttribute(attributes, "service.namespace") ||
+        undefined,
+      runtimeName:
+        this.getResourceStringAttribute(attributes, "process.runtime.name") ||
+        undefined,
+      runtimeVersion:
+        this.getResourceStringAttribute(
+          attributes,
+          "process.runtime.version",
+        ) || undefined,
+      cloudProvider:
+        this.getResourceStringAttribute(attributes, "cloud.provider") ||
+        undefined,
+      cloudPlatform:
+        this.getResourceStringAttribute(attributes, "cloud.platform") ||
+        undefined,
+      cloudRegion:
+        this.getResourceStringAttribute(attributes, "cloud.region") ||
+        undefined,
+      cloudAccountId:
+        this.getResourceStringAttribute(attributes, "cloud.account.id") ||
+        undefined,
+    };
+  }
+
   @CaptureSpan()
   private static async getProjectRetentionContext(
     projectId: ObjectID,
@@ -194,7 +287,10 @@ export default class OTelIngestService {
      * one in-memory cache lookup per batch.
      */
     try {
-      await ServiceService.updateLastSeen(result.serviceId);
+      await ServiceService.updateLastSeen(
+        result.serviceId,
+        this.buildServiceMetadataFromAttributes(data.resourceAttributes),
+      );
     } catch (err) {
       logger.warn(
         `telemetryServiceFromName lastSeen update failed for "${data.serviceName}": ${
@@ -453,6 +549,57 @@ export default class OTelIngestService {
           telemetryRetentionConfig: cluster?.telemetryRetentionConfig ?? null,
         };
       }
+      if (serviceType === ServiceType.ServerlessFunction) {
+        const serverlessFunction: ServerlessFunction | null =
+          await ServerlessFunctionService.findOneById({
+            id: resourceId,
+            select: {
+              retainTelemetryDataForDays: true,
+              telemetryRetentionConfig: true,
+            },
+            props: { isRoot: true },
+          });
+        return {
+          retainTelemetryDataForDays:
+            serverlessFunction?.retainTelemetryDataForDays ?? null,
+          telemetryRetentionConfig:
+            serverlessFunction?.telemetryRetentionConfig ?? null,
+        };
+      }
+      if (serviceType === ServiceType.CloudResource) {
+        const cloudResource: CloudResource | null =
+          await CloudResourceService.findOneById({
+            id: resourceId,
+            select: {
+              retainTelemetryDataForDays: true,
+              telemetryRetentionConfig: true,
+            },
+            props: { isRoot: true },
+          });
+        return {
+          retainTelemetryDataForDays:
+            cloudResource?.retainTelemetryDataForDays ?? null,
+          telemetryRetentionConfig:
+            cloudResource?.telemetryRetentionConfig ?? null,
+        };
+      }
+      if (serviceType === ServiceType.RealUserMonitor) {
+        const rumApplication: RumApplication | null =
+          await RumApplicationService.findOneById({
+            id: resourceId,
+            select: {
+              retainTelemetryDataForDays: true,
+              telemetryRetentionConfig: true,
+            },
+            props: { isRoot: true },
+          });
+        return {
+          retainTelemetryDataForDays:
+            rumApplication?.retainTelemetryDataForDays ?? null,
+          telemetryRetentionConfig:
+            rumApplication?.telemetryRetentionConfig ?? null,
+        };
+      }
     } catch (err) {
       logger.warn(
         `Per-resource retention lookup failed for ${serviceType} ${resourceId.toString()}: ${

package/Server/Services/RumApplicationClientService.ts

@@ -0,0 +1,69 @@
+import DatabaseService from "./DatabaseService";
+import Model from "../../Models/DatabaseModels/RumApplicationClient";
+import ObjectID from "../../Types/ObjectID";
+import OneUptimeDate from "../../Types/Date";
+import CaptureSpan from "../Utils/Telemetry/CaptureSpan";
+import logger from "../Utils/Logger";
+
+export class Service extends DatabaseService<Model> {
+  public constructor() {
+    super(Model);
+  }
+
+  /**
+   * Upsert a RUM client platform (browser.platform / device.model) from
+   * ingest. Refreshes lastSeenAt if it already exists.
+   */
+  @CaptureSpan()
+  public async recordClient(data: {
+    projectId: ObjectID;
+    rumApplicationId: ObjectID;
+    clientName: string;
+    clientType?: string | undefined;
+  }): Promise<void> {
+    try {
+      const existing: Model | null = await this.findOneBy({
+        query: {
+          projectId: data.projectId,
+          rumApplicationId: data.rumApplicationId,
+          clientName: data.clientName,
+        },
+        select: { _id: true },
+        props: { isRoot: true },
+      });
+
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+      const fields: any = { lastSeenAt: OneUptimeDate.getCurrentDate() };
+      if (data.clientType) {
+        fields.clientType = data.clientType;
+      }
+
+      if (existing && existing._id) {
+        await this.updateOneById({
+          id: new ObjectID(existing._id.toString()),
+          data: fields,
+          props: { isRoot: true },
+        });
+        return;
+      }
+
+      const item: Model = new Model();
+      item.projectId = data.projectId;
+      item.rumApplicationId = data.rumApplicationId;
+      item.clientName = data.clientName;
+      item.lastSeenAt = OneUptimeDate.getCurrentDate();
+      if (data.clientType) {
+        item.clientType = data.clientType;
+      }
+      await this.create({ data: item, props: { isRoot: true } });
+    } catch (err) {
+      logger.warn(
+        `RumApplicationClientService.recordClient failed: ${
+          err instanceof Error ? err.message : String(err)
+        }`,
+      );
+    }
+  }
+}
+
+export default new Service();