@oneuptime/common 8.0.5387 → 8.0.5409

package/Server/Services/TelemetryAttributeService.ts

@@ -1,13 +1,66 @@
+import { SQL, Statement } from "../Utils/AnalyticsDatabase/Statement";
 import TelemetryType from "../../Types/Telemetry/TelemetryType";
-import ClickhouseDatabase from "../Infrastructure/ClickhouseDatabase";
-import AnalyticsDatabaseService from "./AnalyticsDatabaseService";
-import TelemetryAttribute from "../../Models/AnalyticsModels/TelemetryAttribute";
+import LogDatabaseService from "./LogService";
+import MetricDatabaseService from "./MetricService";
+import SpanDatabaseService from "./SpanService";
+import TableColumnType from "../../Types/AnalyticsDatabase/TableColumnType";
+import { JSONObject } from "../../Types/JSON";
 import ObjectID from "../../Types/ObjectID";
+import OneUptimeDate from "../../Types/Date";
+import GlobalCache from "../Infrastructure/GlobalCache";
 import CaptureSpan from "../Utils/Telemetry/CaptureSpan";
+import AnalyticsDatabaseService, {
+  DbJSONResponse,
+  Results,
+} from "./AnalyticsDatabaseService";
 
-export class TelemetryAttributeService extends AnalyticsDatabaseService<TelemetryAttribute> {
-  public constructor(clickhouseDatabase?: ClickhouseDatabase | undefined) {
-    super({ modelType: TelemetryAttribute, database: clickhouseDatabase });
+type TelemetrySource = {
+  service: AnalyticsDatabaseService<any>;
+  tableName: string;
+  attributesColumn: string;
+  timeColumn: string;
+};
+
+type TelemetryAttributesCacheEntry = {
+  attributes: Array<string>;
+  refreshedAt: Date;
+};
+
+export class TelemetryAttributeService {
+  private static readonly ATTRIBUTES_LIMIT: number = 5000;
+  private static readonly ROW_SCAN_LIMIT: number = 10000;
+  private static readonly CACHE_NAMESPACE: string = "telemetry-attributes";
+  private static readonly CACHE_STALE_AFTER_MINUTES: number = 5;
+  private static readonly LOOKBACK_WINDOW_IN_DAYS: number = 30;
+
+  private getTelemetrySource(
+    telemetryType: TelemetryType,
+  ): TelemetrySource | null {
+    switch (telemetryType) {
+      case TelemetryType.Log:
+        return {
+          service: LogDatabaseService,
+          tableName: LogDatabaseService.model.tableName,
+          attributesColumn: "attributes",
+          timeColumn: "time",
+        };
+      case TelemetryType.Metric:
+        return {
+          service: MetricDatabaseService,
+          tableName: MetricDatabaseService.model.tableName,
+          attributesColumn: "attributes",
+          timeColumn: "time",
+        };
+      case TelemetryType.Trace:
+        return {
+          service: SpanDatabaseService,
+          tableName: SpanDatabaseService.model.tableName,
+          attributesColumn: "attributes",
+          timeColumn: "startTime",
+        };
+      default:
+        return null;
+    }
   }
 
   @CaptureSpan()
@@ -15,57 +68,217 @@ export class TelemetryAttributeService extends AnalyticsDatabaseService<Telemetr
     projectId: ObjectID;
     telemetryType: TelemetryType;
   }): Promise<string[]> {
-    const telemetryAttribute: TelemetryAttribute | null = await this.findOneBy({
-      query: {
+    const source: TelemetrySource | null = this.getTelemetrySource(
+      data.telemetryType,
+    );
+
+    if (!source) {
+      return [];
+    }
+
+    const cacheKey: string = TelemetryAttributeService.getCacheKey(
+      data.projectId,
+      data.telemetryType,
+    );
+
+    const cachedEntry: TelemetryAttributesCacheEntry | null =
+      await TelemetryAttributeService.getCachedAttributes(cacheKey);
+
+    if (cachedEntry && TelemetryAttributeService.isCacheFresh(cachedEntry)) {
+      return cachedEntry.attributes;
+    }
+
+    let attributes: Array<string> = [];
+
+    try {
+      attributes = await TelemetryAttributeService.fetchAttributesFromDatabase({
         projectId: data.projectId,
-        telemetryType: data.telemetryType,
-      },
-      select: {
-        attributes: true,
-      },
-      props: {
-        isRoot: true,
+        source,
+      });
+    } catch (error) {
+      if (cachedEntry) {
+        return cachedEntry.attributes;
+      }
+
+      throw error;
+    }
+
+    await TelemetryAttributeService.storeAttributesInCache(
+      cacheKey,
+      attributes,
+    );
+
+    if (attributes.length === 0 && cachedEntry) {
+      return cachedEntry.attributes;
+    }
+
+    return attributes;
+  }
+
+  private static getCacheKey(
+    projectId: ObjectID,
+    telemetryType: TelemetryType,
+  ): string {
+    return `${projectId.toString()}:${telemetryType}`;
+  }
+
+  private static getLookbackStartDate(): Date {
+    return OneUptimeDate.addRemoveDays(
+      OneUptimeDate.getCurrentDate(),
+      -TelemetryAttributeService.LOOKBACK_WINDOW_IN_DAYS,
+    );
+  }
+
+  private static async getCachedAttributes(
+    cacheKey: string,
+  ): Promise<TelemetryAttributesCacheEntry | null> {
+    let payload: JSONObject | null = null;
+
+    try {
+      payload = await GlobalCache.getJSONObject(
+        TelemetryAttributeService.CACHE_NAMESPACE,
+        cacheKey,
+      );
+    } catch {
+      return null;
+    }
+
+    if (!payload) {
+      return null;
+    }
+
+    const attributesValue: JSONObject["attributes"] = payload["attributes"];
+    const refreshedAtValue: JSONObject["refreshedAt"] = payload["refreshedAt"];
+
+    if (
+      !Array.isArray(attributesValue) ||
+      typeof refreshedAtValue !== "string"
+    ) {
+      return null;
+    }
+
+    const attributeCandidates: Array<unknown> =
+      attributesValue as Array<unknown>;
+
+    const attributes: Array<string> = attributeCandidates.filter(
+      (attribute: unknown): attribute is string => {
+        return typeof attribute === "string";
       },
-    });
+    );
 
-    return telemetryAttribute &&
-      telemetryAttribute.attributes &&
-      telemetryAttribute
-      ? telemetryAttribute.attributes
-      : [];
+    return {
+      attributes,
+      refreshedAt: OneUptimeDate.fromString(refreshedAtValue),
+    };
   }
 
-  @CaptureSpan()
-  public async refreshAttributes(data: {
+  private static isCacheFresh(
+    cacheEntry: TelemetryAttributesCacheEntry,
+  ): boolean {
+    const now: Date = OneUptimeDate.getCurrentDate();
+    const minutesSinceRefresh: number = Math.abs(
+      OneUptimeDate.getNumberOfMinutesBetweenDates(cacheEntry.refreshedAt, now),
+    );
+
+    return (
+      minutesSinceRefresh <= TelemetryAttributeService.CACHE_STALE_AFTER_MINUTES
+    );
+  }
+
+  private static async storeAttributesInCache(
+    cacheKey: string,
+    attributes: Array<string>,
+  ): Promise<void> {
+    const payload: JSONObject = {
+      attributes,
+      refreshedAt: OneUptimeDate.getCurrentDate().toISOString(),
+    };
+
+    try {
+      await GlobalCache.setJSON(
+        TelemetryAttributeService.CACHE_NAMESPACE,
+        cacheKey,
+        payload,
+        {
+          expiresInSeconds:
+            TelemetryAttributeService.CACHE_STALE_AFTER_MINUTES * 60,
+        },
+      );
+    } catch {
+      return;
+    }
+  }
+
+  private static buildAttributesStatement(data: {
     projectId: ObjectID;
-    telemetryType: TelemetryType;
-    attributes: string[];
-  }): Promise<void> {
-    const { projectId, telemetryType, attributes } = data;
-
-    // delete existing attributes
-    await this.deleteBy({
-      query: {
-        projectId,
-        telemetryType,
-      },
-      props: {
-        isRoot: true,
-      },
-    });
+    tableName: string;
+    attributesColumn: string;
+    timeColumn: string;
+  }): Statement {
+    const lookbackStartDate: Date =
+      TelemetryAttributeService.getLookbackStartDate();
 
-    const telemetryAttribute: TelemetryAttribute = new TelemetryAttribute();
+    const statement: Statement = SQL`
+      WITH filtered AS (
+        SELECT ${data.attributesColumn} AS attrs
+        FROM ${data.tableName}
+        WHERE projectId = ${{
+          type: TableColumnType.ObjectID,
+          value: data.projectId,
+        }}
+          AND ${data.attributesColumn} IS NOT NULL
+          AND ${data.attributesColumn} != ''
+          AND ${data.timeColumn} >= ${{
+            type: TableColumnType.Date,
+            value: lookbackStartDate,
+          }}
+        ORDER BY ${data.timeColumn} DESC
+        LIMIT ${{
+          type: TableColumnType.Number,
+          value: TelemetryAttributeService.ROW_SCAN_LIMIT,
+        }}
+      )
+      SELECT DISTINCT arrayJoin(JSONExtractKeys(attrs)) AS attribute
+      FROM filtered
+      ORDER BY attribute ASC
+      LIMIT ${{
+        type: TableColumnType.Number,
+        value: TelemetryAttributeService.ATTRIBUTES_LIMIT,
+      }}
+    `;
 
-    telemetryAttribute.projectId = projectId;
-    telemetryAttribute.telemetryType = telemetryType;
-    telemetryAttribute.attributes = attributes;
+    return statement;
+  }
 
-    await this.create({
-      data: telemetryAttribute,
-      props: {
-        isRoot: true,
-      },
-    });
+  private static async fetchAttributesFromDatabase(data: {
+    projectId: ObjectID;
+    source: TelemetrySource;
+  }): Promise<Array<string>> {
+    const statement: Statement =
+      TelemetryAttributeService.buildAttributesStatement({
+        projectId: data.projectId,
+        tableName: data.source.tableName,
+        attributesColumn: data.source.attributesColumn,
+        timeColumn: data.source.timeColumn,
+      });
+
+    const dbResult: Results = await data.source.service.executeQuery(statement);
+    const response: DbJSONResponse = await dbResult.json<{
+      data?: Array<JSONObject>;
+    }>();
+
+    const rows: Array<JSONObject> = response.data || [];
+
+    const attributeKeys: Array<string> = rows
+      .map((row: JSONObject) => {
+        const attribute: unknown = row["attribute"];
+        return typeof attribute === "string" ? attribute.trim() : null;
+      })
+      .filter((attribute: string | null): attribute is string => {
+        return Boolean(attribute);
+      });
+
+    return Array.from(new Set(attributeKeys));
   }
 }
 

package/Server/Services/WorkspaceProjectAuthTokenService.ts

@@ -5,6 +5,7 @@ import Model, {
   WorkspaceMiscData,
 } from "../../Models/DatabaseModels/WorkspaceProjectAuthToken";
 import { LIMIT_PER_PROJECT } from "../../Types/Database/LimitMax";
+import BadDataException from "../../Types/Exception/BadDataException";
 import CaptureSpan from "../Utils/Telemetry/CaptureSpan";
 
 export class Service extends DatabaseService<Model> {
@@ -17,6 +18,14 @@ export class Service extends DatabaseService<Model> {
     projectId: ObjectID;
     workspaceType: WorkspaceType;
   }): Promise<Model | null> {
+    if (!data.projectId) {
+      throw new BadDataException("projectId is required");
+    }
+
+    if (!data.workspaceType) {
+      throw new BadDataException("workspaceType is required");
+    }
+
     return await this.findOneBy({
       query: {
         projectId: data.projectId,
@@ -38,6 +47,10 @@ export class Service extends DatabaseService<Model> {
   public async getProjectAuths(data: {
     projectId: ObjectID;
   }): Promise<Array<Model>> {
+    if (!data.projectId) {
+      throw new BadDataException("projectId is required");
+    }
+
     return await this.findBy({
       query: {
         projectId: data.projectId,
@@ -72,6 +85,26 @@ export class Service extends DatabaseService<Model> {
     workspaceProjectId: string;
     miscData: WorkspaceMiscData;
   }): Promise<void> {
+    if (!data.projectId) {
+      throw new BadDataException("projectId is required");
+    }
+
+    if (!data.workspaceType) {
+      throw new BadDataException("workspaceType is required");
+    }
+
+    if (!data.authToken) {
+      throw new BadDataException("authToken is required");
+    }
+
+    if (!data.workspaceProjectId) {
+      throw new BadDataException("workspaceProjectId is required");
+    }
+
+    if (!data.miscData) {
+      throw new BadDataException("miscData is required");
+    }
+
     let projectAuth: Model | null = await this.findOneBy({
       query: {
         projectId: data.projectId,

package/Server/Utils/Monitor/MonitorResource.ts

@@ -1059,22 +1059,6 @@ export default class MonitorResourceUtil {
       logger.error(err);
     });
 
-    // index attributes.
-    TelemetryUtil.indexAttributes({
-      attributes: [
-        "monitorId",
-        "projectId",
-        "probeId",
-        "browserType",
-        "screenSizeType",
-        "diskPath",
-      ],
-      projectId: data.projectId,
-      telemetryType: TelemetryType.Metric,
-    }).catch((err: Error) => {
-      logger.error(err);
-    });
-
     // save monitor log.
     const monitorLog: MonitorLog = new MonitorLog();
     monitorLog.monitorId = data.monitorId;

package/Server/Utils/Telemetry/Telemetry.ts

@@ -1,10 +1,6 @@
 import { JSONArray, JSONObject, JSONValue } from "../../../Types/JSON";
 import ObjectID from "../../../Types/ObjectID";
-import TelemetryType from "../../../Types/Telemetry/TelemetryType";
-import GlobalCache from "../../Infrastructure/GlobalCache";
-import TelemetryAttributeService from "../../Services/TelemetryAttributeService";
 import CaptureSpan from "./CaptureSpan";
-import logger from "../Logger";
 import MetricType from "../../../Models/DatabaseModels/MetricType";
 import MetricTypeService from "../../Services/MetricTypeService";
 import TelemetryService from "../../../Models/DatabaseModels/TelemetryService";
@@ -127,63 +123,6 @@ export default class TelemetryUtil {
     }
   }
 
-  @CaptureSpan()
-  public static async indexAttributes(data: {
-    attributes: Array<string>;
-    projectId: ObjectID;
-    telemetryType: TelemetryType;
-  }): Promise<void> {
-    // index attributes
-
-    logger.debug("Indexing attributes");
-    logger.debug("data: " + JSON.stringify(data, null, 2));
-
-    const cacheKey: string =
-      data.projectId.toString() + "_" + data.telemetryType;
-
-    // get keys from cache
-    const cacheKeys: string[] =
-      (await GlobalCache.getStringArray("telemetryAttributesKeys", cacheKey)) ||
-      [];
-
-    let isKeysMissingInCache: boolean = false;
-
-    // check if keys are missing in cache
-
-    const attributeKeys: string[] = data.attributes;
-
-    for (const key of attributeKeys) {
-      if (!cacheKeys.includes(key)) {
-        isKeysMissingInCache = true;
-        break;
-      }
-    }
-
-    // merge keys and remove duplicates
-    if (isKeysMissingInCache) {
-      const dbKeys: string[] = await TelemetryAttributeService.fetchAttributes({
-        projectId: data.projectId,
-        telemetryType: data.telemetryType,
-      });
-
-      const mergedKeys: Array<string> = Array.from(
-        new Set([...dbKeys, ...attributeKeys, ...cacheKeys]),
-      );
-
-      await GlobalCache.setStringArray(
-        "telemetryAttributesKeys",
-        cacheKey,
-        mergedKeys,
-      );
-
-      await TelemetryAttributeService.refreshAttributes({
-        projectId: data.projectId,
-        telemetryType: data.telemetryType,
-        attributes: mergedKeys,
-      });
-    }
-  }
-
   public static getAttributesForServiceIdAndServiceName(data: {
     serviceId: ObjectID;
     serviceName: string;