@oneuptime/common 10.8.2 → 11.0.1

package/Utils/ModelImportExport.ts

@@ -0,0 +1,369 @@
+import BaseModel, {
+  DatabaseBaseModelType,
+} from "../Models/DatabaseModels/DatabaseBaseModel/DatabaseBaseModel";
+import { ColumnAccessControl } from "../Types/BaseDatabase/AccessControl";
+import Select from "../Types/BaseDatabase/Select";
+import { TableColumnMetadata } from "../Types/Database/TableColumn";
+import TableColumnType from "../Types/Database/TableColumnType";
+import OneUptimeDate from "../Types/Date";
+import Dictionary from "../Types/Dictionary";
+import Recurring from "../Types/Events/Recurring";
+import BadDataException from "../Types/Exception/BadDataException";
+import { JSONArray, JSONObject } from "../Types/JSON";
+
+export const MODEL_EXPORT_FILE_TYPE: string = "oneuptime-resource-export";
+export const MODEL_EXPORT_SCHEMA_VERSION: number = 1;
+
+/*
+ * Columns that are always server-managed and must never be exported or
+ * imported, regardless of model metadata.
+ */
+const SYSTEM_COLUMNS: Array<string> = [
+  "_id",
+  "createdAt",
+  "updatedAt",
+  "deletedAt",
+  "version",
+];
+
+/*
+ * Column types that never make sense in an export file: relations to other
+ * entities (their ids would not resolve in another project or instance),
+ * binary data, and secrets.
+ */
+const EXCLUDED_COLUMN_TYPES: Array<TableColumnType> = [
+  TableColumnType.Entity,
+  TableColumnType.EntityArray,
+  TableColumnType.File,
+  TableColumnType.Buffer,
+  TableColumnType.Slug,
+  TableColumnType.HashedString,
+  TableColumnType.Password,
+  TableColumnType.OTP,
+];
+
+/*
+ * Columns that pass the generic metadata filters but must still never be
+ * exported: plain-text credentials and server-managed runtime state that
+ * would either leak secrets into export files or seed imported resources
+ * with stale state from the source resource.
+ */
+const EXCLUDED_COLUMNS_BY_TABLE: Dictionary<Array<string>> = {
+  StatusPage: ["embeddedOverallStatusToken"],
+  Monitor: [
+    "incomingMonitorRequest",
+    "serverMonitorResponse",
+    "incomingRequestMonitorHeartbeatCheckedAt",
+    "serverMonitorRequestReceivedAt",
+    "telemetryMonitorNextMonitorAt",
+    "telemetryMonitorLastMonitorAt",
+  ],
+};
+
+export default class ModelImportExport {
+  /*
+   * Derives the list of columns that can safely round-trip through an export
+   * file from the model's own metadata. A column qualifies when the current
+   * caller could both read it and set it on create, it is not computed or
+   * server-generated, it holds no secret, and it does not reference another
+   * entity (foreign keys of Entity relations are excluded along with the
+   * relations themselves).
+   */
+  public static getImportExportableColumnNames(
+    modelType: DatabaseBaseModelType,
+  ): Array<string> {
+    const model: BaseModel = new modelType();
+
+    const tenantColumn: string | null = model.getTenantColumn();
+
+    const foreignKeyColumns: Array<string> = [];
+
+    for (const columnName of model.getTableColumns().columns) {
+      const metadata: TableColumnMetadata | undefined =
+        model.getTableColumnMetadata(columnName);
+
+      if (metadata?.manyToOneRelationColumn) {
+        foreignKeyColumns.push(metadata.manyToOneRelationColumn);
+      }
+    }
+
+    const excludedColumnsForTable: Array<string> =
+      EXCLUDED_COLUMNS_BY_TABLE[model.tableName || ""] || [];
+
+    const exportableColumns: Array<string> = [];
+
+    for (const columnName of model.getTableColumns().columns) {
+      if (SYSTEM_COLUMNS.includes(columnName)) {
+        continue;
+      }
+
+      if (excludedColumnsForTable.includes(columnName)) {
+        continue;
+      }
+
+      if (tenantColumn && columnName === tenantColumn) {
+        continue;
+      }
+
+      if (foreignKeyColumns.includes(columnName)) {
+        continue;
+      }
+
+      const metadata: TableColumnMetadata | undefined =
+        model.getTableColumnMetadata(columnName);
+
+      if (!metadata) {
+        continue;
+      }
+
+      if (EXCLUDED_COLUMN_TYPES.includes(metadata.type)) {
+        continue;
+      }
+
+      if (metadata.computed || metadata.hashed || metadata.encrypted) {
+        continue;
+      }
+
+      if (metadata.forceGetDefaultValueOnCreate) {
+        continue;
+      }
+
+      const accessControl: ColumnAccessControl | null =
+        model.getColumnAccessControlFor(columnName);
+
+      if (
+        !accessControl ||
+        accessControl.read.length === 0 ||
+        accessControl.create.length === 0
+      ) {
+        continue;
+      }
+
+      exportableColumns.push(columnName);
+    }
+
+    return exportableColumns;
+  }
+
+  public static getImportExportSelect<TBaseModel extends BaseModel>(modelType: {
+    new (): TBaseModel;
+  }): Select<TBaseModel> {
+    const select: JSONObject = {};
+
+    for (const columnName of this.getImportExportableColumnNames(modelType)) {
+      select[columnName] = true;
+    }
+
+    return select as Select<TBaseModel>;
+  }
+
+  public static toExportJSON<TBaseModel extends BaseModel>(
+    item: TBaseModel,
+    modelType: { new (): TBaseModel },
+  ): JSONObject {
+    const json: JSONObject = BaseModel.toJSON(item, modelType);
+
+    const exportableColumns: Array<string> =
+      this.getImportExportableColumnNames(modelType);
+
+    const exportJson: JSONObject = {};
+
+    for (const key of Object.keys(json)) {
+      if (exportableColumns.includes(key)) {
+        exportJson[key] = json[key];
+      }
+    }
+
+    return exportJson;
+  }
+
+  public static buildExportEnvelope<TBaseModel extends BaseModel>(data: {
+    modelType: { new (): TBaseModel };
+    items: Array<TBaseModel>;
+    exportedAt: Date;
+  }): JSONObject {
+    const model: BaseModel = new data.modelType();
+
+    return {
+      fileType: MODEL_EXPORT_FILE_TYPE,
+      schemaVersion: MODEL_EXPORT_SCHEMA_VERSION,
+      resourceType: model.tableName || "",
+      exportedAt: data.exportedAt.toISOString(),
+      items: data.items.map((item: TBaseModel) => {
+        return this.toExportJSON(item, data.modelType);
+      }),
+    };
+  }
+
+  /*
+   * Accepts an export envelope, a plain array of items, or a single item
+   * object, and returns the list of item JSONs to import. Throws
+   * BadDataException with a user-facing message when the payload is not
+   * usable for this model type.
+   */
+  public static parseImportPayload(data: {
+    modelType: DatabaseBaseModelType;
+    payload: JSONObject | JSONArray;
+  }): Array<JSONObject> {
+    const model: BaseModel = new data.modelType();
+    const resourceName: string = model.singularName || "resource";
+
+    let items: JSONArray | null = null;
+
+    if (Array.isArray(data.payload)) {
+      items = data.payload;
+    } else if (data.payload && typeof data.payload === "object") {
+      if (Array.isArray(data.payload["items"])) {
+        const resourceType: unknown = data.payload["resourceType"];
+
+        if (
+          resourceType &&
+          model.tableName &&
+          resourceType !== model.tableName
+        ) {
+          throw new BadDataException(
+            `This file contains ${resourceType.toString()} resources and cannot be imported here. Please select a ${resourceName} export file.`,
+          );
+        }
+
+        items = data.payload["items"] as JSONArray;
+      } else {
+        // a single item object exported by hand.
+        items = [data.payload];
+      }
+    }
+
+    if (!items) {
+      throw new BadDataException(
+        `This file is not a valid ${resourceName} export file.`,
+      );
+    }
+
+    if (items.length === 0) {
+      throw new BadDataException(
+        `This file does not contain any ${resourceName} to import.`,
+      );
+    }
+
+    for (const item of items) {
+      if (!item || typeof item !== "object" || Array.isArray(item)) {
+        throw new BadDataException(
+          `This file is not a valid ${resourceName} export file.`,
+        );
+      }
+    }
+
+    return items as Array<JSONObject>;
+  }
+
+  /*
+   * Builds a model instance from an import item, keeping only columns that
+   * are importable for this model. Anything else in the file - ids, slugs,
+   * timestamps, relations, unknown keys - is dropped so the server create
+   * API treats the item as a brand new resource.
+   */
+  public static fromImportJSON<TBaseModel extends BaseModel>(data: {
+    json: JSONObject;
+    modelType: { new (): TBaseModel };
+  }): TBaseModel {
+    const item: TBaseModel = BaseModel.fromJSON(
+      data.json,
+      data.modelType,
+    ) as TBaseModel;
+
+    const exportableColumns: Array<string> =
+      this.getImportExportableColumnNames(data.modelType);
+
+    for (const columnName of item.getTableColumns().columns) {
+      if (exportableColumns.includes(columnName)) {
+        continue;
+      }
+
+      if ((item as any)[columnName] !== undefined) {
+        item.removeValue(columnName);
+      }
+    }
+
+    this.normalizeImportItem(item);
+
+    return item;
+  }
+
+  /*
+   * Model-specific fixups so imported items pass server-side create
+   * validation. Currently: recurring scheduled maintenance templates are
+   * rejected on create when their first-event dates are in the past (the
+   * normal case for an export taken from an established template), so the
+   * dates are advanced by the recurrence interval until they are in the
+   * future, preserving the offsets between them.
+   */
+  private static normalizeImportItem(item: BaseModel): void {
+    if (item.tableName !== "ScheduledMaintenanceTemplate") {
+      return;
+    }
+
+    const isRecurringEvent: unknown = item.getValue("isRecurringEvent");
+    const recurringInterval: unknown = item.getValue("recurringInterval");
+
+    if (!isRecurringEvent || !recurringInterval) {
+      return;
+    }
+
+    const dateColumns: Array<string> = [
+      "firstEventScheduledAt",
+      "firstEventStartsAt",
+      "firstEventEndsAt",
+    ];
+
+    const dates: Array<Date> = [];
+
+    for (const dateColumn of dateColumns) {
+      const value: unknown = item.getValue(dateColumn);
+
+      if (value instanceof Date) {
+        dates.push(value);
+      }
+    }
+
+    if (dates.length === 0) {
+      return;
+    }
+
+    /*
+     * Anchor the advance on the earliest date so every date ends up in the
+     * future once shifted by the same amount.
+     */
+    const earliestDate: Date = dates.reduce((a: Date, b: Date) => {
+      return a.getTime() <= b.getTime() ? a : b;
+    });
+
+    if (OneUptimeDate.isInTheFuture(earliestDate)) {
+      return;
+    }
+
+    const interval: Recurring =
+      recurringInterval instanceof Recurring
+        ? recurringInterval
+        : Recurring.fromJSON(recurringInterval as JSONObject);
+
+    const nextEarliestDate: Date = Recurring.getNextDate(
+      earliestDate,
+      interval,
+    );
+
+    const advanceByMs: number =
+      nextEarliestDate.getTime() - earliestDate.getTime();
+
+    if (advanceByMs <= 0) {
+      return;
+    }
+
+    for (const dateColumn of dateColumns) {
+      const value: unknown = item.getValue(dateColumn);
+
+      if (value instanceof Date) {
+        (item as any)[dateColumn] = new Date(value.getTime() + advanceByMs);
+      }
+    }
+  }
+}

package/Utils/Telemetry/EntityKey.ts

@@ -0,0 +1,151 @@
+import Crypto from "../Crypto";
+import Dictionary from "../../Types/Dictionary";
+import EntityType from "../../Types/Telemetry/EntityType";
+
+/*
+ * Isomorphic OpenTelemetry entity-key computation — the single source of
+ * truth shared by ingest (server, stamping `entityKeys`) and reads
+ * (server or browser, building `has(entityKeys, :key)` predicates).
+ * Hashing goes through an injectable SHA-256 provider: the default is
+ * `Common/Utils/Crypto` (crypto-js, pure JS) so this module works in the
+ * browser, and the server swaps in node:crypto for the ingest hot path
+ * (see TelemetryEntity.ts). Any provider MUST produce the same
+ * lowercase-hex SHA-256 — a hard requirement: a read-side key that
+ * doesn't byte-match the ingest-side stamp finds nothing.
+ *
+ * See Internal/Docs/OpenTelemetryEntities.md.
+ */
+
+/** Canonical identity-value form: trimmed + lowercased. */
+export function canonicalizeEntityValue(value: string | undefined): string {
+  return String(value === undefined || value === null ? "" : value)
+    .trim()
+    .toLowerCase();
+}
+
+export type Sha256HexProvider = (input: string) => string;
+
+let sha256HexProvider: Sha256HexProvider = (input: string): string => {
+  return Crypto.getSha256Hash(input);
+};
+
+/**
+ * Swap the SHA-256 implementation (e.g. node:crypto on the server). The
+ * provider must return the full lowercase-hex digest, byte-identical to
+ * the crypto-js default, or ingest-stamped and read-side keys fork.
+ */
+export function setSha256Provider(provider: Sha256HexProvider): void {
+  sha256HexProvider = provider;
+}
+
+/*
+ * '|' separates preimage segments and '=' separates key from value, so
+ * both — and the escape character itself — are backslash-escaped inside
+ * keys and values. This keeps the preimage injective (a value like
+ * `a|service.namespace=b` cannot collide with a two-attribute identity)
+ * while leaving values WITHOUT these characters byte-identical to the
+ * historical preimage, so already-stamped keys for normal data are
+ * unchanged.
+ */
+function escapeIdentityToken(token: string): string {
+  return token.replace(/([\\|=])/g, "\\$1");
+}
+
+/**
+ * Stable 16-hex-char identity key for an entity. Pure: same inputs → same
+ * key. Keys are sorted so attribute order is irrelevant; values are
+ * canonicalized so casing/whitespace drift does not fork identity;
+ * projectId is folded in so keys are tenant-unique (and a
+ * `has(entityKeys, key)` predicate is implicitly project-scoped).
+ */
+export function computeEntityKey(data: {
+  projectId: string;
+  entityType: EntityType;
+  identifyingAttributes: Dictionary<string>;
+}): string {
+  const keys: Array<string> = Object.keys(
+    data.identifyingAttributes || {},
+  ).sort();
+
+  const parts: Array<string> = [];
+  for (const key of keys) {
+    parts.push(
+      `${escapeIdentityToken(key)}=${escapeIdentityToken(
+        canonicalizeEntityValue(data.identifyingAttributes[key]),
+      )}`,
+    );
+  }
+
+  const preimage: string = `${data.projectId}|${data.entityType}|${parts.join(
+    "|",
+  )}`;
+
+  return sha256HexProvider(preimage).slice(0, 16);
+}
+
+/*
+ * ---- Read-side helpers ---------------------------------------------------
+ *
+ * Given a OneUptime resource's identifying value, compute the entity key
+ * to query `has(entityKeys, :key)`. Each MUST mirror the corresponding
+ * ingest-side resolver in `TelemetryEntity.extractEntities` exactly.
+ */
+
+/**
+ * `host.name` is the host identity (matches the Host row's `hostIdentifier`,
+ * which is the canonicalized host.name). Pass `Host.hostIdentifier`.
+ */
+export function keyForHost(projectId: string, hostIdentifier: string): string {
+  return computeEntityKey({
+    projectId,
+    entityType: EntityType.Host,
+    identifyingAttributes: { "host.name": hostIdentifier },
+  });
+}
+
+/**
+ * `service.name` (+ `service.namespace` when the resource carries one) is
+ * the service identity — the ingest resolver folds the namespace into the
+ * key, so a namespaced service is only matchable when the same namespace
+ * is passed here. Pass `Service.name` (+ the namespace if known).
+ */
+export function keyForService(
+  projectId: string,
+  serviceName: string,
+  serviceNamespace?: string | undefined,
+): string {
+  const identifyingAttributes: Dictionary<string> = {
+    "service.name": serviceName,
+  };
+
+  /*
+   * Mirrors the resolver's addIfPresent: blank/whitespace-only namespaces
+   * are not identity-bearing.
+   */
+  if (serviceNamespace && serviceNamespace.trim().length > 0) {
+    identifyingAttributes["service.namespace"] = serviceNamespace;
+  }
+
+  return computeEntityKey({
+    projectId,
+    entityType: EntityType.Service,
+    identifyingAttributes,
+  });
+}
+
+/**
+ * `k8s.cluster.name` is the cluster identity (matches the KubernetesCluster
+ * row's `clusterIdentifier`; the ingest resolver is name-only — see
+ * `TelemetryEntity.k8sClusterIdentity`). Pass
+ * `KubernetesCluster.clusterIdentifier`.
+ */
+export function keyForKubernetesCluster(
+  projectId: string,
+  clusterIdentifier: string,
+): string {
+  return computeEntityKey({
+    projectId,
+    entityType: EntityType.KubernetesCluster,
+    identifyingAttributes: { "k8s.cluster.name": clusterIdentifier },
+  });
+}

package/Utils/Telemetry/EntityRelationship.ts

@@ -0,0 +1,113 @@
+import EntityRelationshipType from "../../Types/Telemetry/EntityRelationshipType";
+import EntityType from "../../Types/Telemetry/EntityType";
+
+/*
+ * Co-occurrence relationship inference (phase 5 of the entity model —
+ * Internal/Docs/OpenTelemetryEntities.md §4). Every ingest batch yields an
+ * entity set; the directed pairs that co-occur in one resource are, by
+ * construction, related. `relType` is inferred from the *type* pair, not
+ * any standardized wire signal (the OTel relationship taxonomy is
+ * undefined), giving an infrastructure topology graph for free. Pure +
+ * synchronous: operates on already-computed entity keys, no DB round trip.
+ */
+
+/** A single directed relationship edge between two entities. */
+export interface EntityRelationshipEdge {
+  fromEntityKey: string;
+  toEntityKey: string;
+  relationshipType: EntityRelationshipType;
+}
+
+/*
+ * Directed rules: a `from`-type that co-occurs with a `to`-type implies the
+ * given relationship. Only the meaningful direction has a rule (e.g. pod →
+ * cluster is `member-of`; cluster → pod yields nothing), so iterating all
+ * ordered pairs and keeping the non-null results produces a correctly
+ * directed edge set without double-counting. Extend as new entity types
+ * graduate from membership-only to first-class.
+ *
+ * `depends-on` is deliberately absent: a caller service and its callee
+ * never share one resource, so it cannot be co-occurrence-inferred. Those
+ * edges are derived from cross-service parent/child span pairs by the
+ * TelemetryEntity:ComputeServiceDependencies worker cron.
+ */
+const RULES: Record<string, EntityRelationshipType> = {
+  [`${EntityType.ServiceInstance}|${EntityType.Service}`]:
+    EntityRelationshipType.InstanceOf,
+  [`${EntityType.Service}|${EntityType.Host}`]: EntityRelationshipType.HostedOn,
+  [`${EntityType.Service}|${EntityType.KubernetesPod}`]:
+    EntityRelationshipType.RunsOn,
+  [`${EntityType.Container}|${EntityType.KubernetesPod}`]:
+    EntityRelationshipType.PartOf,
+  [`${EntityType.Container}|${EntityType.Host}`]: EntityRelationshipType.PartOf,
+  [`${EntityType.Process}|${EntityType.Container}`]:
+    EntityRelationshipType.PartOf,
+  [`${EntityType.Process}|${EntityType.Host}`]: EntityRelationshipType.RunsOn,
+  [`${EntityType.KubernetesPod}|${EntityType.KubernetesNode}`]:
+    EntityRelationshipType.RunsOn,
+  [`${EntityType.KubernetesPod}|${EntityType.KubernetesNamespace}`]:
+    EntityRelationshipType.MemberOf,
+  [`${EntityType.KubernetesPod}|${EntityType.KubernetesCluster}`]:
+    EntityRelationshipType.MemberOf,
+  [`${EntityType.KubernetesNode}|${EntityType.KubernetesCluster}`]:
+    EntityRelationshipType.MemberOf,
+  [`${EntityType.KubernetesNamespace}|${EntityType.KubernetesCluster}`]:
+    EntityRelationshipType.MemberOf,
+  [`${EntityType.KubernetesDeployment}|${EntityType.KubernetesNamespace}`]:
+    EntityRelationshipType.MemberOf,
+  [`${EntityType.KubernetesDeployment}|${EntityType.KubernetesCluster}`]:
+    EntityRelationshipType.MemberOf,
+};
+
+/** The relationship `fromType` → `toType` implies, or null if none. */
+export function inferRelationshipType(
+  fromType: EntityType,
+  toType: EntityType,
+): EntityRelationshipType | null {
+  return RULES[`${fromType}|${toType}`] || null;
+}
+
+/**
+ * Derive every directed relationship edge implied by an entity set (one
+ * resource batch). Deduped; self-pairs and same-key pairs skipped.
+ */
+export function deriveRelationships(
+  entities: Array<{ entityType: EntityType; entityKey: string }>,
+): Array<EntityRelationshipEdge> {
+  const edges: Array<EntityRelationshipEdge> = [];
+  const seen: Set<string> = new Set<string>();
+
+  for (let i: number = 0; i < entities.length; i++) {
+    for (let j: number = 0; j < entities.length; j++) {
+      if (i === j) {
+        continue;
+      }
+      const from: { entityType: EntityType; entityKey: string } = entities[i]!;
+      const to: { entityType: EntityType; entityKey: string } = entities[j]!;
+
+      if (from.entityKey === to.entityKey) {
+        continue;
+      }
+
+      const relationshipType: EntityRelationshipType | null =
+        inferRelationshipType(from.entityType, to.entityType);
+      if (!relationshipType) {
+        continue;
+      }
+
+      const dedupeKey: string = `${from.entityKey}|${to.entityKey}|${relationshipType}`;
+      if (seen.has(dedupeKey)) {
+        continue;
+      }
+      seen.add(dedupeKey);
+
+      edges.push({
+        fromEntityKey: from.entityKey,
+        toEntityKey: to.entityKey,
+        relationshipType,
+      });
+    }
+  }
+
+  return edges;
+}

package/Utils/Traces/CriticalPath.ts

@@ -9,7 +9,7 @@ export interface SpanData {
   startTimeUnixNano: number;
   endTimeUnixNano: number;
   durationUnixNano: number;
-  serviceId: string | undefined;
+  primaryEntityId: string | undefined;
   name: string | undefined;
 }
 
@@ -28,7 +28,7 @@ export interface CriticalPathResult {
 }
 
 export interface ServiceBreakdown {
-  serviceId: string;
+  primaryEntityId: string;
   totalDurationUnixNano: number;
   selfTimeUnixNano: number;
   spanCount: number;
@@ -308,12 +308,12 @@ export default class CriticalPathUtil {
     > = new Map();
 
     for (const span of spans) {
-      const serviceId: string = span.serviceId || "unknown";
+      const primaryEntityId: string = span.primaryEntityId || "unknown";
       const entry: {
         totalDuration: number;
         selfTime: number;
         spanCount: number;
-      } = serviceMap.get(serviceId) || {
+      } = serviceMap.get(primaryEntityId) || {
         totalDuration: 0,
         selfTime: 0,
         spanCount: 0,
@@ -323,13 +323,13 @@ export default class CriticalPathUtil {
       const selfTime: SpanSelfTime | undefined = selfTimes.get(span.spanId);
       entry.selfTime += selfTime ? selfTime.selfTimeUnixNano : 0;
       entry.spanCount += 1;
-      serviceMap.set(serviceId, entry);
+      serviceMap.set(primaryEntityId, entry);
     }
 
     const result: ServiceBreakdown[] = [];
-    for (const [serviceId, data] of serviceMap.entries()) {
+    for (const [primaryEntityId, data] of serviceMap.entries()) {
       result.push({
-        serviceId,
+        primaryEntityId,
         totalDurationUnixNano: data.totalDuration,
         selfTimeUnixNano: data.selfTime,
         spanCount: data.spanCount,