@oneuptime/common 10.8.1 → 11.0.0

package/Server/Utils/Monitor/Criteria/EvaluateOverTime.ts

@@ -36,7 +36,7 @@ export default class EvaluateOverTime {
     const query: Query<Metric> = {
       projectId: data.projectId,
       time: new InBetween(lastMinutesDate, now),
-      serviceId: data.monitorId,
+      primaryEntityId: data.monitorId,
       name: MonitorMetricTypeUtil.getMonitorMeticTypeByCheckOn(data.metricType),
     };
 

package/Server/Utils/Monitor/MonitorLogUtil.ts

@@ -133,9 +133,8 @@ export default class MonitorLogUtil {
         );
 
         const monitorLogRow: JSONObject = {
-          _id: ObjectID.generate().toString(),
+          _id: ObjectID.generateTimeOrdered().toString(),
           createdAt: logTimestamp,
-          updatedAt: logTimestamp,
           projectId: data.projectId.toString(),
           monitorId: data.monitorId.toString(),
           time: logTimestamp,

package/Server/Utils/Monitor/MonitorMetricUtil.ts

@@ -132,12 +132,11 @@ export default class MonitorMetricUtil {
     );
 
     return {
-      _id: ObjectID.generate().toString(),
+      _id: ObjectID.generateTimeOrdered().toString(),
       createdAt: ingestionTimestamp,
-      updatedAt: ingestionTimestamp,
       projectId: data.projectId.toString(),
-      serviceId: data.monitorId.toString(),
-      serviceType: ServiceType.Monitor,
+      primaryEntityId: data.monitorId.toString(),
+      primaryEntityType: ServiceType.Monitor,
       name: data.metricName,
       aggregationTemporality: null,
       metricPointType: data.metricPointType || MetricPointType.Sum,

package/Server/Utils/PrivacyFilterUtil.ts

@@ -0,0 +1,72 @@
+import Includes from "../../Types/BaseDatabase/Includes";
+import { FindWhereProperty } from "../../Types/BaseDatabase/Query";
+import ObjectID from "../../Types/ObjectID";
+import QueryHelper from "../Types/Database/QueryHelper";
+import { And, Equal, FindOperator } from "typeorm";
+
+/*
+ * Combines a caller-supplied filter value with a privacy Raw clause so that
+ * BOTH apply. The privacy filters used to overwrite the caller's value, which
+ * silently widened per-record queries (e.g. `incidentId: <id>` on the
+ * incident's Private Notes page) to every record the user was allowed to see
+ * — leaking notes/feed/owner rows across incidents for non-admin users.
+ *
+ * Runs in onBefore* hooks, i.e. BEFORE QueryUtil.serializeQuery, so the
+ * existing value can be a plain value (string/ObjectID/boolean/array), an
+ * Includes operator, or an already-built TypeORM FindOperator. Values are
+ * converted to TypeORM operators here because serializeQuery does not
+ * transform values nested inside And().
+ */
+export function combineWithPrivacyClause(
+  existingValue: unknown,
+  privacyClause: FindWhereProperty<any>,
+): FindWhereProperty<any> {
+  const privacyOperator: FindOperator<any> =
+    privacyClause as unknown as FindOperator<any>;
+
+  if (existingValue === undefined || existingValue === null) {
+    return privacyClause;
+  }
+
+  if (existingValue instanceof FindOperator) {
+    return And(existingValue, privacyOperator) as FindWhereProperty<any>;
+  }
+
+  if (existingValue instanceof ObjectID) {
+    return And(
+      Equal(existingValue.toString()),
+      privacyOperator,
+    ) as FindWhereProperty<any>;
+  }
+
+  if (existingValue instanceof Includes) {
+    return And(
+      QueryHelper.any(existingValue.values) as unknown as FindOperator<any>,
+      privacyOperator,
+    ) as FindWhereProperty<any>;
+  }
+
+  if (Array.isArray(existingValue)) {
+    return And(
+      QueryHelper.any(
+        existingValue as Array<string | ObjectID | number>,
+      ) as unknown as FindOperator<any>,
+      privacyOperator,
+    ) as FindWhereProperty<any>;
+  }
+
+  if (
+    typeof existingValue === "string" ||
+    typeof existingValue === "number" ||
+    typeof existingValue === "boolean" ||
+    existingValue instanceof Date
+  ) {
+    return And(Equal(existingValue), privacyOperator) as FindWhereProperty<any>;
+  }
+
+  /*
+   * Unrecognized operator shape (no caller passes these on privacy-filtered
+   * keys today). Fail closed: the privacy clause must always apply.
+   */
+  return privacyClause;
+}

package/Server/Utils/Profile/PprofEncoder.ts

@@ -1,17 +1,119 @@
 import { JSONObject } from "../../../Types/JSON";
+import protobuf from "protobufjs";
 import zlib from "zlib";
 
 /**
- * Encodes profile data into a simplified pprof-compatible JSON format.
- * This produces a gzipped JSON representation that captures the essential
- * profile information (stacktraces, values, metadata) in a format that
- * can be consumed by tools that support pprof JSON.
+ * Encodes profile data into the standard pprof wire format: a gzipped
+ * `perftools.profiles.Profile` protobuf message, openable with
+ * `go tool pprof`, Speedscope, and every other pprof consumer.
  *
- * For full protobuf pprof support, a protobuf serializer (e.g., protobufjs
- * with the pprof proto) would be needed. This implementation provides a
- * practical export format.
+ * Stacktrace frames arrive as denormalized "function@file:line" strings
+ * (the format ProfileSample rows store and Dashboard ProfileUtil parses);
+ * the encoder rebuilds the string table / function / location indirection
+ * that pprof requires.
  */
 
+/*
+ * Inline protobuf descriptor for the pprof Profile message, mirroring
+ * google/pprof profile.proto. Embedded as JSON (rather than loaded from
+ * a .proto file at runtime) so this module works in any deployment
+ * without carrying proto assets next to the compiled output.
+ */
+const PPROF_DESCRIPTOR: protobuf.INamespace = {
+  nested: {
+    perftools: {
+      nested: {
+        profiles: {
+          nested: {
+            Profile: {
+              fields: {
+                sampleType: { rule: "repeated", type: "ValueType", id: 1 },
+                sample: { rule: "repeated", type: "Sample", id: 2 },
+                mapping: { rule: "repeated", type: "Mapping", id: 3 },
+                location: { rule: "repeated", type: "Location", id: 4 },
+                function: { rule: "repeated", type: "Function", id: 5 },
+                stringTable: { rule: "repeated", type: "string", id: 6 },
+                dropFrames: { type: "int64", id: 7 },
+                keepFrames: { type: "int64", id: 8 },
+                timeNanos: { type: "int64", id: 9 },
+                durationNanos: { type: "int64", id: 10 },
+                periodType: { type: "ValueType", id: 11 },
+                period: { type: "int64", id: 12 },
+                comment: { rule: "repeated", type: "int64", id: 13 },
+                defaultSampleType: { type: "int64", id: 15 },
+              },
+            },
+            ValueType: {
+              fields: {
+                type: { type: "int64", id: 1 },
+                unit: { type: "int64", id: 2 },
+              },
+            },
+            Sample: {
+              fields: {
+                locationId: { rule: "repeated", type: "uint64", id: 1 },
+                value: { rule: "repeated", type: "int64", id: 2 },
+                label: { rule: "repeated", type: "Label", id: 3 },
+              },
+            },
+            Label: {
+              fields: {
+                key: { type: "int64", id: 1 },
+                str: { type: "int64", id: 2 },
+                num: { type: "int64", id: 3 },
+                numUnit: { type: "int64", id: 4 },
+              },
+            },
+            Mapping: {
+              fields: {
+                id: { type: "uint64", id: 1 },
+                memoryStart: { type: "uint64", id: 2 },
+                memoryLimit: { type: "uint64", id: 3 },
+                fileOffset: { type: "uint64", id: 4 },
+                filename: { type: "int64", id: 5 },
+                buildId: { type: "int64", id: 6 },
+                hasFunctions: { type: "bool", id: 7 },
+                hasFilenames: { type: "bool", id: 8 },
+                hasLineNumbers: { type: "bool", id: 9 },
+                hasInlineFrames: { type: "bool", id: 10 },
+              },
+            },
+            Location: {
+              fields: {
+                id: { type: "uint64", id: 1 },
+                mappingId: { type: "uint64", id: 2 },
+                address: { type: "uint64", id: 3 },
+                line: { rule: "repeated", type: "Line", id: 4 },
+                isFolded: { type: "bool", id: 5 },
+              },
+            },
+            Line: {
+              fields: {
+                functionId: { type: "uint64", id: 1 },
+                line: { type: "int64", id: 2 },
+              },
+            },
+            Function: {
+              fields: {
+                id: { type: "uint64", id: 1 },
+                name: { type: "int64", id: 2 },
+                systemName: { type: "int64", id: 3 },
+                filename: { type: "int64", id: 4 },
+                startLine: { type: "int64", id: 5 },
+              },
+            },
+          },
+        },
+      },
+    },
+  },
+};
+
+const PprofRoot: protobuf.Root = protobuf.Root.fromJSON(PPROF_DESCRIPTOR);
+const PprofProfileMessage: protobuf.Type = PprofRoot.lookupType(
+  "perftools.profiles.Profile",
+);
+
 export interface PprofSample {
   stacktrace: Array<string>;
   value: number;
@@ -139,7 +241,13 @@ export default class PprofEncoder {
     for (const sample of profile.samples) {
       const locationIds: Array<number> = [];
 
-      // Parse each frame in the stacktrace
+      /*
+       * Stored stacktrace arrays preserve the wire order of their
+       * sources — pprof `Sample.location_id` and OTLP `Stack.
+       * location_indices` are both leaf-first — so emitting
+       * location_ids in array order keeps the leaf-first ordering
+       * pprof consumers require.
+       */
       for (const frame of sample.stacktrace) {
         const atIndex: number = frame.indexOf("@");
         let functionName: string = frame;
@@ -202,17 +310,33 @@ export default class PprofEncoder {
   }
 
   /**
-   * Encode and gzip the pprof JSON for download.
+   * Serialize the profile as a gzipped `perftools.profiles.Profile`
+   * protobuf — the on-disk format `go tool pprof` and friends expect.
+   * The intermediate structure from `encode` already uses the proto's
+   * camelCased field names, so it maps onto the message verbatim.
    */
   public static async encodeAndCompress(
     profile: PprofProfile,
   ): Promise<Buffer> {
     const pprofData: PprofProto = PprofEncoder.encode(profile);
-    const jsonString: string = JSON.stringify(pprofData);
+
+    const message: protobuf.Message = PprofProfileMessage.fromObject({
+      sampleType: pprofData.sampleType,
+      sample: pprofData.samples,
+      location: pprofData.locations,
+      function: pprofData.functions,
+      stringTable: pprofData.stringTable,
+      timeNanos: pprofData.timeNanos,
+      durationNanos: pprofData.durationNanos,
+      periodType: pprofData.periodType,
+      period: pprofData.period,
+    });
+
+    const encoded: Uint8Array = PprofProfileMessage.encode(message).finish();
 
     return new Promise<Buffer>(
       (resolve: (value: Buffer) => void, reject: (reason: Error) => void) => {
-        zlib.gzip(jsonString, (err: Error | null, result: Buffer) => {
+        zlib.gzip(encoded, (err: Error | null, result: Buffer) => {
           if (err) {
             reject(err);
           } else {

package/Server/Utils/Telemetry/EntityRegistry.ts

@@ -0,0 +1,316 @@
+import BaseModel from "../../../Models/DatabaseModels/DatabaseBaseModel/DatabaseBaseModel";
+import DatabaseService from "../../Services/DatabaseService";
+import Query from "../../Types/Database/Query";
+import Select from "../../Types/Database/Select";
+import QueryDeepPartialEntity from "../../../Types/Database/PartialEntity";
+import ObjectID from "../../../Types/ObjectID";
+import EntityType from "../../../Types/Telemetry/EntityType";
+import GlobalCache from "../../Infrastructure/GlobalCache";
+import logger from "../Logger";
+import { ExtractedEntity } from "./TelemetryEntity";
+import TelemetryEntityService from "../../Services/TelemetryEntityService";
+import TelemetryEntityRelationshipService from "../../Services/TelemetryEntityRelationshipService";
+import {
+  deriveRelationships,
+  EntityRelationshipEdge,
+} from "../../../Utils/Telemetry/EntityRelationship";
+
+/*
+ * Shared entity-registry reconciliation machinery (phases 2 + 5 of the
+ * entity model — see Internal/Docs/OpenTelemetryEntities.md). Lives in
+ * Common so every ingest path that discovers entities — the OTLP pipeline
+ * (OtelIngestBaseService.resolveTelemetryResource) and the lower-fidelity
+ * name-only paths (syslog / fluent via
+ * OpenTelemetryIngestService.telemetryServiceFromName) — funnels through
+ * the SAME promotion gate and Redis fence.
+ */
+
+/*
+ * High-churn types are membership-only by default (doc §Edge Cases /
+ * Decision 7): their keys flow into the `entityKeys` column on signals,
+ * but they are never promoted to registry rows — container restarts and
+ * pid reuse would otherwise mint unbounded registry rows. Everything
+ * else promotes. `service.instance` is membership-only too: semconv
+ * `service.instance.id` is typically a per-restart UUID, so each deploy
+ * would mint a fresh registry row per instance.
+ */
+const MEMBERSHIP_ONLY_TYPES: ReadonlySet<EntityType> = new Set<EntityType>([
+  EntityType.Container,
+  EntityType.Process,
+  EntityType.ServiceInstance,
+  EntityType.TelemetrySdk,
+]);
+
+/*
+ * Per-(project, entityType) registry budget (doc §Edge Cases — "reuse the
+ * existing per-service metricCardinalityBudget concept as a per-type
+ * entity budget"). Beyond budget, NEW registry rows stop being created —
+ * membership keys still flow into `entityKeys` on signals, and existing
+ * rows keep their `lastSeenAt` bumps so the prune TTL never reaps live
+ * entities. Hardcoded defaults; a per-project override is a follow-up.
+ */
+export const DEFAULT_ENTITY_BUDGET: ReadonlyMap<EntityType, number> = new Map<
+  EntityType,
+  number
+>([
+  [EntityType.Service, 10000],
+  [EntityType.Host, 10000],
+  [EntityType.KubernetesCluster, 10000],
+  [EntityType.KubernetesNode, 1000],
+  [EntityType.KubernetesNamespace, 1000],
+  [EntityType.KubernetesPod, 5000],
+  [EntityType.KubernetesDeployment, 5000],
+]);
+
+// For types not in the map (future promotions of high-churn types).
+export const FALLBACK_ENTITY_BUDGET: number = 5000;
+
+export function getEntityBudget(entityType: EntityType): number {
+  return DEFAULT_ENTITY_BUDGET.get(entityType) ?? FALLBACK_ENTITY_BUDGET;
+}
+
+export const REGISTRY_PROMOTED_TYPES: ReadonlySet<EntityType> =
+  new Set<EntityType>(
+    Object.values(EntityType).filter((entityType: EntityType) => {
+      return !MEMBERSHIP_ONLY_TYPES.has(entityType);
+    }),
+  );
+
+/*
+ * Same Redis namespace / key shape / TTL as
+ * OtelIngestBaseService.shouldRunMaintenance, so an entity set fenced by
+ * one ingest path is fenced for all of them.
+ */
+const FENCE_NAMESPACE: string = "otel-maintenance-fence";
+const FENCE_SCOPE: string = "entity-reconcile";
+const FENCE_TTL_SECONDS: number = 5 * 60; // 5 minutes
+
+async function shouldReconcile(fenceId: string): Promise<boolean> {
+  try {
+    const key: string = `${FENCE_SCOPE}:${fenceId}`;
+    const seen: string | null = await GlobalCache.getString(
+      FENCE_NAMESPACE,
+      key,
+    );
+    if (seen) {
+      return false;
+    }
+    await GlobalCache.setString(FENCE_NAMESPACE, key, "1", {
+      expiresInSeconds: FENCE_TTL_SECONDS,
+    });
+    return true;
+  } catch {
+    // If the cache is down, default to running the reconcile.
+    return true;
+  }
+}
+
+/*
+ * Over-budget skips happen per row, but the warn log is fenced to once per
+ * (project, entityType) per fence window — an over-budget project would
+ * otherwise emit a warn line for every skipped entity of every batch.
+ */
+export async function shouldWarnEntityBudgetOnce(data: {
+  projectId: ObjectID;
+  entityType: EntityType;
+}): Promise<boolean> {
+  try {
+    const key: string = `entity-budget-warn:${data.projectId.toString()}:${
+      data.entityType
+    }`;
+    const seen: string | null = await GlobalCache.getString(
+      FENCE_NAMESPACE,
+      key,
+    );
+    if (seen) {
+      return false;
+    }
+    await GlobalCache.setString(FENCE_NAMESPACE, key, "1", {
+      expiresInSeconds: FENCE_TTL_SECONDS,
+    });
+    return true;
+  } catch {
+    // If the cache is down, default to warning (visibility over silence).
+    return true;
+  }
+}
+
+/*
+ * Upsert discovered entities into the `TelemetryEntity` registry and their
+ * co-occurrence edges into `TelemetryEntityRelationship`. Gated by a single
+ * per-batch Redis fence keyed on the PROMOTED entity subset, so a stable
+ * resource reconciles at most once per window while a changed set (e.g. a
+ * pod reschedule) reconciles immediately — and the high-churn
+ * membership-only keys (container restarts, pid reuse) cannot churn the
+ * fence id and defeat the throttle. Edges are derived only among promoted
+ * entities so no edge references a key the registry never registered.
+ *
+ * Forward-only and best-effort: every error is swallowed (logged) here, so
+ * callers may fire-and-forget — a registry failure must never break signal
+ * ingest. `entityKeys` stamping on signal rows is independent of this and
+ * stays synchronous at the call sites.
+ */
+export async function reconcileEntityRegistryThrottled(data: {
+  projectId: ObjectID;
+  entities: Array<ExtractedEntity>;
+}): Promise<void> {
+  try {
+    const promoted: Array<ExtractedEntity> = data.entities.filter(
+      (entity: ExtractedEntity) => {
+        return REGISTRY_PROMOTED_TYPES.has(entity.entityType);
+      },
+    );
+
+    if (promoted.length === 0) {
+      return;
+    }
+
+    const fenceId: string = `${data.projectId.toString()}:${promoted
+      .map((entity: ExtractedEntity) => {
+        return entity.entityKey;
+      })
+      .sort()
+      .join(",")}`;
+
+    if (!(await shouldReconcile(fenceId))) {
+      return;
+    }
+
+    await TelemetryEntityService.reconcileEntities({
+      projectId: data.projectId,
+      entities: promoted,
+    });
+
+    /*
+     * Topology (phase 5): the co-occurrence edges derive from the same
+     * entity set, so they reconcile under the same fence — no extra
+     * Redis check. A stable resource bumps both the registry and the
+     * graph once per window; a changed set re-derives both.
+     */
+    const edges: Array<EntityRelationshipEdge> = deriveRelationships(
+      promoted.map((entity: ExtractedEntity) => {
+        return {
+          entityType: entity.entityType,
+          entityKey: entity.entityKey,
+        };
+      }),
+    );
+    if (edges.length > 0) {
+      await TelemetryEntityRelationshipService.reconcileRelationships({
+        projectId: data.projectId,
+        edges,
+      });
+    }
+  } catch (err) {
+    logger.error("Entity registry reconciliation failed:");
+    logger.error(err as Error);
+  }
+}
+
+/*
+ * Find-or-create by natural key, bumping `lastSeenAt` — the one upsert
+ * scaffold shared by TelemetryEntityService and
+ * TelemetryEntityRelationshipService. Both tables have a unique index on
+ * their natural key, so a concurrent first-contact create loses the race
+ * with a unique-violation: in that case the winning row is re-fetched and
+ * its `lastSeenAt` bumped immediately (instead of waiting a full throttle
+ * window). A create failure with NO winning row is a real error (e.g. a
+ * column constraint violation) and is surfaced at warn.
+ */
+export async function reconcileByNaturalKey<
+  TBaseModel extends BaseModel & { lastSeenAt?: Date },
+>(data: {
+  service: DatabaseService<TBaseModel>;
+  query: Query<TBaseModel>;
+  buildModel: () => TBaseModel;
+  lastSeenAt: Date;
+  /** Human-readable row identity for log lines, e.g. "entity host/abc123". */
+  describe: string;
+  /** Extra columns to select on the existing row, for `buildUpdate` diffing. */
+  select?: Select<TBaseModel> | undefined;
+  /**
+   * Extra fields to fold into the `lastSeenAt` bump (descriptive-attribute
+   * merge, label union). Return an empty object when nothing changed so the
+   * bump stays a single-column update.
+   */
+  buildUpdate?:
+    | ((existing: TBaseModel) => QueryDeepPartialEntity<TBaseModel>)
+    | undefined;
+  /**
+   * Gate run only when a NEW row is about to be created (entity budget).
+   * Returning false skips creation silently — the gate owns its own
+   * logging/throttling. Existing-row bumps are never gated.
+   */
+  beforeCreate?: (() => Promise<boolean>) | undefined;
+}): Promise<void> {
+  const select: Select<TBaseModel> = {
+    ...({ _id: true } as Select<TBaseModel>),
+    ...(data.select || {}),
+  };
+
+  const buildBump: (
+    existing: TBaseModel,
+  ) => QueryDeepPartialEntity<TBaseModel> = (
+    existing: TBaseModel,
+  ): QueryDeepPartialEntity<TBaseModel> => {
+    // Unresolved generic mapped type — TS cannot prove overlap directly.
+    return {
+      lastSeenAt: data.lastSeenAt,
+      ...(data.buildUpdate ? data.buildUpdate(existing) : {}),
+    } as unknown as QueryDeepPartialEntity<TBaseModel>;
+  };
+
+  const existing: TBaseModel | null = await data.service.findOneBy({
+    query: data.query,
+    select,
+    props: { isRoot: true },
+  });
+
+  if (existing) {
+    // Throttled bump of lastSeenAt (+ any caller-supplied merge fields).
+    await data.service.updateOneById({
+      id: existing.id!,
+      data: buildBump(existing),
+      props: { isRoot: true },
+    });
+    return;
+  }
+
+  if (data.beforeCreate && !(await data.beforeCreate())) {
+    return;
+  }
+
+  try {
+    await data.service.create({
+      data: data.buildModel(),
+      props: { isRoot: true },
+    });
+    return;
+  } catch (err) {
+    /*
+     * Re-find to disambiguate: a row now exists means a concurrent worker
+     * won the unique-indexed create race (harmless — bump it); no row
+     * means the insert itself was invalid, which must be visible.
+     */
+    const winner: TBaseModel | null = await data.service.findOneBy({
+      query: data.query,
+      select,
+      props: { isRoot: true },
+    });
+
+    if (winner) {
+      logger.debug(
+        `EntityRegistry: create raced for ${data.describe} (concurrent insert); bumping lastSeenAt on the winning row.`,
+      );
+      await data.service.updateOneById({
+        id: winner.id!,
+        data: buildBump(winner),
+        props: { isRoot: true },
+      });
+      return;
+    }
+
+    logger.warn(`EntityRegistry: create failed for ${data.describe}:`);
+    logger.warn(err as Error);
+  }
+}

package/Server/Utils/Telemetry/ResourceFacetResolver.ts

@@ -25,8 +25,12 @@ import CaptureSpan from "./CaptureSpan";
  * Resolving the value list from Postgres instead means every project resource
  * shows up regardless of recent telemetry activity, and the sidebar search
  * matches across the full set (not just the loaded subset).
+ *
+ * `serviceId` is the pre-rename alias of `primaryEntityId`, kept so stale
+ * clients keep resolving the Services facet across a deploy.
  */
 export const RESOURCE_FACET_KEYS: ReadonlySet<string> = new Set([
+  "primaryEntityId",
   "serviceId",
   "hostId",
   "dockerHostId",
@@ -92,6 +96,7 @@ export default class ResourceFacetResolver {
         : undefined;
 
     switch (spec.facetKey) {
+      case "primaryEntityId":
       case "serviceId":
         return ResourceFacetResolver.queryServices(
           projectId,
@@ -389,8 +394,8 @@ export default class ResourceFacetResolver {
       });
     }
 
-    const apps: Array<RumApplicationModel> =
-      await RumApplicationService.findBy({
+    const apps: Array<RumApplicationModel> = await RumApplicationService.findBy(
+      {
         query: query as any,
         select: {
           _id: true,
@@ -400,7 +405,8 @@ export default class ResourceFacetResolver {
         limit: new PositiveNumber(limit),
         skip: new PositiveNumber(0),
         props: { isRoot: true },
-      });
+      },
+    );
 
     return ResourceFacetResolver.mergeCounts(
       apps.map(