@oneuptime/common 12.0.4 → 12.0.6

package/Server/Utils/Telemetry/CaptureSpan.ts

@@ -38,35 +38,47 @@ function CaptureSpan(data?: {
       data?.name || `${className}.${propertyKey}`;
 
     descriptor.value = function (...args: any[]) {
-      if (DisableTelemetry) {
+      /*
+       * Skip the span machinery entirely when telemetry is disabled OR no
+       * span exporter is installed. Without an exporter the span is never
+       * shipped anywhere, but creating it still costs attribute flattening,
+       * a span allocation, and an AsyncLocalStorage context transition —
+       * decorated methods sit on ingest hot paths that run millions of
+       * times per minute, so this must be a plain method call there.
+       */
+      if (DisableTelemetry || !Telemetry.isSpanExportEnabled()) {
         return originalMethod.apply(this, args);
       }
 
-      let functionArguments: JSONObject = {};
+      /*
+       * Only pay for attribute flattening when there is something to
+       * flatten — the overwhelmingly common bare @CaptureSpan() form has
+       * neither captured arguments nor static attributes.
+       */
+      let spanAttributes: { [key: string]: any } | undefined = undefined;
 
-      if (data?.captureArguments) {
-        functionArguments = args.reduce(
-          (acc: { [key: string]: any }, arg: any, index: number) => {
-            acc[`arg${index}`] = arg;
-            return acc;
-          },
-          {},
-        );
-      }
+      if (data?.captureArguments || data?.attributes) {
+        let functionArguments: JSONObject = {};
+
+        if (data?.captureArguments) {
+          functionArguments = args.reduce(
+            (acc: { [key: string]: any }, arg: any, index: number) => {
+              acc[`arg${index}`] = arg;
+              return acc;
+            },
+            {},
+          );
+        }
 
-      const spanAttributes: { [key: string]: any } =
-        JSONFunctions.flattenObject({
+        spanAttributes = JSONFunctions.flattenObject({
           ...functionArguments,
           ...data?.attributes,
         }) as { [key: string]: any };
+      }
 
       return Telemetry.startActiveSpan({
         name: name,
-        options: {
-          attributes: {
-            ...spanAttributes,
-          },
-        },
+        options: spanAttributes ? { attributes: spanAttributes } : {},
         fn: (span: Span) => {
           let result: any = null;
           try {

package/Server/Utils/Telemetry/IngestionTimestamp.ts

@@ -0,0 +1,75 @@
+import OneUptimeDate from "../../../Types/Date";
+
+/*
+ * Per-second memo for the wall-clock stamps every ingested telemetry row
+ * carries (createdAt and retentionDate, both stored with second
+ * granularity).
+ *
+ * The Otel logs / traces / metrics row builders used to derive these per
+ * row: a current-Date allocation, a ClickHouse DateTime format, a
+ * moment-based add-days, and a second format — for every log record, span,
+ * and metric datapoint. All of that output only changes once per second
+ * (createdAt has no sub-second component) and once per distinct retention
+ * window, so a burst of rows in the same second re-derived identical
+ * strings millions of times. This memo pays the derivation once per second
+ * (and once per retention window within that second) instead.
+ *
+ * Not thread-shared state to worry about: workers are single-threaded, and
+ * the memo is replaced wholesale when the second rolls over, so the map of
+ * retention strings can never grow past the number of distinct retention
+ * windows seen within one second.
+ */
+
+export interface IngestionStamp {
+  /** Current time truncated to the second the stamp was minted. */
+  date: Date;
+  /** ClickHouse DateTime string ("YYYY-MM-DD HH:mm:ss", UTC) of `date`. */
+  db: string;
+}
+
+type CachedStamp = IngestionStamp & {
+  epochSecond: number;
+  retentionDbByDays: Map<number, string>;
+};
+
+let cachedStamp: CachedStamp | null = null;
+
+export function getIngestionStamp(): IngestionStamp {
+  const epochSecond: number = Math.floor(Date.now() / 1000);
+
+  if (!cachedStamp || cachedStamp.epochSecond !== epochSecond) {
+    const date: Date = new Date(epochSecond * 1000);
+    cachedStamp = {
+      epochSecond: epochSecond,
+      date: date,
+      db: OneUptimeDate.toClickhouseDateTime(date),
+      retentionDbByDays: new Map<number, string>(),
+    };
+  }
+
+  return cachedStamp;
+}
+
+/*
+ * ClickHouse DateTime string for `stamp.date + retentionDays`, memoized per
+ * (stamp second, retention window). Must be called with a stamp obtained
+ * from getIngestionStamp() in the same row-building pass, so the retention
+ * date stays consistent with the row's createdAt.
+ */
+export function getRetentionDateDb(
+  stamp: IngestionStamp,
+  retentionDays: number,
+): string {
+  const cached: CachedStamp = stamp as CachedStamp;
+
+  let db: string | undefined = cached.retentionDbByDays?.get(retentionDays);
+
+  if (db === undefined) {
+    db = OneUptimeDate.toClickhouseDateTime(
+      OneUptimeDate.addRemoveDays(stamp.date, retentionDays),
+    );
+    cached.retentionDbByDays?.set(retentionDays, db);
+  }
+
+  return db;
+}

package/Server/Utils/Telemetry/ResourceHeartbeat.ts

@@ -3,6 +3,7 @@ import QueryDeepPartialEntity from "../../../Types/Database/PartialEntity";
 import ObjectID from "../../../Types/ObjectID";
 import DatabaseService from "../../Services/DatabaseService";
 import GlobalCache from "../../Infrastructure/GlobalCache";
+import InProcessMemo from "../InProcessMemo";
 import SingleFlight from "../SingleFlight";
 import logger from "../Logger";
 
@@ -116,10 +117,55 @@ function writeWindowNamespace(cacheNamespace: string): string {
 }
 
 export default class ResourceHeartbeat {
+  /*
+   * Time-based L1 over the whole gate cascade. SingleFlight collapses
+   * CONCURRENT duplicates, but sequential batches — the next job for the
+   * same host arriving a second later — each still paid 1-3 Redis round
+   * trips (liveness SET NX, fingerprint compare-and-claim, rate-limit
+   * SET NX) to be told "nothing to do". Once THIS process has settled a
+   * heartbeat for a (resource, fingerprint), that answer holds for the
+   * rest of the window, so re-asking Redis per batch buys nothing.
+   *
+   * The memo key includes the FINGERPRINT, so a genuinely changed payload
+   * bypasses the memo and reaches the compare-and-claim gate as promptly
+   * as it did before — the L1 only ever suppresses re-asking a question
+   * whose inputs are identical.
+   *
+   * Entries live for min(60s, the caller's liveness window, pre-jitter).
+   * Within that span the Redis liveness key this process (or another pod)
+   * claimed cannot have expired yet (its jittered TTL is >= the window),
+   * so a skipped probe would have been refused anyway. The one real trade
+   * is cross-pod lastSeen PRECISION: when a DIFFERENT pod's claim expires
+   * mid-window, this pod would previously have won the reopened gate on
+   * its next batch, and now reacts up to one memo TTL later (any other pod
+   * still reacts immediately). That delay is bounded by the existing
+   * window — the same staleness the jittered TTL already permits — and
+   * sits far inside the 15-minute markDisconnected* thresholds.
+   *
+   * Outcomes that must retry are NEVER memoized: a failed or lock-skipped
+   * metadata write (those release their Redis gates precisely so the next
+   * batch retries) and any outcome where a gate probe threw (a Redis
+   * outage must keep failing open per batch, not once per memo TTL).
+   */
+  private static readonly recentHeartbeatMemo: InProcessMemo<boolean> =
+    new InProcessMemo<boolean>({
+      ttlInMs: 60 * 1000,
+      maxEntries: 10_000,
+    });
+
   public static async write<TBaseModel extends BaseModel>(
     input: ResourceHeartbeatWrite<TBaseModel>,
   ): Promise<void> {
     const cacheKey: string = input.id.toString();
+    const memoKey: string = `${input.cacheNamespace}:${cacheKey}:${input.fingerprint}`;
+
+    /*
+     * L1: this process already settled this exact heartbeat within the
+     * window. Zero network cost, no gates touched.
+     */
+    if (this.recentHeartbeatMemo.get(memoKey)) {
+      return;
+    }
 
     /*
      * Collapse duplicate concurrent callers inside THIS process first. One
@@ -128,18 +174,41 @@ export default class ResourceHeartbeat {
      * and without this every one of them independently asks Redis the same
      * question and throws the answer away.
      */
-    await SingleFlight.run(
-      `${input.cacheNamespace}:${cacheKey}:${input.fingerprint}`,
-      async () => {
-        await this.writeOnce(input, cacheKey);
-      },
-    );
+    await SingleFlight.run(memoKey, async () => {
+      await this.writeOnce(input, cacheKey, memoKey);
+    });
+  }
+
+  /**
+   * Drop the recent-heartbeat memo. For tests only — suites that reuse one
+   * resource id across cases simulate the memo TTL expiring with this, the
+   * same way they already clear their fake Redis maps and SingleFlight.
+   */
+  public static clearRecentHeartbeatMemo(): void {
+    this.recentHeartbeatMemo.clear();
   }
 
   private static async writeOnce<TBaseModel extends BaseModel>(
     input: ResourceHeartbeatWrite<TBaseModel>,
     cacheKey: string,
+    memoKey: string,
   ): Promise<void> {
+    /*
+     * Whether every Redis gate we consulted actually answered. A probe that
+     * THREW fell back to a hardcoded outcome (open for liveness, closed for
+     * enrichment) — that outcome is a per-batch policy, not a fact about the
+     * window, so it must never be memoized into the L1.
+     */
+    let gatesAnswered: boolean = true;
+
+    /*
+     * The memo must expire no later than the PRE-JITTER window, because that
+     * is the earliest instant the Redis liveness key claimed alongside it
+     * could expire — and never later than the 60-second norm.
+     */
+    const memoTtlInMs: number =
+      Math.min(60, Math.max(input.throttleInSeconds, 0)) * 1000;
+
     /*
      * Liveness gate. Presence-only and jittered: a fleet-wide restart would
      * otherwise re-synchronise every resource's window and rebuild the herd on
@@ -163,6 +232,7 @@ export default class ResourceHeartbeat {
        * the write below cannot queue.
        */
       writeLiveness = true;
+      gatesAnswered = false;
     }
 
     const hasMetadata: boolean =
@@ -194,6 +264,7 @@ export default class ResourceHeartbeat {
          * write storm this exists to prevent.
          */
         metadataChanged = false;
+        gatesAnswered = false;
       }
     }
 
@@ -218,11 +289,19 @@ export default class ResourceHeartbeat {
         );
       } catch {
         writeMetadata = false;
+        gatesAnswered = false;
       }
     }
 
     if (!writeLiveness && !writeMetadata) {
-      // Nothing to say. Issue no statement at all.
+      /*
+       * Nothing to say. Issue no statement at all — and remember that for
+       * the memo TTL, since every gate answered and the same inputs can
+       * only produce the same silence for the rest of the window.
+       */
+      if (gatesAnswered) {
+        this.recentHeartbeatMemo.set(memoKey, true, memoTtlInMs);
+      }
       return;
     }
 
@@ -283,6 +362,19 @@ export default class ResourceHeartbeat {
        * statement that returns immediately when contended.
        */
       await this.releaseGates(input.cacheNamespace, cacheKey, true);
+      return;
+    }
+
+    /*
+     * The write LANDED: nothing is left for this exact input to do this
+     * window, so remember that. Memoized only when every gate genuinely
+     * answered (an outage outcome is per-batch policy, never a fact) and
+     * NOT for the liveness-only lock-skip above `wrote === false` — that
+     * case is rare, already free of gate churn next batch, and memoizing
+     * a write we did not perform is a subtlety not worth carrying.
+     */
+    if (wrote && gatesAnswered) {
+      this.recentHeartbeatMemo.set(memoKey, true, memoTtlInMs);
     }
   }
 

package/Server/Utils/Telemetry/Telemetry.ts

@@ -451,7 +451,6 @@ export default class TelemetryUtil {
     };
   }
 
-  @CaptureSpan()
   public static getAttributes(data: {
     prefixKeysWithString: string;
     items: JSONArray;
@@ -655,4 +654,116 @@ export default class TelemetryUtil {
 
     return Object.keys(attributes).sort();
   }
+
+  /*
+   * OTLP scope attributes, prefixed for row storage. The three OTel ingest
+   * services stamp every key of the scope envelope onto the row's attribute
+   * map under a `scope.` prefix. The scope object is invariant for every
+   * record/span/datapoint inside its scopeLogs/scopeSpans/scopeMetrics
+   * envelope, so callers build this map ONCE per scope and merge it per row
+   * — instead of re-deriving the prefixed key strings for every row, which
+   * was a visible per-record cost on large batches.
+   *
+   * A missing/empty scope returns an empty map, which merges to a no-op —
+   * exactly what the previous inline `if (scope && keys.length > 0)` guard
+   * produced.
+   */
+  public static getPrefixedScopeAttributes(
+    scope: JSONObject | undefined | null,
+  ): Dictionary<AttributeType | Array<AttributeType>> {
+    const prefixedScopeAttributes: Dictionary<
+      AttributeType | Array<AttributeType>
+    > = {};
+
+    if (!scope || typeof scope !== "object") {
+      return prefixedScopeAttributes;
+    }
+
+    for (const key of Object.keys(scope)) {
+      prefixedScopeAttributes[`scope.${key}`] = scope[key] as AttributeType;
+    }
+
+    return prefixedScopeAttributes;
+  }
+
+  /*
+   * Sorted attribute-key union for rows whose attribute map is "large
+   * invariant base + few per-row keys" (metric datapoints). Produces exactly
+   * `Object.keys(mergedAttributes).sort()` — same elements, same order —
+   * without re-sorting the full key set per row:
+   *
+   *   - `sortedBaseAttributeKeys` is the base map's key list, sorted once by
+   *     the caller (per metric, not per datapoint);
+   *   - `additionalAttributeKeys` is the per-row key list (typically 0-5
+   *     keys), sorted here at its own tiny cost;
+   *   - a linear two-pointer merge then yields the sorted, de-duplicated
+   *     union.
+   *
+   * Order equivalence holds because Array.prototype.sort's default
+   * comparator orders strings by UTF-16 code units, and the `<` / `>`
+   * relational operators used in the merge apply the same code-unit
+   * comparison — so merging two lists sorted by that comparator gives the
+   * same sequence a full sort of the union would.
+   *
+   * Precondition: each input list is duplicate-free (both come from
+   * Object.keys, whose output cannot repeat a key). Duplicates ACROSS the
+   * two lists (a per-row key overriding a base key) collapse to one entry,
+   * matching Object.keys on the merged map.
+   */
+  public static mergeAttributeKeysWithSortedBase(data: {
+    sortedBaseAttributeKeys: Array<string>;
+    additionalAttributeKeys: Array<string>;
+  }): Array<string> {
+    const { sortedBaseAttributeKeys, additionalAttributeKeys } = data;
+
+    /*
+     * Always return a fresh array: rows must never share an attributeKeys
+     * reference, because downstream pipeline rules replace a row's key list
+     * independently of its siblings.
+     */
+    if (additionalAttributeKeys.length === 0) {
+      return sortedBaseAttributeKeys.slice();
+    }
+
+    const sortedAdditionalKeys: Array<string> = additionalAttributeKeys
+      .slice()
+      .sort();
+
+    const mergedKeys: Array<string> = [];
+    let baseIndex: number = 0;
+    let additionalIndex: number = 0;
+
+    while (
+      baseIndex < sortedBaseAttributeKeys.length &&
+      additionalIndex < sortedAdditionalKeys.length
+    ) {
+      const baseKey: string = sortedBaseAttributeKeys[baseIndex]!;
+      const additionalKey: string = sortedAdditionalKeys[additionalIndex]!;
+
+      if (baseKey < additionalKey) {
+        mergedKeys.push(baseKey);
+        baseIndex++;
+      } else if (baseKey > additionalKey) {
+        mergedKeys.push(additionalKey);
+        additionalIndex++;
+      } else {
+        // Same key in base and per-row attributes — the map holds it once.
+        mergedKeys.push(baseKey);
+        baseIndex++;
+        additionalIndex++;
+      }
+    }
+
+    while (baseIndex < sortedBaseAttributeKeys.length) {
+      mergedKeys.push(sortedBaseAttributeKeys[baseIndex]!);
+      baseIndex++;
+    }
+
+    while (additionalIndex < sortedAdditionalKeys.length) {
+      mergedKeys.push(sortedAdditionalKeys[additionalIndex]!);
+      additionalIndex++;
+    }
+
+    return mergedKeys;
+  }
 }

package/Server/Utils/Telemetry.ts

@@ -87,6 +87,13 @@ export enum SpanStatusCode {
   ERROR = 2,
 }
 
+/*
+ * Tells a descriptive error `code` ("ECONNREFUSED") from one that is really a
+ * status number in disguise (an HTTP status, a Postgres SQLSTATE). See
+ * Telemetry.getExceptionTypeName.
+ */
+const NUMERIC_ONLY_REGEX: RegExp = /^\d+$/;
+
 export default class Telemetry {
   public static sdk: opentelemetry.NodeSDK | null = null;
 
@@ -102,6 +109,20 @@ export default class Telemetry {
 
   public static serviceName: string | null = null;
 
+  /*
+   * True only when init() installed a real OTLP span exporter. @CaptureSpan
+   * consults this to skip span creation entirely on deployments that never
+   * export traces — without an exporter the spans go nowhere, but the wrapper
+   * would still pay attribute flattening, span allocation, and an
+   * AsyncLocalStorage context switch on every decorated call, which includes
+   * hot ingest paths that run millions of times per minute.
+   */
+  private static spanExportEnabled: boolean = false;
+
+  public static isSpanExportEnabled(): boolean {
+    return this.spanExportEnabled;
+  }
+
   public static getHeaders(): Dictionary<string> {
     if (!process.env["OPENTELEMETRY_EXPORTER_OTLP_HEADERS"]) {
       return {};
@@ -192,6 +213,7 @@ export default class Telemetry {
           headers: headers,
           compression: CompressionAlgorithm.GZIP,
         }) as unknown as SpanExporter;
+        this.spanExportEnabled = true;
       }
 
       if (this.getOltpMetricsEndpoint() && hasHeaders) {
@@ -546,7 +568,36 @@ export default class Telemetry {
        * empty "Error" status.
        */
       span.setAttributes(exceptionAttributes);
-      span.recordException(exception as SpanException);
+
+      /*
+       * A NORMALIZED {name, message, stack} is handed to recordException
+       * rather than the raw thrown value, because the OTel SDK reads
+       * `exception.code` BEFORE `exception.name` when deciding
+       * `exception.type` (sdk-trace-base Span.recordException). OneUptime's
+       * Exception base class exposes an HTTP STATUS as `code`
+       * (NotAuthenticatedException = 401, BadDataException = 400,
+       * ServerException = 500 — see Types/Exception/ExceptionCode), so every
+       * exception this platform raises was being typed "401"/"400"/"500".
+       *
+       * That type is the exception event's identity downstream: it is hashed
+       * into the TelemetryException group fingerprint and it is what the
+       * Issues list and the AI insight titles show. A whole service's worth of
+       * unrelated failures collapsed into "401", telling nobody anything.
+       *
+       * The payload is built from the attributes getExceptionAttributes has
+       * already extracted, which means it inherits their truncation (4k
+       * message, 8k stack) and their crash-safety — a hostile thrown value is
+       * read exactly once, defensively, rather than twice.
+       */
+      span.recordException({
+        name: (exceptionAttributes["exception.type"] as string) || "Error",
+        message: (exceptionAttributes["exception.message"] as string) || "",
+        ...(exceptionAttributes["exception.stacktrace"]
+          ? {
+              stack: exceptionAttributes["exception.stacktrace"] as string,
+            }
+          : {}),
+      } as SpanException);
       span.setStatus({
         code: SpanStatusCode.ERROR,
         message:
@@ -588,8 +639,7 @@ export default class Telemetry {
       }
 
       if (exception instanceof Error) {
-        attributes["exception.type"] =
-          exception.name || exception.constructor?.name || "Error";
+        attributes["exception.type"] = this.getExceptionTypeName(exception);
         attributes["exception.message"] = this.truncate(
           exception.message || "",
           4000,
@@ -661,6 +711,57 @@ export default class Telemetry {
     return attributes;
   }
 
+  /*
+   * The most specific type name available for a thrown Error, in the order a
+   * human would want to read it in the Issues list.
+   *
+   * `name` is checked first but "Error" is rejected: an Error subclass that
+   * never assigns `this.name` inherits the generic "Error" from
+   * Error.prototype, and OneUptime's own Exception base does exactly that —
+   * so `name` alone reports "Error" for NotAuthenticatedException,
+   * BadDataException and every other one of them.
+   *
+   * A `code` is only used when it is a NON-NUMERIC string. That keeps the
+   * genuinely descriptive Node system codes (ECONNREFUSED, ENOTFOUND) while
+   * rejecting numeric ones — HTTP statuses on OneUptime's Exception and
+   * Postgres SQLSTATEs ("23505"), for which the constructor name
+   * (QueryFailedError) says far more. The numeric code is not lost — it keeps
+   * its own `exception.code` attribute.
+   *
+   * Never throws: it runs on the universal error path and the thrown value
+   * can be a Proxy or carry throwing getters.
+   */
+  private static getExceptionTypeName(exception: Error): string {
+    try {
+      const name: unknown = exception.name;
+      if (typeof name === "string" && name.trim() !== "" && name !== "Error") {
+        return name;
+      }
+
+      const code: unknown = (exception as { code?: unknown }).code;
+      if (
+        typeof code === "string" &&
+        code.trim() !== "" &&
+        !NUMERIC_ONLY_REGEX.test(code.trim())
+      ) {
+        return code.trim();
+      }
+
+      const constructorName: unknown = exception.constructor?.name;
+      if (
+        typeof constructorName === "string" &&
+        constructorName.trim() !== "" &&
+        constructorName !== "Object"
+      ) {
+        return constructorName;
+      }
+    } catch {
+      // Fall through to the generic name below.
+    }
+
+    return "Error";
+  }
+
   private static truncate(value: string, maxLength: number): string {
     return value.length > maxLength ? value.substring(0, maxLength) : value;
   }