@oneuptime/common 11.2.3 → 11.3.1

package/Server/Utils/AnalyticsDatabase/StatementGenerator.ts

@@ -1327,10 +1327,26 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
       );
     }
 
+    /*
+     * ON CLUSTER is appended as RAW SQL right after the table reference. The
+     * analytics schema is ALWAYS a sharded + replicated cluster (see
+     * ClusterConfig: a single node is just a "cluster of one"). A bare
+     * `ALTER … ADD COLUMN` reaches only the shard the client is connected to —
+     * Keeper replicates within a shard but never across shards — so the column
+     * lands on one shard and a later scatter-gather read through the Distributed
+     * wrapper hits a shard that lacks it and fails with
+     * "Missing columns: '<col>'" (Code 47 UNKNOWN_IDENTIFIER). ON CLUSTER also
+     * makes this ADD COLUMN wait for cluster-wide completion, so the separate
+     * `ADD INDEX` that addColumnInDatabase issues next never races a column that
+     * has not yet propagated to the node the index DDL lands on.
+     */
     const statement: Statement = SQL`
             ALTER TABLE ${this.database.getDatasourceOptions().database!}.${getStorageTableName(
               this.model.tableName,
-            )} ADD COLUMN IF NOT EXISTS `.append(columnDef);
+            )}`
+      .append(onClusterClause())
+      .append(" ADD COLUMN IF NOT EXISTS ")
+      .append(columnDef);
 
     logger.debug(`${this.model.tableName} Add Column Statement`);
     logger.debug(statement);
@@ -1359,8 +1375,14 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
 
     const databaseName: string = this.database.getDatasourceOptions().database!;
     const statement: Statement = new Statement();
+    /*
+     * ON CLUSTER (raw SQL) so the skip index is added on every shard, matching
+     * toAddColumnStatement. A bare ADD INDEX only reaches the connected shard,
+     * and if the column it references has not propagated to that node yet it
+     * fails with "Missing columns: '<col>'" (Code 47).
+     */
     statement.append(
-      `ALTER TABLE ${databaseName}.${getStorageTableName(this.model.tableName)} ADD INDEX IF NOT EXISTS ${idx.name} ${columnExpr} TYPE ${idx.type}${paramsStr} GRANULARITY ${idx.granularity}`,
+      `ALTER TABLE ${databaseName}.${getStorageTableName(this.model.tableName)}${onClusterClause()} ADD INDEX IF NOT EXISTS ${idx.name} ${columnExpr} TYPE ${idx.type}${paramsStr} GRANULARITY ${idx.granularity}`,
     );
 
     logger.debug(`${this.model.tableName} Add Skip Index Statement`);
@@ -1371,7 +1393,7 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
 
   public toDropSkipIndexStatement(indexName: string): string {
     const databaseName: string = this.database.getDatasourceOptions().database!;
-    const statement: string = `ALTER TABLE ${databaseName}.${getStorageTableName(this.model.tableName)} DROP INDEX IF EXISTS ${indexName}`;
+    const statement: string = `ALTER TABLE ${databaseName}.${getStorageTableName(this.model.tableName)}${onClusterClause()} DROP INDEX IF EXISTS ${indexName}`;
 
     logger.debug(`${this.model.tableName} Drop Skip Index Statement`);
     logger.debug(statement);
@@ -1383,7 +1405,7 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
     const statement: string = `ALTER TABLE ${this.database.getDatasourceOptions()
       .database!}.${getStorageTableName(
       this.model.tableName,
-    )} DROP COLUMN IF EXISTS ${columnName}`;
+    )}${onClusterClause()} DROP COLUMN IF EXISTS ${columnName}`;
 
     logger.debug(`${this.model.tableName} Drop Column Statement`);
     logger.debug(statement);

package/Server/Utils/Monitor/MonitorResource.ts

@@ -188,6 +188,21 @@ export default class MonitorResourceUtil {
      * below guarantees the lock is released on every exit path (return or
      * throw); acquireTimeout/retryInterval cap the acquire spin so a contended
      * lock fails fast instead of polling Redis for 10s.
+     *
+     * On acquire timeout we DO NOT continue unlocked. Falling through without
+     * the lock silently abandons the per-monitor serialization this lock exists
+     * to provide — concurrent results for the same monitor would then race
+     * incident/alert dedup and status-timeline writes (duplicate
+     * incidents/alerts, status flaps) exactly under the high contention the
+     * lock is meant to handle. Instead we surface the contention so the work is
+     * retried once the lock frees: the Telemetry queue re-runs each ingest job
+     * up to 3x with exponential backoff, and the per-monitor crons catch-and-
+     * skip to re-evaluate on their next tick. The dominant source of
+     * same-monitor contention (an external sender hammering one Incoming
+     * Request URL) is collapsed upstream by BullMQ job coalescing at enqueue
+     * time (see TelemetryQueueService.addIncomingRequestIngestJob), so this
+     * throw is a correctness backstop for the rare residual collision (e.g.
+     * cron vs ingest), not the steady-state path.
      */
     let mutex: SemaphoreMutex | null = null;
 
@@ -200,7 +215,10 @@ export default class MonitorResourceUtil {
         acquireAttemptsLimit: 20,
       });
     } catch (err) {
-      logger.error(err);
+      logger.debug(
+        `${dataToProcess.monitorId.toString()} - Could not acquire per-monitor processing lock within the acquire window; another worker is processing this monitor. Deferring to retry to preserve serialization.`,
+      );
+      throw err;
     }
 
     const releaseMutex: () => Promise<void> = async (): Promise<void> => {

package/Server/Utils/Telemetry/LlmSpan.ts

@@ -0,0 +1,251 @@
+import Dictionary from "../../../Types/Dictionary";
+import { AttributeType } from "./Telemetry";
+
+/*
+ * First-class detection of LLM / GenAI / AI-agent spans.
+ *
+ * OneUptime ingests OpenTelemetry spans generically. To make LLM and agent
+ * telemetry a first-class signal (filterable lists, token/cost/latency
+ * rollups) we denormalize a small set of values out of the span attributes at
+ * ingest time. We recognize the OpenTelemetry GenAI semantic conventions
+ * (gen_ai.*) as primary, with cheap fallbacks for the two dominant
+ * instrumentation libraries:
+ *   - OpenLLMetry / Traceloop  (gen_ai.* + traceloop.*)
+ *   - OpenInference / Arize    (llm.* + openinference.span.kind)
+ *
+ * Prompt/completion CONTENT is intentionally NOT denormalized here — it stays
+ * in the span's attributes/events map (already captured + scrubbed) and is
+ * rendered by the LLM span panel in the dashboard.
+ */
+
+export interface LlmSpanFields {
+  // True when this span looks like an LLM / GenAI / agent operation.
+  isLlmSpan: boolean;
+  // Provider / system, e.g. "openai", "anthropic", "aws.bedrock".
+  llmSystem: string;
+  // Operation, e.g. "chat", "embeddings", "execute_tool", "invoke_agent".
+  llmOperation: string;
+  // Model requested by the caller.
+  llmRequestModel: string;
+  // Model the provider actually served (often the resolved/pinned model).
+  llmResponseModel: string;
+  // Token usage. 0 when the instrumentation did not report it.
+  llmInputTokens: number;
+  llmOutputTokens: number;
+  llmTotalTokens: number;
+  // Cost in USD. Only populated when the SDK reports it (no built-in pricing).
+  llmCost: number;
+  // Agent / tool names for agent-framework spans.
+  llmAgentName: string;
+  llmToolName: string;
+}
+
+type SpanAttributes = Dictionary<AttributeType | Array<AttributeType>>;
+
+export default class LlmSpanUtil {
+  /**
+   * Return the empty/default LLM field set (non-LLM span).
+   */
+  public static empty(): LlmSpanFields {
+    return {
+      isLlmSpan: false,
+      llmSystem: "",
+      llmOperation: "",
+      llmRequestModel: "",
+      llmResponseModel: "",
+      llmInputTokens: 0,
+      llmOutputTokens: 0,
+      llmTotalTokens: 0,
+      llmCost: 0,
+      llmAgentName: "",
+      llmToolName: "",
+    };
+  }
+
+  /**
+   * Extract first-class LLM fields from a flattened span attribute dictionary.
+   * Pure + side-effect free so it can be unit tested in isolation.
+   */
+  public static extract(attributes: SpanAttributes): LlmSpanFields {
+    const fields: LlmSpanFields = this.empty();
+
+    if (!attributes || typeof attributes !== "object") {
+      return fields;
+    }
+
+    const keys: Array<string> = Object.keys(attributes);
+
+    if (keys.length === 0) {
+      return fields;
+    }
+
+    fields.llmSystem = this.getString(attributes, [
+      "gen_ai.system",
+      "gen_ai.provider.name",
+      "llm.system",
+      "llm.provider",
+    ]);
+
+    fields.llmOperation = this.getString(attributes, [
+      "gen_ai.operation.name",
+      "llm.request.type",
+      "openinference.span.kind",
+    ]);
+
+    fields.llmRequestModel = this.getString(attributes, [
+      "gen_ai.request.model",
+      "llm.model_name",
+      "llm.request.model",
+    ]);
+
+    fields.llmResponseModel = this.getString(attributes, [
+      "gen_ai.response.model",
+      "llm.response.model",
+    ]);
+
+    // Fall back to the response model when no request model was reported.
+    if (!fields.llmRequestModel && fields.llmResponseModel) {
+      fields.llmRequestModel = fields.llmResponseModel;
+    }
+
+    /*
+     * Token columns are ClickHouse Int32 — truncate any fractional value a
+     * malformed SDK might report, otherwise the JSONEachRow insert would
+     * reject the row and fail the whole span batch.
+     */
+    fields.llmInputTokens = Math.trunc(
+      this.getNumber(attributes, [
+        "gen_ai.usage.input_tokens",
+        "gen_ai.usage.prompt_tokens",
+        "llm.token_count.prompt",
+        "llm.usage.prompt_tokens",
+      ]),
+    );
+
+    fields.llmOutputTokens = Math.trunc(
+      this.getNumber(attributes, [
+        "gen_ai.usage.output_tokens",
+        "gen_ai.usage.completion_tokens",
+        "llm.token_count.completion",
+        "llm.usage.completion_tokens",
+      ]),
+    );
+
+    fields.llmTotalTokens = Math.trunc(
+      this.getNumber(attributes, [
+        "gen_ai.usage.total_tokens",
+        "llm.token_count.total",
+        "llm.usage.total_tokens",
+      ]),
+    );
+
+    // Derive total when only the parts were reported.
+    if (
+      fields.llmTotalTokens === 0 &&
+      (fields.llmInputTokens > 0 || fields.llmOutputTokens > 0)
+    ) {
+      fields.llmTotalTokens = fields.llmInputTokens + fields.llmOutputTokens;
+    }
+
+    fields.llmCost = this.getNumber(attributes, [
+      "gen_ai.usage.cost",
+      "gen_ai.usage.cost_usd",
+      "gen_ai.usage.total_cost",
+      "llm.usage.total_cost",
+    ]);
+
+    fields.llmAgentName = this.getString(attributes, [
+      "gen_ai.agent.name",
+      "agent.name",
+    ]);
+
+    fields.llmToolName = this.getString(attributes, [
+      "gen_ai.tool.name",
+      "tool.name",
+    ]);
+
+    fields.isLlmSpan = this.detectIsLlmSpan(keys, fields);
+
+    return fields;
+  }
+
+  private static detectIsLlmSpan(
+    keys: Array<string>,
+    fields: LlmSpanFields,
+  ): boolean {
+    if (
+      fields.llmSystem ||
+      fields.llmOperation ||
+      fields.llmRequestModel ||
+      fields.llmResponseModel ||
+      fields.llmAgentName ||
+      fields.llmToolName ||
+      fields.llmTotalTokens > 0
+    ) {
+      return true;
+    }
+
+    // Last-resort: any GenAI/LLM-namespaced attribute at all.
+    return keys.some((key: string) => {
+      return (
+        key.startsWith("gen_ai.") ||
+        key.startsWith("llm.") ||
+        key.startsWith("traceloop.")
+      );
+    });
+  }
+
+  private static getString(
+    attributes: SpanAttributes,
+    candidateKeys: Array<string>,
+  ): string {
+    for (const key of candidateKeys) {
+      const value: AttributeType | Array<AttributeType> | undefined =
+        attributes[key];
+
+      if (value === undefined || value === null) {
+        continue;
+      }
+
+      if (Array.isArray(value)) {
+        continue;
+      }
+
+      const stringValue: string = String(value).trim();
+
+      if (stringValue) {
+        return stringValue;
+      }
+    }
+
+    return "";
+  }
+
+  private static getNumber(
+    attributes: SpanAttributes,
+    candidateKeys: Array<string>,
+  ): number {
+    for (const key of candidateKeys) {
+      const value: AttributeType | Array<AttributeType> | undefined =
+        attributes[key];
+
+      if (value === undefined || value === null || Array.isArray(value)) {
+        continue;
+      }
+
+      if (typeof value === "number" && isFinite(value)) {
+        return value;
+      }
+
+      if (typeof value === "string" && value.trim() !== "") {
+        const parsed: number = Number(value);
+
+        if (isFinite(parsed)) {
+          return parsed;
+        }
+      }
+    }
+
+    return 0;
+  }
+}

package/Server/Utils/Telemetry.ts

@@ -524,15 +524,162 @@ export default class Telemetry {
   }): void {
     const { span, exception } = data;
 
-    // log the exception as well
-    logger.error(exception);
+    /*
+     * This runs on the universal catch path of every @CaptureSpan-decorated
+     * function, so it must NEVER throw — a throw here would mask the original
+     * error and skip marking the span. The whole body is wrapped defensively
+     * and the span is always marked-as-error and ended (in the finally).
+     */
+    try {
+      const exceptionAttributes: Attributes =
+        this.getExceptionAttributes(exception);
+
+      // log the exception as well
+      logger.error(exception);
 
-    span.recordException(exception as SpanException);
-    span.setStatus({
-      code: SpanStatusCode.ERROR,
-    });
+      /*
+       * Span *events* (from recordException) are not reliably surfaced when the
+       * span is read back, and setStatus on its own only records the error CODE,
+       * not the message. So we also attach the exception details as queryable
+       * span attributes — including DB driver fields like the failing constraint
+       * and table — so the actual cause is visible in the trace UI instead of an
+       * empty "Error" status.
+       */
+      span.setAttributes(exceptionAttributes);
+      span.recordException(exception as SpanException);
+      span.setStatus({
+        code: SpanStatusCode.ERROR,
+        message:
+          (exceptionAttributes["exception.message"] as string | undefined) ||
+          "Error",
+      });
+    } catch {
+      // Enrichment failed on some exotic thrown value — still flag the span.
+      try {
+        span.setStatus({ code: SpanStatusCode.ERROR });
+      } catch {
+        // span may already be ended; nothing more we can do.
+      }
+    } finally {
+      this.endSpan(span);
+    }
+  }
+
+  /*
+   * Pulls every useful field off an unknown thrown value into OpenTelemetry
+   * span attributes. Error message/stack are non-enumerable, so they are read
+   * explicitly. For database failures (TypeORM QueryFailedError / pg errors),
+   * the Postgres fields (SQLSTATE code, detail, constraint, table, column) live
+   * either on the error itself or on `driverError`; these are what tell us which
+   * constraint failed during e.g. a cascade delete.
+   *
+   * The thrown value is `unknown`, so anything (a Proxy, an object with a
+   * throwing getter or a throwing toString) could be passed — every field read
+   * and string coercion is guarded so this can never throw on the error path.
+   */
+  private static getExceptionAttributes(exception: unknown): Attributes {
+    const attributes: Attributes = {};
+
+    try {
+      if (exception === null || exception === undefined) {
+        attributes["exception.message"] =
+          "Unknown error: null or undefined was thrown";
+        return attributes;
+      }
+
+      if (exception instanceof Error) {
+        attributes["exception.type"] =
+          exception.name || exception.constructor?.name || "Error";
+        attributes["exception.message"] = this.truncate(
+          exception.message || "",
+          4000,
+        );
+        if (exception.stack) {
+          attributes["exception.stacktrace"] = this.truncate(
+            exception.stack,
+            8000,
+          );
+        }
+      } else if (typeof exception === "string") {
+        attributes["exception.message"] = this.truncate(exception, 4000);
+      } else {
+        attributes["exception.message"] = this.truncate(
+          this.safeStringify(exception),
+          4000,
+        );
+      }
 
-    this.endSpan(span);
+      type PotentialDatabaseError = {
+        code?: unknown;
+        detail?: unknown;
+        constraint?: unknown;
+        table?: unknown;
+        column?: unknown;
+        schema?: unknown;
+        query?: unknown;
+        driverError?: PotentialDatabaseError;
+      };
+
+      const error: PotentialDatabaseError = exception as PotentialDatabaseError;
+      const databaseError: PotentialDatabaseError = error.driverError || error;
+
+      const setStringAttribute: (key: string, value: unknown) => void = (
+        key: string,
+        value: unknown,
+      ): void => {
+        try {
+          if (value !== undefined && value !== null && value !== "") {
+            attributes[key] = this.truncate(String(value), 2000);
+          }
+        } catch {
+          // A single unserializable field must not abort the whole enrichment.
+        }
+      };
+
+      // SQLSTATE (e.g. "23503" = foreign key violation) or a Node error code.
+      setStringAttribute("exception.code", error.code ?? databaseError.code);
+      setStringAttribute("db.error.detail", databaseError.detail);
+      setStringAttribute("db.error.constraint", databaseError.constraint);
+      setStringAttribute("db.error.table", databaseError.table);
+      setStringAttribute("db.error.column", databaseError.column);
+      setStringAttribute("db.error.schema", databaseError.schema);
+
+      if (typeof error.query === "string" && error.query.length > 0) {
+        attributes["db.statement"] = this.truncate(error.query, 2000);
+      }
+    } catch {
+      /*
+       * Reading exotic thrown values (throwing getters, Proxies) must never
+       * crash the error-reporting path. Keep whatever was collected so far.
+       */
+      if (!attributes["exception.message"]) {
+        attributes["exception.message"] =
+          "Error (exception details could not be extracted)";
+      }
+    }
+
+    return attributes;
+  }
+
+  private static truncate(value: string, maxLength: number): string {
+    return value.length > maxLength ? value.substring(0, maxLength) : value;
+  }
+
+  private static safeStringify(value: unknown): string {
+    try {
+      const serialized: string | undefined = JSON.stringify(value);
+      if (serialized) {
+        return serialized;
+      }
+    } catch {
+      // fall through to String() below
+    }
+
+    try {
+      return String(value);
+    } catch {
+      return "[unserializable error value]";
+    }
   }
 
   public static endSpan(span: Span): void {

package/Tests/Server/Utils/AnalyticsDatabase/ClusterAwareSchema.test.ts

@@ -232,20 +232,33 @@ describe("ClickHouse cluster-aware schema (always-on)", () => {
       );
     });
 
-    test("schema ALTERs target the local table", () => {
+    test("schema ALTERs target the local table ON CLUSTER", () => {
       const column: AnalyticsTableColumn = new SpanModel().tableColumns[1]!;
-      expect(fullText(spanGen.toAddColumnStatement(column))).toContain(
-        "SpanItemV3Local",
-      );
-      expect(fullText(spanGen.toAddSkipIndexStatement(column)!)).toContain(
-        "SpanItemV3Local",
-      );
-      expect(spanGen.toDropColumnStatement("traceId")).toContain(
-        "SpanItemV3Local",
-      );
-      expect(spanGen.toDropSkipIndexStatement("idx_trace_id")).toContain(
-        "SpanItemV3Local",
+
+      /*
+       * Column / skip-index ADD + DROP must carry ON CLUSTER: the local storage
+       * table is Replicated per shard, and Keeper only replicates within a
+       * shard. Without ON CLUSTER a reconciled column/index lands on a single
+       * shard, so a scatter-gather read through the Distributed wrapper hits a
+       * shard that lacks it and fails with "Missing columns" (Code 47).
+       */
+      const addColumn: string = fullText(spanGen.toAddColumnStatement(column));
+      expect(addColumn).toContain("SpanItemV3Local");
+      expect(addColumn).toContain("ON CLUSTER 'oneuptime'");
+
+      const addIndex: string = fullText(
+        spanGen.toAddSkipIndexStatement(column)!,
       );
+      expect(addIndex).toContain("SpanItemV3Local");
+      expect(addIndex).toContain("ON CLUSTER 'oneuptime'");
+
+      const dropColumn: string = spanGen.toDropColumnStatement("traceId");
+      expect(dropColumn).toContain("SpanItemV3Local");
+      expect(dropColumn).toContain("ON CLUSTER 'oneuptime'");
+
+      const dropIndex: string = spanGen.toDropSkipIndexStatement("idx_trace_id");
+      expect(dropIndex).toContain("SpanItemV3Local");
+      expect(dropIndex).toContain("ON CLUSTER 'oneuptime'");
     });
 
     test("ALTER UPDATE mutates the local table with ON CLUSTER", () => {