@oneuptime/common 11.2.2 → 11.3.0

package/Models/AnalyticsModels/Span.ts

@@ -893,6 +893,192 @@ export default class Span extends AnalyticsBaseModel {
       },
     });
 
+    /*
+     * First-class LLM / GenAI / AI-agent columns. Denormalized at ingest from
+     * gen_ai.* (and OpenLLMetry/OpenInference) span attributes so LLM calls can
+     * be listed, filtered, and aggregated (tokens/cost/latency) cheaply without
+     * scanning the attributes map. See Common/Server/Utils/Telemetry/LlmSpan.ts.
+     */
+    const llmColumnAccessControl: {
+      read: Array<Permission>;
+      create: Array<Permission>;
+      update: Array<Permission>;
+    } = {
+      read: [
+        Permission.ProjectOwner,
+        Permission.ProjectAdmin,
+        Permission.ProjectMember,
+        Permission.Viewer,
+        Permission.TelemetryAdmin,
+        Permission.TelemetryMember,
+        Permission.TelemetryViewer,
+        Permission.ReadTelemetryServiceTraces,
+      ],
+      create: [
+        Permission.ProjectOwner,
+        Permission.ProjectAdmin,
+        Permission.ProjectMember,
+        Permission.TelemetryAdmin,
+        Permission.TelemetryMember,
+        Permission.CreateTelemetryServiceTraces,
+      ],
+      update: [],
+    };
+
+    const isLlmSpanColumn: AnalyticsTableColumn = new AnalyticsTableColumn({
+      key: "isLlmSpan",
+      title: "Is LLM Span",
+      description:
+        "Whether this span is an LLM / GenAI / AI-agent operation, populated at ingest time for fast filtering of AI calls",
+      required: true,
+      defaultValue: false,
+      type: TableColumnType.Boolean,
+      skipIndex: {
+        name: "idx_is_llm_span",
+        type: SkipIndexType.Set,
+        params: [2],
+        granularity: 4,
+      },
+      accessControl: llmColumnAccessControl,
+    });
+
+    const llmSystemColumn: AnalyticsTableColumn = new AnalyticsTableColumn({
+      key: "llmSystem",
+      isLowCardinality: true,
+      title: "LLM System",
+      description:
+        "GenAI provider / system, e.g. openai, anthropic, aws.bedrock (gen_ai.system)",
+      required: false,
+      type: TableColumnType.Text,
+      skipIndex: {
+        name: "idx_llm_system",
+        type: SkipIndexType.Set,
+        params: [100],
+        granularity: 4,
+      },
+      accessControl: llmColumnAccessControl,
+    });
+
+    const llmOperationColumn: AnalyticsTableColumn = new AnalyticsTableColumn({
+      key: "llmOperation",
+      isLowCardinality: true,
+      title: "LLM Operation",
+      description:
+        "GenAI operation, e.g. chat, embeddings, execute_tool, invoke_agent (gen_ai.operation.name)",
+      required: false,
+      type: TableColumnType.Text,
+      skipIndex: {
+        name: "idx_llm_operation",
+        type: SkipIndexType.Set,
+        params: [100],
+        granularity: 4,
+      },
+      accessControl: llmColumnAccessControl,
+    });
+
+    const llmRequestModelColumn: AnalyticsTableColumn =
+      new AnalyticsTableColumn({
+        key: "llmRequestModel",
+        isLowCardinality: true,
+        title: "LLM Request Model",
+        description: "Model requested by the caller (gen_ai.request.model)",
+        required: false,
+        type: TableColumnType.Text,
+        skipIndex: {
+          name: "idx_llm_request_model",
+          type: SkipIndexType.Set,
+          params: [1000],
+          granularity: 4,
+        },
+        accessControl: llmColumnAccessControl,
+      });
+
+    const llmResponseModelColumn: AnalyticsTableColumn =
+      new AnalyticsTableColumn({
+        key: "llmResponseModel",
+        isLowCardinality: true,
+        title: "LLM Response Model",
+        description:
+          "Model the provider actually served (gen_ai.response.model)",
+        required: false,
+        type: TableColumnType.Text,
+        accessControl: llmColumnAccessControl,
+      });
+
+    const llmAgentNameColumn: AnalyticsTableColumn = new AnalyticsTableColumn({
+      key: "llmAgentName",
+      codec: { codec: "ZSTD", level: 1 },
+      title: "LLM Agent Name",
+      description:
+        "Name of the agent for agent-framework spans (gen_ai.agent.name)",
+      required: false,
+      type: TableColumnType.Text,
+      accessControl: llmColumnAccessControl,
+    });
+
+    const llmToolNameColumn: AnalyticsTableColumn = new AnalyticsTableColumn({
+      key: "llmToolName",
+      codec: { codec: "ZSTD", level: 1 },
+      title: "LLM Tool Name",
+      description:
+        "Name of the tool invoked for tool-call spans (gen_ai.tool.name)",
+      required: false,
+      type: TableColumnType.Text,
+      accessControl: llmColumnAccessControl,
+    });
+
+    const llmInputTokensColumn: AnalyticsTableColumn = new AnalyticsTableColumn(
+      {
+        key: "llmInputTokens",
+        title: "LLM Input Tokens",
+        description: "Input/prompt token count (gen_ai.usage.input_tokens)",
+        required: true,
+        defaultValue: 0,
+        type: TableColumnType.Number,
+        accessControl: llmColumnAccessControl,
+      },
+    );
+
+    const llmOutputTokensColumn: AnalyticsTableColumn =
+      new AnalyticsTableColumn({
+        key: "llmOutputTokens",
+        title: "LLM Output Tokens",
+        description:
+          "Output/completion token count (gen_ai.usage.output_tokens)",
+        required: true,
+        defaultValue: 0,
+        type: TableColumnType.Number,
+        accessControl: llmColumnAccessControl,
+      });
+
+    const llmTotalTokensColumn: AnalyticsTableColumn = new AnalyticsTableColumn(
+      {
+        key: "llmTotalTokens",
+        title: "LLM Total Tokens",
+        description:
+          "Total token count (gen_ai.usage.total_tokens, or input + output)",
+        required: true,
+        defaultValue: 0,
+        type: TableColumnType.Number,
+        accessControl: llmColumnAccessControl,
+      },
+    );
+
+    const llmCostColumn: AnalyticsTableColumn = new AnalyticsTableColumn({
+      key: "llmCost",
+      title: "LLM Cost (USD)",
+      description:
+        "Cost of this LLM call in USD. Only populated when the instrumentation reports it (gen_ai.usage.cost); OneUptime does not compute pricing.",
+      required: true,
+      defaultValue: 0,
+      /*
+       * Decimal (ClickHouse Double) — cost is fractional; an Int column would
+       * reject the fractional JSONEachRow value and poison the whole batch.
+       */
+      type: TableColumnType.Decimal,
+      accessControl: llmColumnAccessControl,
+    });
+
     const retentionDateColumn: AnalyticsTableColumn = new AnalyticsTableColumn({
       key: "retentionDate",
       codec: [{ codec: "DoubleDelta" }, { codec: "ZSTD", level: 1 }],
@@ -974,6 +1160,17 @@ export default class Span extends AnalyticsBaseModel {
         kindColumn,
         hasExceptionColumn,
         isRootSpanColumn,
+        isLlmSpanColumn,
+        llmSystemColumn,
+        llmOperationColumn,
+        llmRequestModelColumn,
+        llmResponseModelColumn,
+        llmAgentNameColumn,
+        llmToolNameColumn,
+        llmInputTokensColumn,
+        llmOutputTokensColumn,
+        llmTotalTokensColumn,
+        llmCostColumn,
         retentionDateColumn,
       ],
       projections: [
@@ -1199,4 +1396,92 @@ export default class Span extends AnalyticsBaseModel {
   public set retentionDate(v: Date | undefined) {
     this.setColumnValue("retentionDate", v);
   }
+
+  public get isLlmSpan(): boolean | undefined {
+    return this.getColumnValue("isLlmSpan") as boolean | undefined;
+  }
+
+  public set isLlmSpan(v: boolean | undefined) {
+    this.setColumnValue("isLlmSpan", v);
+  }
+
+  public get llmSystem(): string | undefined {
+    return this.getColumnValue("llmSystem") as string | undefined;
+  }
+
+  public set llmSystem(v: string | undefined) {
+    this.setColumnValue("llmSystem", v);
+  }
+
+  public get llmOperation(): string | undefined {
+    return this.getColumnValue("llmOperation") as string | undefined;
+  }
+
+  public set llmOperation(v: string | undefined) {
+    this.setColumnValue("llmOperation", v);
+  }
+
+  public get llmRequestModel(): string | undefined {
+    return this.getColumnValue("llmRequestModel") as string | undefined;
+  }
+
+  public set llmRequestModel(v: string | undefined) {
+    this.setColumnValue("llmRequestModel", v);
+  }
+
+  public get llmResponseModel(): string | undefined {
+    return this.getColumnValue("llmResponseModel") as string | undefined;
+  }
+
+  public set llmResponseModel(v: string | undefined) {
+    this.setColumnValue("llmResponseModel", v);
+  }
+
+  public get llmAgentName(): string | undefined {
+    return this.getColumnValue("llmAgentName") as string | undefined;
+  }
+
+  public set llmAgentName(v: string | undefined) {
+    this.setColumnValue("llmAgentName", v);
+  }
+
+  public get llmToolName(): string | undefined {
+    return this.getColumnValue("llmToolName") as string | undefined;
+  }
+
+  public set llmToolName(v: string | undefined) {
+    this.setColumnValue("llmToolName", v);
+  }
+
+  public get llmInputTokens(): number | undefined {
+    return this.getColumnValue("llmInputTokens") as number | undefined;
+  }
+
+  public set llmInputTokens(v: number | undefined) {
+    this.setColumnValue("llmInputTokens", v);
+  }
+
+  public get llmOutputTokens(): number | undefined {
+    return this.getColumnValue("llmOutputTokens") as number | undefined;
+  }
+
+  public set llmOutputTokens(v: number | undefined) {
+    this.setColumnValue("llmOutputTokens", v);
+  }
+
+  public get llmTotalTokens(): number | undefined {
+    return this.getColumnValue("llmTotalTokens") as number | undefined;
+  }
+
+  public set llmTotalTokens(v: number | undefined) {
+    this.setColumnValue("llmTotalTokens", v);
+  }
+
+  public get llmCost(): number | undefined {
+    return this.getColumnValue("llmCost") as number | undefined;
+  }
+
+  public set llmCost(v: number | undefined) {
+    this.setColumnValue("llmCost", v);
+  }
 }

package/Server/Infrastructure/Queue.ts

@@ -219,6 +219,19 @@ export default class Queue {
        * timestamp.
        */
       skipExistenceCheck?: boolean | undefined;
+      /**
+       * Coalesce same-key jobs so they are never processed in parallel.
+       * When set, BullMQ keeps at most one active + one waiting job per
+       * `deduplication.id`: additional adds while one is active collapse into
+       * the single waiting slot, keeping only the latest job data
+       * (keepLastIfActive). Used by the incoming-request ingest path to stop
+       * an external sender hammering one monitor's URL from fanning out into
+       * many concurrent same-monitor jobs that contend on the per-monitor
+       * lock. Independent of `jobId`, which stays unique.
+       */
+      deduplication?:
+        | { id: string; keepLastIfActive?: boolean | undefined }
+        | undefined;
     },
   ): Promise<Job> {
     const sanitizedJobId: string = this.sanitizeJobId(jobId.toString());
@@ -242,6 +255,13 @@ export default class Queue {
       };
     }
 
+    if (options?.deduplication?.id) {
+      optionsObject.deduplication = {
+        id: this.sanitizeJobId(options.deduplication.id),
+        keepLastIfActive: options.deduplication.keepLastIfActive ?? false,
+      };
+    }
+
     const queue: BullQueue = this.getQueue(queueName);
 
     if (options && options.scheduleAt) {

package/Server/Utils/Monitor/MonitorResource.ts

@@ -188,6 +188,21 @@ export default class MonitorResourceUtil {
      * below guarantees the lock is released on every exit path (return or
      * throw); acquireTimeout/retryInterval cap the acquire spin so a contended
      * lock fails fast instead of polling Redis for 10s.
+     *
+     * On acquire timeout we DO NOT continue unlocked. Falling through without
+     * the lock silently abandons the per-monitor serialization this lock exists
+     * to provide — concurrent results for the same monitor would then race
+     * incident/alert dedup and status-timeline writes (duplicate
+     * incidents/alerts, status flaps) exactly under the high contention the
+     * lock is meant to handle. Instead we surface the contention so the work is
+     * retried once the lock frees: the Telemetry queue re-runs each ingest job
+     * up to 3x with exponential backoff, and the per-monitor crons catch-and-
+     * skip to re-evaluate on their next tick. The dominant source of
+     * same-monitor contention (an external sender hammering one Incoming
+     * Request URL) is collapsed upstream by BullMQ job coalescing at enqueue
+     * time (see TelemetryQueueService.addIncomingRequestIngestJob), so this
+     * throw is a correctness backstop for the rare residual collision (e.g.
+     * cron vs ingest), not the steady-state path.
      */
     let mutex: SemaphoreMutex | null = null;
 
@@ -200,7 +215,10 @@ export default class MonitorResourceUtil {
         acquireAttemptsLimit: 20,
       });
     } catch (err) {
-      logger.error(err);
+      logger.debug(
+        `${dataToProcess.monitorId.toString()} - Could not acquire per-monitor processing lock within the acquire window; another worker is processing this monitor. Deferring to retry to preserve serialization.`,
+      );
+      throw err;
     }
 
     const releaseMutex: () => Promise<void> = async (): Promise<void> => {

package/Server/Utils/Telemetry/LlmSpan.ts

@@ -0,0 +1,251 @@
+import Dictionary from "../../../Types/Dictionary";
+import { AttributeType } from "./Telemetry";
+
+/*
+ * First-class detection of LLM / GenAI / AI-agent spans.
+ *
+ * OneUptime ingests OpenTelemetry spans generically. To make LLM and agent
+ * telemetry a first-class signal (filterable lists, token/cost/latency
+ * rollups) we denormalize a small set of values out of the span attributes at
+ * ingest time. We recognize the OpenTelemetry GenAI semantic conventions
+ * (gen_ai.*) as primary, with cheap fallbacks for the two dominant
+ * instrumentation libraries:
+ *   - OpenLLMetry / Traceloop  (gen_ai.* + traceloop.*)
+ *   - OpenInference / Arize    (llm.* + openinference.span.kind)
+ *
+ * Prompt/completion CONTENT is intentionally NOT denormalized here — it stays
+ * in the span's attributes/events map (already captured + scrubbed) and is
+ * rendered by the LLM span panel in the dashboard.
+ */
+
+export interface LlmSpanFields {
+  // True when this span looks like an LLM / GenAI / agent operation.
+  isLlmSpan: boolean;
+  // Provider / system, e.g. "openai", "anthropic", "aws.bedrock".
+  llmSystem: string;
+  // Operation, e.g. "chat", "embeddings", "execute_tool", "invoke_agent".
+  llmOperation: string;
+  // Model requested by the caller.
+  llmRequestModel: string;
+  // Model the provider actually served (often the resolved/pinned model).
+  llmResponseModel: string;
+  // Token usage. 0 when the instrumentation did not report it.
+  llmInputTokens: number;
+  llmOutputTokens: number;
+  llmTotalTokens: number;
+  // Cost in USD. Only populated when the SDK reports it (no built-in pricing).
+  llmCost: number;
+  // Agent / tool names for agent-framework spans.
+  llmAgentName: string;
+  llmToolName: string;
+}
+
+type SpanAttributes = Dictionary<AttributeType | Array<AttributeType>>;
+
+export default class LlmSpanUtil {
+  /**
+   * Return the empty/default LLM field set (non-LLM span).
+   */
+  public static empty(): LlmSpanFields {
+    return {
+      isLlmSpan: false,
+      llmSystem: "",
+      llmOperation: "",
+      llmRequestModel: "",
+      llmResponseModel: "",
+      llmInputTokens: 0,
+      llmOutputTokens: 0,
+      llmTotalTokens: 0,
+      llmCost: 0,
+      llmAgentName: "",
+      llmToolName: "",
+    };
+  }
+
+  /**
+   * Extract first-class LLM fields from a flattened span attribute dictionary.
+   * Pure + side-effect free so it can be unit tested in isolation.
+   */
+  public static extract(attributes: SpanAttributes): LlmSpanFields {
+    const fields: LlmSpanFields = this.empty();
+
+    if (!attributes || typeof attributes !== "object") {
+      return fields;
+    }
+
+    const keys: Array<string> = Object.keys(attributes);
+
+    if (keys.length === 0) {
+      return fields;
+    }
+
+    fields.llmSystem = this.getString(attributes, [
+      "gen_ai.system",
+      "gen_ai.provider.name",
+      "llm.system",
+      "llm.provider",
+    ]);
+
+    fields.llmOperation = this.getString(attributes, [
+      "gen_ai.operation.name",
+      "llm.request.type",
+      "openinference.span.kind",
+    ]);
+
+    fields.llmRequestModel = this.getString(attributes, [
+      "gen_ai.request.model",
+      "llm.model_name",
+      "llm.request.model",
+    ]);
+
+    fields.llmResponseModel = this.getString(attributes, [
+      "gen_ai.response.model",
+      "llm.response.model",
+    ]);
+
+    // Fall back to the response model when no request model was reported.
+    if (!fields.llmRequestModel && fields.llmResponseModel) {
+      fields.llmRequestModel = fields.llmResponseModel;
+    }
+
+    /*
+     * Token columns are ClickHouse Int32 — truncate any fractional value a
+     * malformed SDK might report, otherwise the JSONEachRow insert would
+     * reject the row and fail the whole span batch.
+     */
+    fields.llmInputTokens = Math.trunc(
+      this.getNumber(attributes, [
+        "gen_ai.usage.input_tokens",
+        "gen_ai.usage.prompt_tokens",
+        "llm.token_count.prompt",
+        "llm.usage.prompt_tokens",
+      ]),
+    );
+
+    fields.llmOutputTokens = Math.trunc(
+      this.getNumber(attributes, [
+        "gen_ai.usage.output_tokens",
+        "gen_ai.usage.completion_tokens",
+        "llm.token_count.completion",
+        "llm.usage.completion_tokens",
+      ]),
+    );
+
+    fields.llmTotalTokens = Math.trunc(
+      this.getNumber(attributes, [
+        "gen_ai.usage.total_tokens",
+        "llm.token_count.total",
+        "llm.usage.total_tokens",
+      ]),
+    );
+
+    // Derive total when only the parts were reported.
+    if (
+      fields.llmTotalTokens === 0 &&
+      (fields.llmInputTokens > 0 || fields.llmOutputTokens > 0)
+    ) {
+      fields.llmTotalTokens = fields.llmInputTokens + fields.llmOutputTokens;
+    }
+
+    fields.llmCost = this.getNumber(attributes, [
+      "gen_ai.usage.cost",
+      "gen_ai.usage.cost_usd",
+      "gen_ai.usage.total_cost",
+      "llm.usage.total_cost",
+    ]);
+
+    fields.llmAgentName = this.getString(attributes, [
+      "gen_ai.agent.name",
+      "agent.name",
+    ]);
+
+    fields.llmToolName = this.getString(attributes, [
+      "gen_ai.tool.name",
+      "tool.name",
+    ]);
+
+    fields.isLlmSpan = this.detectIsLlmSpan(keys, fields);
+
+    return fields;
+  }
+
+  private static detectIsLlmSpan(
+    keys: Array<string>,
+    fields: LlmSpanFields,
+  ): boolean {
+    if (
+      fields.llmSystem ||
+      fields.llmOperation ||
+      fields.llmRequestModel ||
+      fields.llmResponseModel ||
+      fields.llmAgentName ||
+      fields.llmToolName ||
+      fields.llmTotalTokens > 0
+    ) {
+      return true;
+    }
+
+    // Last-resort: any GenAI/LLM-namespaced attribute at all.
+    return keys.some((key: string) => {
+      return (
+        key.startsWith("gen_ai.") ||
+        key.startsWith("llm.") ||
+        key.startsWith("traceloop.")
+      );
+    });
+  }
+
+  private static getString(
+    attributes: SpanAttributes,
+    candidateKeys: Array<string>,
+  ): string {
+    for (const key of candidateKeys) {
+      const value: AttributeType | Array<AttributeType> | undefined =
+        attributes[key];
+
+      if (value === undefined || value === null) {
+        continue;
+      }
+
+      if (Array.isArray(value)) {
+        continue;
+      }
+
+      const stringValue: string = String(value).trim();
+
+      if (stringValue) {
+        return stringValue;
+      }
+    }
+
+    return "";
+  }
+
+  private static getNumber(
+    attributes: SpanAttributes,
+    candidateKeys: Array<string>,
+  ): number {
+    for (const key of candidateKeys) {
+      const value: AttributeType | Array<AttributeType> | undefined =
+        attributes[key];
+
+      if (value === undefined || value === null || Array.isArray(value)) {
+        continue;
+      }
+
+      if (typeof value === "number" && isFinite(value)) {
+        return value;
+      }
+
+      if (typeof value === "string" && value.trim() !== "") {
+        const parsed: number = Number(value);
+
+        if (isFinite(parsed)) {
+          return parsed;
+        }
+      }
+    }
+
+    return 0;
+  }
+}