@oneuptime/common 12.0.23 → 12.0.25

package/Server/Utils/StartServer.ts

@@ -9,6 +9,7 @@ import {
 } from "../EnvironmentConfig";
 import LocalCache from "../Infrastructure/LocalCache";
 import HttpMetricsMiddleware from "../Middleware/HttpMetricsMiddleware";
+import GzipRequestBodyMiddleware from "../Middleware/GzipRequestBody";
 import CorsOptions, {
   CORS_EXPOSED_HEADERS,
   CORS_PREFLIGHT_MAX_AGE_SECONDS,
@@ -41,7 +42,6 @@ import StatusCode from "../../Types/API/StatusCode";
 import HTTPErrorResponse from "../../Types/API/HTTPErrorResponse";
 import Exception from "../../Types/Exception/Exception";
 import NotFoundException from "../../Types/Exception/NotFoundException";
-import ServerException from "../../Types/Exception/ServerException";
 import { PromiseVoidFunction } from "../../Types/FunctionTypes";
 import { JSONObject } from "../../Types/JSON";
 import JSONFunctions from "../../Types/JSONFunctions";
@@ -49,7 +49,6 @@ import Port from "../../Types/Port";
 import Typeof from "../../Types/Typeof";
 import CookieParser from "cookie-parser";
 import cors from "cors";
-import zlib from "zlib";
 import crypto from "crypto";
 import path from "path";
 import "ejs";
@@ -206,10 +205,11 @@ app.use((req: OneUptimeRequest, res: ExpressResponse, next: NextFunction) => {
    * several prefixes, so /telemetry/otlp/v1/... and
    * /telemetry/session-replay/v1/... are equally live. A prefix-anchored
    * test would let the prefixed path fall into the gzip fast-path below,
-   * which has NO size limit and no content-length pre-check (a zip-bomb
-   * vector on a public browser endpoint) and which sets req.body to the
-   * DECOMPRESSED buffer — that in turn trips the ingest middleware's
-   * "already parsed" early-out, so its byte cap would never run at all.
+   * which sets req.body to the DECOMPRESSED buffer — that in turn trips
+   * the ingest middleware's "already parsed" early-out, so its own,
+   * tighter, byte cap would never run at all. (The fast path has its own
+   * limits since GHSA-cp58-wc9q-qv53, but they are the generic 50 MiB
+   * body-parser numbers, not the 4 MiB these routes are sized for.)
    */
   if (
     req.path.includes("/otlp/v1/") ||
@@ -226,32 +226,12 @@ app.use((req: OneUptimeRequest, res: ExpressResponse, next: NextFunction) => {
   );
 
   if (contentEncoding?.includes("gzip")) {
-    const buffers: any = [];
-
-    req.on("data", (chunk: any) => {
-      buffers.push(chunk);
-    });
-
-    req.on("end", () => {
-      const buffer: Buffer = Buffer.concat(buffers);
-      zlib.gunzip(buffer as Uint8Array, (err: unknown, decoded: Buffer) => {
-        if (err) {
-          logger.error(
-            err,
-            getLogAttributesFromRequest(req as OneUptimeRequest),
-          );
-          return Response.sendErrorResponse(
-            req,
-            res,
-            new ServerException("Error decompressing data"),
-          );
-        }
-
-        req.body = decoded;
-
-        next();
-      });
-    });
+    /*
+     * Bounded on both sides - see GzipRequestBody. This used to buffer the
+     * whole compressed body and hand it to an unbounded zlib.gunzip, which
+     * turned 130 KB of anonymous request into 128 MiB of resident Buffer.
+     */
+    GzipRequestBodyMiddleware.parseBody(req, res, next);
   } else if (
     contentType &&
     (contentType.includes("application/x-protobuf") ||

package/Server/Utils/Telemetry/LlmCostBudgetEvaluator.ts

@@ -19,14 +19,22 @@ import MetricService from "../../Services/MetricService";
 import SpanService from "../../Services/SpanService";
 import logger, { LogAttributes } from "../Logger";
 import CaptureSpan from "../Telemetry/CaptureSpan";
+import LlmMetricSpend, { LlmMetricScope } from "./LlmMetricSpend";
 import TelemetryUtil from "./Telemetry";
 
 /*
- * Evaluates LLM cost budgets against the day's LLM span spend.
+ * Evaluates LLM cost budgets against the day's LLM spend.
+ *
+ * Spend is sourced from GenAI trace spans first and, when a scope produced no
+ * span spend at all, from the GenAI metric stream instead (see resolveSpend
+ * and Common/Server/Utils/Telemetry/LlmMetricSpend.ts). That second source is
+ * what lets a metrics-only emitter — an SDK exporting metrics without traces,
+ * or a gateway that publishes spend counters and no spans — hold a budget at
+ * all. The two are never summed; see resolveSpend for why.
  *
  * The worker job (App/FeatureSet/Workers/Jobs/Llm/EvaluateLlmCostBudgets.ts)
  * calls evaluateAllBudgets() on a 15-minute tick. For each enabled budget it
- * sums the UTC day's llmCost from ClickHouse (scoped by the budget's optional
+ * sums the UTC day's spend from ClickHouse (scoped by the budget's optional
  * service / provider / model filters), stamps the spend back onto the budget
  * row for the dashboard, and publishes two gauge metrics:
  *
@@ -65,6 +73,17 @@ export const LLM_BUDGET_SERVICE_ID_ATTRIBUTE: string =
  */
 const BUDGET_METRIC_RETENTION_IN_DAYS: number = 15;
 
+/*
+ * Which telemetry signal a budget's spend figure came from. "none" means both
+ * sources were empty — a genuinely idle scope, not a failure.
+ */
+export type LlmSpendSource = "spans" | "metrics" | "none";
+
+export interface LlmResolvedSpend {
+  spendInUSD: number;
+  source: LlmSpendSource;
+}
+
 export default class LlmCostBudgetEvaluator {
   /**
    * Percent of the daily budget consumed. 0 when the budget is missing or
@@ -262,12 +281,18 @@ export default class LlmCostBudgetEvaluator {
 
     const dayStart: Date = this.getUtcDayStart(data.now);
 
-    const spendInUSD: number = await this.getSpendInUSD({
+    const resolved: LlmResolvedSpend = await this.resolveSpend({
       budget: budget,
       dayStart: dayStart,
       now: data.now,
     });
 
+    const spendInUSD: number = resolved.spendInUSD;
+
+    logger.debug(
+      `LLM cost budget ${budget.id.toString()}: spend ${spendInUSD} USD sourced from ${resolved.source}`,
+    );
+
     const percentUsed: number = this.computePercentUsed({
       spendInUSD: spendInUSD,
       dailyBudgetInUSD: budget.dailyBudgetInUSD,
@@ -293,11 +318,83 @@ export default class LlmCostBudgetEvaluator {
     });
   }
 
+  /**
+   * Translate a budget's scope into the equivalent metric-stream scope, so
+   * the span query and the metric query narrow to the same slice of traffic.
+   *
+   * Service scoping is exact — Span.primaryEntityId and Metric.primaryEntityId
+   * are the same telemetry-service id. Provider/model scoping matches the
+   * primary semantic-convention attribute keys only; see the caveat on
+   * LlmMetricSpend.
+   */
+  public static buildMetricScope(data: {
+    budget: LlmCostBudget;
+    dayStart: Date;
+    now: Date;
+  }): LlmMetricScope {
+    return {
+      projectId: data.budget.projectId!,
+      startTime: data.dayStart,
+      endTime: data.now,
+      serviceId: data.budget.serviceId,
+      llmSystem: data.budget.llmSystem,
+      llmModel: data.budget.llmModel,
+    };
+  }
+
+  /**
+   * The day's spend for a budget, and which telemetry signal it came from.
+   *
+   * Spans are authoritative: a GenAI span carries model, tokens and cost on
+   * one row, and it is the only source that can also back the per-call views.
+   * Metrics are consulted ONLY when the span-sourced total is zero.
+   *
+   * That ordering is a deliberate fallback rather than a sum. Instrumentations
+   * that emit both signals (OpenLLMetry is the common case) would otherwise be
+   * counted twice, and for a budget that gates alerting an inflated figure is
+   * worse than a missing one — it fires on spend that never happened. The
+   * fallback turns on exactly for the emitters that need it: those publishing
+   * GenAI metrics and no GenAI spans, whose span total is structurally zero.
+   *
+   * Known limit, documented rather than papered over: a scope where SOME
+   * services emit spans and others emit only metrics resolves to "spans" and
+   * under-counts the metrics-only services. Splitting a budget per service is
+   * the workaround; scoping each to its own service gives each the source it
+   * actually has.
+   *
+   * A metric-query failure is allowed to propagate for the same reason the
+   * span aggregate passes timeoutOverflowMode 'throw': returning 0 on error
+   * would silently suppress a real breach. evaluateAllBudgets isolates the
+   * failure to the one budget.
+   */
+  @CaptureSpan()
+  public static async resolveSpend(data: {
+    budget: LlmCostBudget;
+    dayStart: Date;
+    now: Date;
+  }): Promise<LlmResolvedSpend> {
+    const spanSpend: number = await this.getSpanSpendInUSD(data);
+
+    if (spanSpend > 0) {
+      return { spendInUSD: spanSpend, source: "spans" };
+    }
+
+    const metricSpend: number = await LlmMetricSpend.getCostInUSD(
+      this.buildMetricScope(data),
+    );
+
+    if (metricSpend > 0) {
+      return { spendInUSD: metricSpend, source: "metrics" };
+    }
+
+    return { spendInUSD: 0, source: "none" };
+  }
+
   /**
    * Sum the day's llmCost for the budget's scope from ClickHouse.
    */
   @CaptureSpan()
-  private static async getSpendInUSD(data: {
+  private static async getSpanSpendInUSD(data: {
     budget: LlmCostBudget;
     dayStart: Date;
     now: Date;

package/Server/Utils/Telemetry/LlmMetricSpend.ts

@@ -0,0 +1,181 @@
+import Metric from "../../../Models/AnalyticsModels/Metric";
+import AggregatedResult from "../../../Types/BaseDatabase/AggregatedResult";
+import AggregationInterval from "../../../Types/BaseDatabase/AggregationInterval";
+import AggregationType from "../../../Types/BaseDatabase/AggregationType";
+import { LIMIT_PER_PROJECT } from "../../../Types/Database/LimitMax";
+import { LlmTokenTypeAttributeKeys } from "../../../Types/Telemetry/LlmMetricConventions";
+import LlmMetricQuery, {
+  LlmMetricScope,
+  LlmMetricTokenTotals,
+} from "../../../Utils/Telemetry/LlmMetricQuery";
+import AggregateBy from "../../Types/AnalyticsDatabase/AggregateBy";
+import MetricService from "../../Services/MetricService";
+import CaptureSpan from "./CaptureSpan";
+
+/*
+ * Metric-sourced LLM token and cost totals.
+ *
+ * The LLM feature reads spans first — a GenAI span carries model, tokens and
+ * cost on one row, which is the richest source and the only one that can back
+ * the per-call views. This module is the SECOND source, for the emitters that
+ * publish GenAI metrics but no GenAI spans:
+ *
+ *   - An OpenTelemetry SDK wired for metrics without tracing.
+ *   - AI gateways and vendor exporters that publish spend/usage counters and
+ *     never export traces.
+ *
+ * Callers must not add these totals to the span-sourced ones. Instrumentations
+ * that emit both (OpenLLMetry is the common case) would then be counted twice,
+ * and for a cost budget that gates alerting a doubled figure is worse than a
+ * missing one. LlmCostBudgetEvaluator therefore treats this as a FALLBACK: it
+ * is consulted only when the span-sourced total for the same scope is zero.
+ *
+ * Query construction and result folding live in the isomorphic
+ * Common/Utils/Telemetry/LlmMetricQuery, so the dashboard's AI / LLM overview
+ * asks the same questions of the same data. This module only adds the
+ * AggregateBy envelope and executes them with root props.
+ */
+
+export { LlmMetricScope, LlmMetricTokenTotals };
+
+export default class LlmMetricSpend {
+  /**
+   * Aggregate descriptor for the cost sum: one total over the whole window.
+   *
+   * `value` is the right column for every point type here — for a histogram
+   * ClickHouse stores the bucket sum in it, and MetricService resolves the
+   * point type before building the statement so distribution metrics skip the
+   * materialized views. Cost metrics are counters (delta or cumulative spend),
+   * so summing them across the window is the intended reduction.
+   */
+  public static buildCostAggregateBy(
+    scope: LlmMetricScope,
+  ): AggregateBy<Metric> {
+    return {
+      query: LlmMetricQuery.buildCostQuery(scope),
+      aggregationType: AggregationType.Sum,
+      aggregateColumnName: "value",
+      aggregationTimestampColumnName: "time",
+      startTimestamp: scope.startTime,
+      endTimestamp: scope.endTime,
+      // One total over the whole window — no time bucketing.
+      aggregationInterval: AggregationInterval.Total,
+      /*
+       * Same reasoning as the span path: this figure feeds budget monitors, so
+       * a query timeout must fail loudly rather than return a
+       * silently-partial sum that would understate spend and suppress a real
+       * breach.
+       */
+      timeoutOverflowMode: "throw",
+      limit: LIMIT_PER_PROJECT,
+      skip: 0,
+      props: {
+        isRoot: true,
+      },
+    } as AggregateBy<Metric>;
+  }
+
+  /**
+   * Aggregate descriptor for the micro-USD cost sum.
+   *
+   * Identical in shape to buildCostAggregateBy, deliberately a SECOND query
+   * rather than a wider name list on the first: the two name lists carry
+   * different units, and a single Sum over both would add micro-USD figures
+   * to USD ones with no way to recover the unit afterwards. Codex spend would
+   * arrive a million times too large and trip every cost budget in the
+   * project. getCostInUSD scales this total once, via
+   * LlmMetricQuery.combineCostTotals.
+   */
+  public static buildMicroUsdCostAggregateBy(
+    scope: LlmMetricScope,
+  ): AggregateBy<Metric> {
+    return {
+      query: LlmMetricQuery.buildMicroUsdCostQuery(scope),
+      aggregationType: AggregationType.Sum,
+      aggregateColumnName: "value",
+      aggregationTimestampColumnName: "time",
+      startTimestamp: scope.startTime,
+      endTimestamp: scope.endTime,
+      aggregationInterval: AggregationInterval.Total,
+      // Same fail-loud reasoning as the USD query above.
+      timeoutOverflowMode: "throw",
+      limit: LIMIT_PER_PROJECT,
+      skip: 0,
+      props: {
+        isRoot: true,
+      },
+    } as AggregateBy<Metric>;
+  }
+
+  /**
+   * Aggregate descriptor for the token sum.
+   *
+   * Grouping by every candidate token-type attribute key in ONE query —
+   * rather than issuing a query per key and per direction — is what keeps this
+   * both complete and free of double counting: each returned group carries the
+   * attribute values that produced it, so LlmMetricQuery.reduceTokenRows
+   * classifies them against the full set of recognized spellings instead of
+   * the query having to pick one.
+   */
+  public static buildTokenAggregateBy(
+    scope: LlmMetricScope,
+  ): AggregateBy<Metric> {
+    return {
+      query: LlmMetricQuery.buildTokenQuery(scope),
+      aggregationType: AggregationType.Sum,
+      aggregateColumnName: "value",
+      aggregationTimestampColumnName: "time",
+      startTimestamp: scope.startTime,
+      endTimestamp: scope.endTime,
+      aggregationInterval: AggregationInterval.Total,
+      groupByAttributeKeys: [...LlmTokenTypeAttributeKeys],
+      timeoutOverflowMode: "throw",
+      limit: LIMIT_PER_PROJECT,
+      skip: 0,
+      props: {
+        isRoot: true,
+      },
+    } as AggregateBy<Metric>;
+  }
+
+  /**
+   * Metric-sourced spend in USD for the scope, or 0 when nothing matched.
+   *
+   * Two aggregates, not one, because the recognized cost metrics come in two
+   * UNITS: dollars (the gateways, Claude Code, Cursor) and millionths of a
+   * dollar (the Codex CLI). They are queried apart and scaled before being
+   * added, in LlmMetricQuery.combineCostTotals — the one place the 1e-6
+   * factor lives. Folding the micro-USD names into the USD query would report
+   * Codex spend a million times too high, which for a figure that gates
+   * budget alerting is far worse than reporting nothing.
+   *
+   * The two run in parallel: they hit the same table with disjoint name
+   * filters, and serializing them would double the latency of every budget
+   * evaluation for no benefit.
+   */
+  @CaptureSpan()
+  public static async getCostInUSD(scope: LlmMetricScope): Promise<number> {
+    const [usdResult, microUsdResult]: [AggregatedResult, AggregatedResult] =
+      await Promise.all([
+        MetricService.aggregateBy(this.buildCostAggregateBy(scope)),
+        MetricService.aggregateBy(this.buildMicroUsdCostAggregateBy(scope)),
+      ]);
+
+    return LlmMetricQuery.combineCostTotals({
+      usd: LlmMetricQuery.sumAggregatedRows(usdResult?.data),
+      microUsd: LlmMetricQuery.sumAggregatedRows(microUsdResult?.data),
+    });
+  }
+
+  /** Metric-sourced input/output token totals for the scope. */
+  @CaptureSpan()
+  public static async getTokenTotals(
+    scope: LlmMetricScope,
+  ): Promise<LlmMetricTokenTotals> {
+    const result: AggregatedResult = await MetricService.aggregateBy(
+      this.buildTokenAggregateBy(scope),
+    );
+
+    return LlmMetricQuery.reduceTokenRows(result?.data);
+  }
+}

package/Server/Utils/Telemetry/LlmSpan.ts

@@ -14,8 +14,11 @@ import {
   LlmRequestModelAttributeKeys,
   LlmResponseModelAttributeKeys,
   LlmSystemAttributeKeys,
+  LlmTeamAttributeKeys,
   LlmToolNameAttributeKeys,
   LlmTotalTokenAttributeKeys,
+  LlmUserEmailAttributeKeys,
+  LlmUserIdAttributeKeys,
 } from "../../../Types/Telemetry/LlmConventions";
 import { AttributeType } from "./Telemetry";
 
@@ -61,6 +64,20 @@ export interface LlmSpanFields {
   llmToolName: string;
   // Conversation / session id grouping calls of one interaction (gen_ai.conversation.id).
   llmConversationId: string;
+  /*
+   * WHO ran the call — the EMPLOYEE / internal actor, never the caller's own
+   * downstream customer. See the block comment above
+   * LlmUserIdAttributeKeys in Common/Types/Telemetry/LlmConventions.ts for
+   * why that distinction is load-bearing rather than pedantic.
+   *
+   * "" when the instrumentation reports no identity, which is still the
+   * common case for library-instrumented server code — only the coding-agent
+   * CLIs and the gateways stamp identity by default.
+   */
+  llmUserId: string;
+  llmUserEmail: string;
+  // Team / cost centre the spend charges to (team.id, cost_center, ...).
+  llmTeam: string;
 }
 
 type SpanAttributes = Dictionary<AttributeType | Array<AttributeType>>;
@@ -83,6 +100,9 @@ export default class LlmSpanUtil {
       llmAgentName: "",
       llmToolName: "",
       llmConversationId: "",
+      llmUserId: "",
+      llmUserEmail: "",
+      llmTeam: "",
     };
   }
 
@@ -184,6 +204,32 @@ export default class LlmSpanUtil {
         attributes,
         LlmConversationIdAttributeKeys,
       );
+
+      /*
+       * Identity is gated on isLlmSpan for exactly the reason the
+       * conversation id above is. "user.id", "user.email" and "team.id" are
+       * GENERIC OTel general-semconv keys — RUM browser spans and ordinary
+       * backend HTTP spans routinely carry them, and those are the
+       * highest-volume span classes there are. Stamping them onto every such
+       * span would copy the value (and pay for its skip index) across the
+       * whole fleet to serve a reader that only ever asks "which employee
+       * spent what on LLM calls". The LLM spans are the only rows that
+       * question reads, so they are the only rows that carry the columns.
+       *
+       * The keys that carry the caller's DOWNSTREAM CUSTOMER rather than the
+       * employee (gen_ai.user, llm.user,
+       * litellm.metadata.user_api_key_end_user_id) are deliberately absent
+       * from these lists — see LlmEndUserAttributeKeys. Reading one of them
+       * here would silently misattribute internal chargeback.
+       */
+      fields.llmUserId = this.getString(attributes, LlmUserIdAttributeKeys);
+
+      fields.llmUserEmail = this.getString(
+        attributes,
+        LlmUserEmailAttributeKeys,
+      );
+
+      fields.llmTeam = this.getString(attributes, LlmTeamAttributeKeys);
     }
 
     /*

package/Server/Utils/Workspace/Slack/Actions/Auth.ts

@@ -65,8 +65,6 @@ export default class SlackAuthAction {
     const { req } = data;
 
     logger.debug("Starting Slack request authorization");
-    logger.debug(`Request body: `);
-    logger.debug(req.body);
 
     let payload: JSONObject = req.body;
 
@@ -74,9 +72,6 @@ export default class SlackAuthAction {
       payload = JSON.parse(payload["payload"]);
     }
 
-    logger.debug(`Payload: `);
-    logger.debug(payload);
-
     let slackUserId: string | undefined = (
       (payload as JSONObject)["user"] as JSONObject
     )?.["id"] as string;
@@ -216,11 +211,6 @@ export default class SlackAuthAction {
       };
 
       actions.push(action);
-      logger.debug("View values: ");
-      logger.debug(viewValues);
-
-      logger.debug("Actions: ");
-      logger.debug(actions);
     }
 
     if (payload["callback_id"]) {
@@ -296,8 +286,6 @@ export default class SlackAuthAction {
     logger.debug("Slack request authorized successfully", {
       projectId: projectId.toString(),
     });
-    logger.debug("Slack request: ");
-    logger.debug(slackRequest);
 
     return slackRequest;
   }