@oneuptime/common 11.7.3 → 12.0.0

package/Server/Services/TraceAggregationService.ts

@@ -11,6 +11,10 @@ import CaptureSpan from "../Utils/Telemetry/CaptureSpan";
 import { DbJSONResponse, Results } from "./AnalyticsDatabaseService";
 import logger from "../Utils/Logger";
 import ServiceType from "../../Types/Telemetry/ServiceType";
+import ResourceFacetResolver, {
+  ResolvedFacetValue,
+  ResourceFacetSpec,
+} from "../Utils/Telemetry/ResourceFacetResolver";
 
 export interface HistogramBucket {
   time: string;
@@ -18,6 +22,14 @@ export interface HistogramBucket {
   count: number;
 }
 
+/*
+ * An exact attribute predicate. A single string is `= value`; an array is
+ * `IN (...)`, which is what a multi-select dashboard variable resolves to.
+ * Mirrors LogAttributeFilterValue so both explorers filter identically.
+ */
+export type TraceAttributeFilterValue = string | Array<string>;
+export type TraceAttributeFilters = Record<string, TraceAttributeFilterValue>;
+
 export interface TraceFilters {
   serviceIds?: Array<ObjectID> | undefined;
   entityKeys?: Array<string> | undefined;
@@ -47,7 +59,7 @@ export interface TraceFilters {
   maxDurationNano?: number | undefined;
   exactDurationNano?: number | undefined;
   rootOnly?: boolean | undefined;
-  attributes?: Record<string, string> | undefined;
+  attributes?: TraceAttributeFilters | undefined;
   /*
    * Substring (contains) attribute matches — `@key:~value` in the explorer.
    * Same case-insensitive key matching as `attributes`, value via ILIKE.
@@ -1151,6 +1163,27 @@ export class TraceAggregationService {
          * LogAggregationService.appendCommonFilters. Casings vary across
          * OTEL conventions and app-emitted attributes.
          */
+        if (Array.isArray(attrValue)) {
+          /*
+           * Multi-value selection (a multi-select dashboard variable) — an
+           * empty array means "no selection", which must widen to everything
+           * rather than compile to `IN ()` and match nothing.
+           */
+          if (attrValue.length === 0) {
+            continue;
+          }
+          statement.append(
+            SQL` AND arrayExists((k, v) -> lowerUTF8(k) = lowerUTF8(${{
+              type: TableColumnType.Text,
+              value: attrKey,
+            }}) AND v IN (${{
+              type: TableColumnType.Text,
+              value: new Includes(attrValue),
+            }}), mapKeys(attributes), mapValues(attributes))`,
+          );
+          continue;
+        }
+
         statement.append(
           SQL` AND arrayExists((k, v) -> lowerUTF8(k) = lowerUTF8(${{
             type: TableColumnType.Text,
@@ -1229,7 +1262,116 @@ export class TraceAggregationService {
    * (LogAggregationService.getAnalyticsTimeseries and friends) and shares
    * appendCommonFilters with the histogram/facets, so every explorer filter
    * applies identically.
+   *
+   * A resource dimension (rumApplicationId, hostId, kubernetesClusterId, …)
+   * selects the bare `toString(primaryEntityId)`, so grouping by one returns
+   * internal ObjectIDs. Nothing downstream turns those into names: the traces
+   * explorer resolves only its own `primaryEntityId` split, client-side, and
+   * the dashboard TraceChart / TraceTable widgets have no equivalent — so the
+   * raw id reaches the chart legend, the tooltip series name and the table
+   * cell. Resolve them here, once per query, through the same resolver the
+   * facet sidebar uses.
+   *
+   * Deliberately NOT applied to getAnalyticsTopList: the timeseries path
+   * feeds that method's values straight back into the query as an IN filter
+   * (see getAnalyticsTimeseries), and the AI latency detectors match on them,
+   * so those must stay raw.
+   *
+   * Unresolvable ids — a deleted resource, or one past the resolver's page
+   * size — keep their raw value rather than vanishing from the result.
    */
+  private static async applyResourceDisplayValues<
+    TRow extends { groupValues: Record<string, string> },
+  >(
+    projectId: ObjectID,
+    groupByKeys: Array<string>,
+    rows: Array<TRow>,
+  ): Promise<Array<TRow>> {
+    /*
+     * Both memberships are required, and the two sets genuinely differ:
+     *
+     * - `proxmoxClusterId` / `cephClusterId` are resource dimensions here
+     *   but have no branch in ResourceFacetResolver.resolveOne, which
+     *   returns [] for them. Asking anyway would cost a round-trip and
+     *   still leave the cell raw.
+     * - `primaryEntityId` / `serviceId` are resolvable but are NOT resource
+     *   dimensions here — they select the OpenTelemetry-service slot, and
+     *   the traces explorer already resolves that split client-side
+     *   (TracesAnalyticsView's serviceNameMap). Renaming their values
+     *   server-side would change an existing surface for no gain.
+     *
+     * Taking the intersection means a key becomes resolvable here the
+     * moment the resolver learns it, with no second list to update.
+     */
+    const resourceKeys: Array<string> = groupByKeys.filter(
+      (key: string): boolean => {
+        return (
+          TraceAggregationService.RESOURCE_FACET_KEYS.has(key) &&
+          ResourceFacetResolver.isResourceFacet(key)
+        );
+      },
+    );
+
+    if (resourceKeys.length === 0 || rows.length === 0) {
+      return rows;
+    }
+
+    const specs: Array<ResourceFacetSpec> = resourceKeys.map(
+      (key: string): ResourceFacetSpec => {
+        const counts: Map<string, number> = new Map();
+
+        for (const row of rows) {
+          const raw: string | undefined = row.groupValues[key];
+          if (raw) {
+            counts.set(raw, (counts.get(raw) || 0) + 1);
+          }
+        }
+
+        return { facetKey: key, counts: counts };
+      },
+    );
+
+    let resolved: Record<string, Array<ResolvedFacetValue>> = {};
+
+    try {
+      resolved = await ResourceFacetResolver.resolve(projectId, specs);
+    } catch (err: unknown) {
+      logger.warn(
+        `Could not resolve resource display names for trace analytics group-by; falling back to raw ids: ${err}`,
+      );
+      return rows;
+    }
+
+    const displayNamesByKey: Map<string, Map<string, string>> = new Map();
+
+    for (const key of resourceKeys) {
+      const displayNames: Map<string, string> = new Map();
+
+      for (const entry of resolved[key] || []) {
+        if (entry.displayName) {
+          displayNames.set(entry.value, entry.displayName);
+        }
+      }
+
+      displayNamesByKey.set(key, displayNames);
+    }
+
+    for (const row of rows) {
+      for (const key of resourceKeys) {
+        const raw: string | undefined = row.groupValues[key];
+        const displayName: string | undefined = raw
+          ? displayNamesByKey.get(key)?.get(raw)
+          : undefined;
+
+        if (displayName) {
+          row.groupValues[key] = displayName;
+        }
+      }
+    }
+
+    return rows;
+  }
+
   @CaptureSpan()
   public static async getAnalyticsTimeseries(
     request: TraceAnalyticsRequest,
@@ -1285,20 +1427,31 @@ export class TraceAggregationService {
       );
     }
 
-    return rows.map((row: JSONObject): TraceAnalyticsTimeseriesRow => {
-      const groupValues: Record<string, string> = {};
+    const timeseriesRows: Array<TraceAnalyticsTimeseriesRow> = rows.map(
+      (row: JSONObject): TraceAnalyticsTimeseriesRow => {
+        const groupValues: Record<string, string> = {};
 
-      for (const [index, key] of groupByKeys.entries()) {
-        const alias: string = TraceAggregationService.groupByAlias(key, index);
-        groupValues[key] = String(row[alias] ?? "");
-      }
+        for (const [index, key] of groupByKeys.entries()) {
+          const alias: string = TraceAggregationService.groupByAlias(
+            key,
+            index,
+          );
+          groupValues[key] = String(row[alias] ?? "");
+        }
 
-      return {
-        time: String(row["bucket"] || ""),
-        value: Number(row["val"] || 0),
-        groupValues,
-      };
-    });
+        return {
+          time: String(row["bucket"] || ""),
+          value: Number(row["val"] || 0),
+          groupValues,
+        };
+      },
+    );
+
+    return TraceAggregationService.applyResourceDisplayValues(
+      request.projectId,
+      groupByKeys,
+      timeseriesRows,
+    );
   }
 
   @CaptureSpan()
@@ -1370,27 +1523,38 @@ export class TraceAggregationService {
 
     const groupByKeys: Array<string> = request.groupBy;
 
-    return rows.map((row: JSONObject): TraceAnalyticsTableRow => {
-      const groupValues: Record<string, string> = {};
+    const tableRows: Array<TraceAnalyticsTableRow> = rows.map(
+      (row: JSONObject): TraceAnalyticsTableRow => {
+        const groupValues: Record<string, string> = {};
 
-      for (const [index, key] of groupByKeys.entries()) {
-        const alias: string = TraceAggregationService.groupByAlias(key, index);
-        groupValues[key] = String(row[alias] ?? "");
-      }
+        for (const [index, key] of groupByKeys.entries()) {
+          const alias: string = TraceAggregationService.groupByAlias(
+            key,
+            index,
+          );
+          groupValues[key] = String(row[alias] ?? "");
+        }
 
-      return {
-        groupValues,
-        count: Number(row["cnt"] || 0),
-        errorCount: Number(row["err_cnt"] || 0),
-        avgDurationMs: Number(row["avg_ms"] || 0),
-        p50DurationMs: Number(row["p50_ms"] || 0),
-        p90DurationMs: Number(row["p90_ms"] || 0),
-        p95DurationMs: Number(row["p95_ms"] || 0),
-        p99DurationMs: Number(row["p99_ms"] || 0),
-        minDurationMs: Number(row["min_ms"] || 0),
-        maxDurationMs: Number(row["max_ms"] || 0),
-      };
-    });
+        return {
+          groupValues,
+          count: Number(row["cnt"] || 0),
+          errorCount: Number(row["err_cnt"] || 0),
+          avgDurationMs: Number(row["avg_ms"] || 0),
+          p50DurationMs: Number(row["p50_ms"] || 0),
+          p90DurationMs: Number(row["p90_ms"] || 0),
+          p95DurationMs: Number(row["p95_ms"] || 0),
+          p99DurationMs: Number(row["p99_ms"] || 0),
+          minDurationMs: Number(row["min_ms"] || 0),
+          maxDurationMs: Number(row["max_ms"] || 0),
+        };
+      },
+    );
+
+    return TraceAggregationService.applyResourceDisplayValues(
+      request.projectId,
+      groupByKeys,
+      tableRows,
+    );
   }
 
   /*

package/Server/Services/TraceDropFilterService.ts

@@ -1,10 +1,143 @@
 import DatabaseService from "./DatabaseService";
 import Model from "../../Models/DatabaseModels/TraceDropFilter";
+import CreateBy from "../Types/Database/CreateBy";
+import UpdateBy from "../Types/Database/UpdateBy";
+import { OnCreate, OnUpdate } from "../Types/Database/Hooks";
+import LIMIT_MAX from "../../Types/Database/LimitMax";
+import ObjectID from "../../Types/ObjectID";
+import OneUptimeDate from "../../Types/Date";
+import PartialEntity from "../../Types/Database/PartialEntity";
+import {
+  DropFilterCandidate,
+  DropFilterValidationOptions,
+  isSqlExpressionValue,
+  validateDropFilter,
+} from "../Utils/DropFilterValidation";
+
+const VALIDATION_OPTIONS: DropFilterValidationOptions = {
+  recordNoun: "spans",
+};
 
 export class Service extends DatabaseService<Model> {
   public constructor() {
     super(Model);
   }
+
+  protected override async onBeforeCreate(
+    createBy: CreateBy<Model>,
+  ): Promise<OnCreate<Model>> {
+    validateDropFilter(
+      {
+        action: createBy.data.action,
+        samplePercentage: createBy.data.samplePercentage,
+        filterQuery: createBy.data.filterQuery as string | undefined,
+      },
+      VALIDATION_OPTIONS,
+    );
+
+    return { createBy, carryForward: null };
+  }
+
+  protected override async onBeforeUpdate(
+    updateBy: UpdateBy<Model>,
+  ): Promise<OnUpdate<Model>> {
+    /*
+     * An update can make a row invalid without naming every field: flipping
+     * `action` to "sample" while the stored `samplePercentage` is null, or
+     * blanking `filterQuery` on a row whose action is already "drop". So
+     * validate the MERGED row, not the incoming partial.
+     *
+     * Only pay for the SELECT when the update actually touches one of the
+     * three fields that can break the row. Drop-filter edits are rare,
+     * human-driven config changes — this is not a hot path.
+     */
+    const incomingAction: unknown = updateBy.data.action;
+    const incomingSamplePercentage: unknown = updateBy.data.samplePercentage;
+    const incomingFilterQuery: unknown = updateBy.data.filterQuery;
+
+    const touchesValidatedFields: boolean =
+      incomingAction !== undefined ||
+      incomingSamplePercentage !== undefined ||
+      incomingFilterQuery !== undefined;
+
+    if (!touchesValidatedFields) {
+      return { updateBy, carryForward: null };
+    }
+
+    if (
+      isSqlExpressionValue(incomingAction) ||
+      isSqlExpressionValue(incomingSamplePercentage) ||
+      isSqlExpressionValue(incomingFilterQuery)
+    ) {
+      // Cannot evaluate a raw SQL expression here. See isSqlExpressionValue.
+      return { updateBy, carryForward: null };
+    }
+
+    const existingRows: Array<Model> = await this.findBy({
+      query: updateBy.query,
+      skip: 0,
+      limit: LIMIT_MAX,
+      select: {
+        _id: true,
+        action: true,
+        samplePercentage: true,
+        filterQuery: true,
+      },
+      props: {
+        isRoot: true,
+      },
+    });
+
+    for (const existing of existingRows) {
+      const merged: DropFilterCandidate = {
+        action:
+          incomingAction !== undefined
+            ? (incomingAction as string | null)
+            : existing.action,
+        samplePercentage:
+          incomingSamplePercentage !== undefined
+            ? (incomingSamplePercentage as number | null)
+            : existing.samplePercentage,
+        filterQuery:
+          incomingFilterQuery !== undefined
+            ? (incomingFilterQuery as string | null)
+            : (existing.filterQuery as string | null),
+      };
+
+      validateDropFilter(merged, VALIDATION_OPTIONS);
+    }
+
+    return { updateBy, carryForward: null };
+  }
+
+  /*
+   * Ingest-path counter flush: add `count` drops to this filter's running
+   * total and stamp `lastDroppedAt`, in one atomic statement.
+   *
+   * Called from the telemetry ingest workers on a throttle (many dropped
+   * spans batched into one UPDATE), never once per record, and never on the
+   * request path. Runs no hooks and does not bump `version`, so it cannot
+   * conflict with a human editing the same filter in the dashboard.
+   *
+   * See App/FeatureSet/Telemetry/Utils/DropFilterDropRecorder.ts.
+   */
+  public async addToDroppedCount(data: {
+    id: ObjectID;
+    count: number;
+    lastDroppedAt?: Date | undefined;
+  }): Promise<void> {
+    if (!data.count || data.count <= 0) {
+      return;
+    }
+
+    await this.atomicAddToColumnsByIdWithoutHooks({
+      id: data.id,
+      add: { droppedCount: data.count },
+      set: {
+        lastDroppedAt: data.lastDroppedAt || OneUptimeDate.getCurrentDate(),
+      } as PartialEntity<Model>,
+    });
+  }
 }
 
 export default new Service();

package/Server/Types/Billing/MeteredPlan/AllMeteredPlans.ts

@@ -32,12 +32,27 @@ export const ProfilesDataIngestMeteredPlan: TelemetryMeteredPlanType =
     unitCostInUSD: 0.1 / 15, // 0.10 per 15 days per GB
   });
 
+/*
+ * Session replay is priced above the other pillars on purpose.
+ *
+ * At telemetry parity (0.1 / 15) a project recording 100k sessions a month
+ * bills well under a dollar, which does not cover the cost and mis-signals the
+ * value of the feature. This rate is still far below comparable products while
+ * scaling with the actual driver, which is bytes of recording retained.
+ */
+export const SessionReplayDataIngestMeteredPlan: TelemetryMeteredPlanType =
+  new TelemetryMeteredPlanType({
+    productType: ProductType.SessionReplay,
+    unitCostInUSD: 2.0 / 15, // 2.00 per 15 days per GB
+  });
+
 const AllMeteredPlans: Array<ServerMeteredPlan> = [
   ActiveMonitoringMeteredPlan,
   LogDataIngestMeteredPlan,
   MetricsDataIngestMeteredPlan,
   TracesDataIngestMetredPlan,
   ProfilesDataIngestMeteredPlan,
+  SessionReplayDataIngestMeteredPlan,
 ];
 
 export class MeteredPlanUtil {
@@ -53,6 +68,8 @@ export class MeteredPlanUtil {
       return TracesDataIngestMetredPlan;
     } else if (productType === ProductType.Profiles) {
       return ProfilesDataIngestMeteredPlan;
+    } else if (productType === ProductType.SessionReplay) {
+      return SessionReplayDataIngestMeteredPlan;
     } else if (productType === ProductType.ActiveMonitoring) {
       return ActiveMonitoringMeteredPlan;
     }

package/Server/Types/Billing/MeteredPlan/ServerMeteredPlan.ts

@@ -30,4 +30,14 @@ export default class ServerMeteredPlan {
   public getPriceId(): string {
     return BillingService.getMeteredPlanPriceId(this.getProductType());
   }
+
+  /*
+   * Whether this plan has a Stripe price yet. A plan whose price ids are not
+   * created in Stripe must be skipped when building Stripe subscription
+   * items - getPriceId throws for it - while still staging usage into
+   * TelemetryUsageBilling.
+   */
+  public hasPriceId(): boolean {
+    return BillingService.hasMeteredPlanPriceId(this.getProductType());
+  }
 }

package/Server/Types/Workflow/Components/API/Delete.ts

@@ -48,6 +48,18 @@ export default class ApiDelete extends ComponentCode {
         headers: args["request-headers"] as Dictionary<string>,
       });
 
+      /*
+       * On the server API resolves with an HTTPErrorResponse instead of
+       * throwing, so the catch below never sees an HTTP error - a 4xx/5xx used
+       * to take the success port.
+       */
+      if (apiResult instanceof HTTPErrorResponse) {
+        return Promise.resolve({
+          returnValues: ApiComponentUtils.getReturnValues(apiResult),
+          executePort: result.errorPort,
+        });
+      }
+
       return Promise.resolve({
         returnValues: ApiComponentUtils.getReturnValues(apiResult),
         executePort: result.successPort,

package/Server/Types/Workflow/Components/API/Get.ts

@@ -48,6 +48,18 @@ export default class ApiGet extends ComponentCode {
         headers: args["request-headers"] as Dictionary<string>,
       });
 
+      /*
+       * On the server API resolves with an HTTPErrorResponse instead of
+       * throwing, so the catch below never sees an HTTP error - a 4xx/5xx used
+       * to take the success port.
+       */
+      if (apiResult instanceof HTTPErrorResponse) {
+        return Promise.resolve({
+          returnValues: ApiComponentUtils.getReturnValues(apiResult),
+          executePort: result.errorPort,
+        });
+      }
+
       return Promise.resolve({
         returnValues: ApiComponentUtils.getReturnValues(apiResult),
         executePort: result.successPort,

package/Server/Types/Workflow/Components/API/Patch.ts

@@ -48,6 +48,18 @@ export default class ApiPut extends ComponentCode {
         headers: args["request-headers"] as Dictionary<string>,
       });
 
+      /*
+       * On the server API resolves with an HTTPErrorResponse instead of
+       * throwing, so the catch below never sees an HTTP error - a 4xx/5xx used
+       * to take the success port.
+       */
+      if (apiResult instanceof HTTPErrorResponse) {
+        return Promise.resolve({
+          returnValues: ApiComponentUtils.getReturnValues(apiResult),
+          executePort: result.errorPort,
+        });
+      }
+
       return Promise.resolve({
         returnValues: ApiComponentUtils.getReturnValues(apiResult),
         executePort: result.successPort,

package/Server/Types/Workflow/Components/API/Post.ts

@@ -81,6 +81,18 @@ export default class ApiPost extends ComponentCode {
 
       logger.debug("API Post Component is done.", workflowLogAttributes);
 
+      /*
+       * On the server API resolves with an HTTPErrorResponse instead of
+       * throwing, so the catch below never sees an HTTP error - a 4xx/5xx used
+       * to take the success port.
+       */
+      if (apiResult instanceof HTTPErrorResponse) {
+        return Promise.resolve({
+          returnValues: ApiComponentUtils.getReturnValues(apiResult),
+          executePort: result.errorPort,
+        });
+      }
+
       return Promise.resolve({
         returnValues: ApiComponentUtils.getReturnValues(apiResult),
         executePort: result.successPort,

package/Server/Types/Workflow/Components/API/Put.ts

@@ -48,6 +48,19 @@ export default class ApiPut extends ComponentCode {
         headers: args["request-headers"] as Dictionary<string>,
       });
 
+      /*
+       * On the server API resolves with an HTTPErrorResponse instead of
+       * throwing (only the UI subclass throws), so the catch below never sees
+       * an HTTP error - a 4xx/5xx used to take the success port and log
+       * "Success" at the customer.
+       */
+      if (apiResult instanceof HTTPErrorResponse) {
+        return Promise.resolve({
+          returnValues: ApiComponentUtils.getReturnValues(apiResult),
+          executePort: result.errorPort,
+        });
+      }
+
       return Promise.resolve({
         returnValues: ApiComponentUtils.getReturnValues(apiResult),
         executePort: result.successPort,

package/Server/Utils/AI/Chat/ObservabilityAssistant.ts

@@ -5,7 +5,7 @@ import { JSONObject } from "../../../../Types/JSON";
 import { AIChatCitation } from "../../../../Types/AI/AIChatTypes";
 import AIService, { AILogResponse } from "../../../Services/AIService";
 import logger from "../../Logger";
-import { LLMMessage } from "../../LLM/LLMService";
+import { LLMMessage, LLMToolDefinition } from "../../LLM/LLMService";
 import AIToolbox, { ToolCallOutcome } from "../Toolbox/Index";
 import { ToolContext } from "../Toolbox/ToolTypes";
 import AIChatPermissionMode from "../../../../Types/AI/AIChatPermissionMode";
@@ -91,6 +91,25 @@ export interface ObservabilityAssistantRequest {
    * failing handler should not break the run.
    */
   onStep?: ((step: ObservabilityAssistantStep) => Promise<void>) | undefined;
+  /*
+   * Run-scoped tools offered to the model IN ADDITION to the read-only
+   * toolbox. The registered toolbox stays authoritative for its own names —
+   * an extra tool can never shadow a registered one.
+   */
+  extraTools?: Array<ObservabilityAssistantExtraTool> | undefined;
+}
+
+/*
+ * A caller-supplied tool grafted onto the loop for one run — how the AI
+ * remediation-execution runner adds its command tools without registering
+ * them in the global chat toolbox (where every conversation would see them).
+ * The execute callback owns its own authorization and timeouts; the loop
+ * wraps it in a catch so a throw becomes a self-correctable error envelope
+ * for the model, never a crashed run.
+ */
+export interface ObservabilityAssistantExtraTool {
+  definition: LLMToolDefinition;
+  execute: (args: JSONObject) => Promise<ToolCallOutcome>;
 }
 
 export interface ObservabilityAssistantResult {
@@ -144,6 +163,18 @@ export default class ObservabilityAssistant {
       props: request.props,
     };
 
+    /*
+     * Registered toolbox names win over extra tools so a run-scoped tool can
+     * never intercept (and man-in-the-middle) a registered tool's calls.
+     */
+    const extraToolsByName: Map<string, ObservabilityAssistantExtraTool> =
+      new Map();
+    for (const extraTool of request.extraTools || []) {
+      if (!AIToolbox.getToolByName(extraTool.definition.name)) {
+        extraToolsByName.set(extraTool.definition.name, extraTool);
+      }
+    }
+
     let systemPromptContent: string = buildObservabilityChatSystemPrompt({
       currentTime: OneUptimeDate.getCurrentDate(),
       /*
@@ -204,7 +235,14 @@ export default class ObservabilityAssistant {
         messages: messages,
         tools: budgetExhausted
           ? undefined
-          : AIToolbox.getLlmToolDefinitions(AIChatPermissionMode.ReadOnly),
+          : [
+              ...AIToolbox.getLlmToolDefinitions(AIChatPermissionMode.ReadOnly),
+              ...Array.from(extraToolsByName.values()).map(
+                (extraTool: ObservabilityAssistantExtraTool) => {
+                  return extraTool.definition;
+                },
+              ),
+            ],
         maxTokens: maxOutputTokens,
         temperature: TEMPERATURE,
         // Chat-ops content is per-user — do not persist previews to LlmLog.
@@ -269,11 +307,29 @@ export default class ObservabilityAssistant {
             toolArguments: toolCall.arguments,
           });
 
-          const outcome: ToolCallOutcome = await AIToolbox.executeTool({
-            name: toolCall.name,
-            args: toolCall.arguments,
-            ctx: toolContext,
-          });
+          const extraTool: ObservabilityAssistantExtraTool | undefined =
+            extraToolsByName.get(toolCall.name);
+
+          let outcome: ToolCallOutcome;
+          if (extraTool) {
+            try {
+              outcome = await extraTool.execute(toolCall.arguments);
+            } catch (err) {
+              const errorMessage: string =
+                err instanceof Error ? err.message : String(err);
+              outcome = {
+                success: false,
+                textForLlm: `Error executing ${toolCall.name}: ${errorMessage}. Adjust the arguments and try again, or answer with the data you already have.`,
+                errorMessage: errorMessage,
+              };
+            }
+          } else {
+            outcome = await AIToolbox.executeTool({
+              name: toolCall.name,
+              args: toolCall.arguments,
+              ctx: toolContext,
+            });
+          }
 
           if (!outcome.success || !outcome.result) {
             await emitStep({