assistant-cloud 0.1.43 → 0.2.0

package/src/runTelemetry.test.ts

@@ -1,12 +1,296 @@
 import { describe, expect, it } from "vitest";
 import {
+  createRunReport,
   createRunTelemetryToolCall,
+  deriveRunOutcome,
+  describeRunError,
+  extractRunTelemetryModelId,
   normalizeRunTelemetryUsage,
   truncateRunTelemetryText,
 } from "./runTelemetry";
 
 const MAX = 50_000;
 
+describe("deriveRunOutcome", () => {
+  it.each([
+    [{ isError: true }, { status: "error" }],
+    [{ isAbort: true }, { status: "incomplete", outcome: "aborted" }],
+    [{ isDisconnect: true }, { status: "incomplete", outcome: "disconnected" }],
+    [{ finishReason: "length" }, { status: "incomplete", outcome: "length" }],
+    [
+      { finishReason: "content-filter" },
+      { status: "incomplete", outcome: "content_filter" },
+    ],
+    [
+      { finishReason: "content_filter" },
+      { status: "incomplete", outcome: "content_filter" },
+    ],
+    [
+      { finishReason: "cancelled" },
+      { status: "incomplete", outcome: "aborted" },
+    ],
+    [{ finishReason: "stop" }, { status: "completed" }],
+    [{ finishReason: "tool-calls" }, { status: "completed" }],
+    [{}, { status: "completed" }],
+    [{ finishReason: "other" }, { status: "completed" }],
+    [{ finishReason: "error" }, { status: "error" }],
+  ])("maps %o", (input, expected) => {
+    expect(deriveRunOutcome(input)).toEqual(expected);
+  });
+
+  it("uses the fallback status only when the event carries no finish reason", () => {
+    expect(deriveRunOutcome({}, "incomplete")).toEqual({
+      status: "incomplete",
+    });
+    expect(deriveRunOutcome({ finishReason: "stop" }, "incomplete")).toEqual({
+      status: "completed",
+    });
+    expect(deriveRunOutcome({ finishReason: "length" }, "incomplete")).toEqual({
+      status: "incomplete",
+      outcome: "length",
+    });
+    expect(deriveRunOutcome({ isError: true }, "incomplete")).toEqual({
+      status: "error",
+    });
+  });
+});
+
+describe("describeRunError", () => {
+  it("reads the message and the class name of an AI SDK error", () => {
+    const error = new Error("Rate limited");
+    error.name = "AI_APICallError";
+    expect(describeRunError(error)).toEqual({
+      error: "Rate limited",
+      errorCode: "AI_APICallError",
+    });
+  });
+
+  it("prefers an explicit code and skips the plain Error name", () => {
+    expect(
+      describeRunError(Object.assign(new Error("boom"), { code: "ETIMEDOUT" })),
+    ).toEqual({ error: "boom", errorCode: "ETIMEDOUT" });
+    expect(describeRunError(new Error("boom"))).toEqual({ error: "boom" });
+    expect(describeRunError("boom")).toEqual({ error: "boom" });
+    expect(describeRunError(undefined)).toEqual({});
+    expect(describeRunError(42)).toEqual({});
+  });
+
+  it("clamps the message and code to what the runs endpoint accepts", () => {
+    const error = new Error("m".repeat(3000));
+    error.name = "c".repeat(100);
+    expect(describeRunError(error)).toEqual({
+      error: "m".repeat(2048),
+      errorCode: "c".repeat(64),
+    });
+  });
+});
+
+describe("createRunReport", () => {
+  const toolCall = {
+    tool_name: "weather",
+    tool_call_id: "call-1",
+    tool_args: '{"city":"Singapore"}',
+  };
+
+  it("creates the full aui/v0 report", () => {
+    expect(
+      createRunReport({
+        threadId: "aui-thread",
+        status: "incomplete",
+        outcome: "length",
+        errorCode: "model_limit",
+        error: "response cut off",
+        messageId: "cloud-message",
+        traceId: "AABBCCDDEEFF00112233445566778899",
+        modelId: "provider/model",
+        provider: "openai",
+        usage: {
+          promptTokens: 11,
+          completionTokens: 7,
+          reasoningTokens: 3,
+          cachedInputTokens: 2,
+        },
+        steps: [
+          {
+            usage: { inputTokens: 5, outputTokens: 4 },
+            toolCalls: [toolCall],
+            startMs: 10,
+            endMs: 20,
+            finishReason: "tool-calls",
+          },
+          {
+            usage: { inputTokens: 6, outputTokens: 3, reasoningTokens: 3 },
+            startMs: 21,
+            endMs: 42,
+            finishReason: "length".repeat(8),
+          },
+        ],
+        toolCalls: [toolCall],
+        durationMs: 42.5,
+        firstTokenMs: -4,
+        outputText: "hello from aui/v0",
+        metadata: { tenant: "acme" },
+        telemetry: {
+          environment: "production",
+          release: "web-2026.09.09",
+          tags: [" region:sg ", "region:sg", "tier:paid"],
+        },
+      }),
+    ).toEqual({
+      thread_id: "aui-thread",
+      status: "incomplete",
+      outcome_type: "length",
+      error_code: "model_limit",
+      error: "response cut off",
+      message_id: "cloud-message",
+      trace_id: "aabbccddeeff00112233445566778899",
+      model_id: "provider/model",
+      provider: "openai",
+      provider_type: "openai",
+      input_tokens: 11,
+      output_tokens: 7,
+      reasoning_tokens: 3,
+      cached_input_tokens: 2,
+      steps: [
+        {
+          input_tokens: 5,
+          output_tokens: 4,
+          tool_calls: [toolCall],
+          start_ms: 10,
+          end_ms: 20,
+          finish_reason: "tool-calls",
+        },
+        {
+          input_tokens: 6,
+          output_tokens: 3,
+          reasoning_tokens: 3,
+          start_ms: 21,
+          end_ms: 42,
+          finish_reason: "lengthlengthlengthlengthlengthle",
+        },
+      ],
+      total_steps: 2,
+      tool_calls: [toolCall],
+      duration_ms: 43,
+      first_token_ms: 0,
+      output_text: "hello from aui/v0",
+      metadata: { tenant: "acme" },
+      environment: "production",
+      release: "web-2026.09.09",
+      tags: ["region:sg", "tier:paid"],
+    });
+  });
+
+  it("creates the full ai-sdk/v6 report", () => {
+    expect(
+      createRunReport({
+        threadId: "ai-sdk-thread",
+        status: "completed",
+        traceId: "00112233445566778899aabbccddeeff",
+        modelId: "gpt-5.6-terra",
+        provider: "gateway",
+        usage: {
+          inputTokens: 100,
+          outputTokens: 50,
+          inputTokenDetails: { cacheReadTokens: 10 },
+          outputTokenDetails: { reasoningTokens: 20 },
+        },
+        steps: [
+          {
+            usage: {
+              inputTokens: 100,
+              outputTokens: 50,
+              inputTokenDetails: { cacheReadTokens: 10 },
+              outputTokenDetails: { reasoningTokens: 20 },
+            },
+            finishReason: "stop",
+          },
+        ],
+        durationMs: 125.2,
+        firstTokenMs: 12.6,
+        outputText: "hello from ai-sdk/v6",
+        telemetry: { tags: ["sdk:v6"] },
+      }),
+    ).toEqual({
+      thread_id: "ai-sdk-thread",
+      status: "completed",
+      trace_id: "00112233445566778899aabbccddeeff",
+      model_id: "gpt-5.6-terra",
+      provider: "gateway",
+      provider_type: "gateway",
+      input_tokens: 100,
+      output_tokens: 50,
+      reasoning_tokens: 20,
+      cached_input_tokens: 10,
+      steps: [
+        {
+          input_tokens: 100,
+          output_tokens: 50,
+          reasoning_tokens: 20,
+          cached_input_tokens: 10,
+          finish_reason: "stop",
+        },
+      ],
+      total_steps: 1,
+      duration_ms: 125,
+      first_token_ms: 13,
+      output_text: "hello from ai-sdk/v6",
+      tags: ["sdk:v6"],
+    });
+  });
+
+  it("normalizes tags and limits them to twenty 64-character values", () => {
+    const longTag = "a".repeat(70);
+    const tags = [" one ", "", "one", longTag];
+    for (let index = 0; index < 24; index++) tags.push(`tag:${index}`);
+
+    const report = createRunReport({
+      threadId: "thread",
+      status: "completed",
+      telemetry: { tags },
+    });
+
+    expect(report.tags).toEqual([
+      "one",
+      "a".repeat(64),
+      ...Array.from({ length: 18 }, (_, index) => `tag:${index}`),
+    ]);
+  });
+
+  it("trims the whitespace a 64-character cut leaves behind", () => {
+    const report = createRunReport({
+      threadId: "thread",
+      status: "completed",
+      telemetry: { tags: [`${"b".repeat(63)} tail`] },
+    });
+
+    expect(report.tags).toEqual(["b".repeat(63)]);
+  });
+
+  it("lowercases valid trace IDs, omits invalid IDs, and omits undefined keys", () => {
+    expect(
+      createRunReport({
+        threadId: "thread",
+        status: "completed",
+        traceId: "AABBCCDDEEFF00112233445566778899",
+      }).trace_id,
+    ).toBe("aabbccddeeff00112233445566778899");
+    expect(
+      createRunReport({
+        threadId: "thread",
+        status: "completed",
+        traceId: "aabbccdd",
+      }),
+    ).toEqual({ thread_id: "thread", status: "completed" });
+    expect(
+      createRunReport({
+        threadId: "thread",
+        status: "completed",
+      }),
+    ).toEqual({ thread_id: "thread", status: "completed" });
+  });
+});
+
 describe("truncateRunTelemetryText", () => {
   it("passes text at or under the cap through unchanged", () => {
     expect(truncateRunTelemetryText("hello")).toBe("hello");
@@ -169,3 +453,44 @@ describe("normalizeRunTelemetryUsage", () => {
     ).toBeUndefined();
   });
 });
+
+describe("extractRunTelemetryModelId", () => {
+  it("prefers an explicit model ID over every fallback", () => {
+    expect(
+      extractRunTelemetryModelId({
+        modelId: "explicit",
+        custom: { modelId: "custom" },
+        steps: [{ response: { modelId: "step" } }],
+      }),
+    ).toBe("explicit");
+  });
+
+  it("falls back to the custom bag before the steps", () => {
+    expect(
+      extractRunTelemetryModelId({
+        custom: { modelId: "custom" },
+        steps: [{ response: { modelId: "step" } }],
+      }),
+    ).toBe("custom");
+  });
+
+  it("reads the first step that carries a response model ID", () => {
+    expect(
+      extractRunTelemetryModelId({
+        steps: [{ usage: { inputTokens: 1 } }, { response: { modelId: "b" } }],
+      }),
+    ).toBe("b");
+  });
+
+  it("returns undefined when no channel carries one", () => {
+    expect(extractRunTelemetryModelId(undefined)).toBeUndefined();
+    expect(extractRunTelemetryModelId({})).toBeUndefined();
+    expect(
+      extractRunTelemetryModelId({
+        modelId: 7,
+        custom: { modelId: null },
+        steps: [null, "step", { response: null }, { response: { modelId: 7 } }],
+      }),
+    ).toBeUndefined();
+  });
+});

package/src/runTelemetry.ts

@@ -1,4 +1,5 @@
 import type { SamplingCallData } from "./instrumentMcpSampling";
+import type { AssistantCloudRunReport } from "./AssistantCloudRuns";
 
 const MAX_TELEMETRY_TEXT_LENGTH = 50_000;
 
@@ -15,6 +16,86 @@ export type AssistantCloudRunReportToolCall = {
   sampling_calls?: SamplingCallData[];
 };
 
+export type RunReportOutcome =
+  | "aborted"
+  | "disconnected"
+  | "length"
+  | "content_filter";
+
+/**
+ * Maps a finish event to the report status and outcome. `fallbackStatus`
+ * applies when the event carries neither a finish reason nor a failure flag.
+ */
+export function deriveRunOutcome(
+  input: {
+    finishReason?: string | undefined;
+    isAbort?: boolean | undefined;
+    isDisconnect?: boolean | undefined;
+    isError?: boolean | undefined;
+  },
+  fallbackStatus: "completed" | "incomplete" = "completed",
+): {
+  status: "completed" | "incomplete" | "error";
+  outcome?: RunReportOutcome;
+} {
+  if (input.isError) return { status: "error" };
+  if (input.isAbort) return { status: "incomplete", outcome: "aborted" };
+  if (input.isDisconnect) {
+    return { status: "incomplete", outcome: "disconnected" };
+  }
+  switch (input.finishReason) {
+    case "length":
+      return { status: "incomplete", outcome: "length" };
+    case "content-filter":
+    case "content_filter":
+      return { status: "incomplete", outcome: "content_filter" };
+    case "cancelled":
+      return { status: "incomplete", outcome: "aborted" };
+    case "error":
+      return { status: "error" };
+    default:
+      return {
+        status: input.finishReason === undefined ? fallbackStatus : "completed",
+      };
+  }
+}
+
+const MAX_RUN_ERROR_CODE_LENGTH = 64;
+const MAX_RUN_ERROR_LENGTH = 2048;
+
+/**
+ * Reads the message and code the runs endpoint stores for a failed run. The
+ * code is the error's `code` when it has one, else its class name.
+ */
+export function describeRunError(error: unknown): {
+  error?: string;
+  errorCode?: string;
+} {
+  if (error == null) return {};
+  const record =
+    typeof error === "object" ? (error as Record<string, unknown>) : undefined;
+  const message =
+    typeof error === "string"
+      ? error
+      : typeof record?.message === "string"
+        ? record.message
+        : undefined;
+  const code =
+    typeof record?.code === "string"
+      ? record.code
+      : typeof record?.name === "string" && record.name !== "Error"
+        ? record.name
+        : undefined;
+  return {
+    ...(message
+      ? { error: message.slice(0, MAX_RUN_ERROR_LENGTH) }
+      : undefined),
+    ...(code
+      ? { errorCode: code.slice(0, MAX_RUN_ERROR_CODE_LENGTH) }
+      : undefined),
+  };
+}
+
 /**
  * Clamps a string to the size the runs endpoint accepts for a single span
  * field.
@@ -95,20 +176,190 @@ export function createRunTelemetryToolCall(
   return call;
 }
 
+/**
+ * Resolves the model ID a run reports, in the order an app can supply it: an
+ * explicit `modelId`, the `custom` bag, then the per-step `response.modelId`
+ * that a `messageMetadata` callback copies off the AI SDK's finish-step part.
+ * The AI SDK puts no model ID on a UI message part, so message metadata is the
+ * only channel one arrives on.
+ */
+export function extractRunTelemetryModelId(
+  metadata: Record<string, unknown> | undefined,
+): string | undefined {
+  if (!metadata) return undefined;
+  if (typeof metadata.modelId === "string") return metadata.modelId;
+  const custom = metadata.custom as Record<string, unknown> | undefined;
+  if (typeof custom?.modelId === "string") return custom.modelId;
+
+  const steps: unknown = metadata.steps;
+  if (!Array.isArray(steps)) return undefined;
+  for (const step of steps as unknown[]) {
+    if (!step || typeof step !== "object") continue;
+    const response = (step as Record<string, unknown>).response;
+    if (!response || typeof response !== "object") continue;
+    const modelId = (response as Record<string, unknown>).modelId;
+    if (typeof modelId === "string") return modelId;
+  }
+  return undefined;
+}
+
 export type RunTelemetryUsage = {
-  inputTokens?: number;
-  outputTokens?: number;
-  reasoningTokens?: number;
-  cachedInputTokens?: number;
+  inputTokens?: number | undefined;
+  outputTokens?: number | undefined;
+  reasoningTokens?: number | undefined;
+  cachedInputTokens?: number | undefined;
 };
 
 export type RunTelemetryUsageInit = RunTelemetryUsage & {
-  promptTokens?: number;
-  completionTokens?: number;
+  promptTokens?: number | undefined;
+  completionTokens?: number | undefined;
   inputTokenDetails?: { cacheReadTokens?: number };
   outputTokenDetails?: { reasoningTokens?: number };
 };
 
+export type RunReportStepInit = {
+  usage?: RunTelemetryUsageInit | undefined;
+  toolCalls?: AssistantCloudRunReportToolCall[] | undefined;
+  startMs?: number | undefined;
+  endMs?: number | undefined;
+  finishReason?: string | undefined;
+};
+
+export type RunReportInit = {
+  threadId: string;
+  status: AssistantCloudRunReport["status"];
+  outcome?: RunReportOutcome | undefined;
+  errorCode?: string | undefined;
+  error?: string | undefined;
+  messageId?: string | undefined;
+  traceId?: string | undefined;
+  modelId?: string | undefined;
+  provider?: string | undefined;
+  usage?: RunTelemetryUsageInit | undefined;
+  steps?: RunReportStepInit[] | undefined;
+  totalSteps?: number | undefined;
+  toolCalls?: AssistantCloudRunReportToolCall[] | undefined;
+  durationMs?: number | undefined;
+  firstTokenMs?: number | undefined;
+  outputText?: string | undefined;
+  metadata?: Record<string, unknown> | undefined;
+  telemetry?: {
+    environment?: string | undefined;
+    release?: string | undefined;
+    tags?: readonly string[] | undefined;
+  };
+};
+
+function assignUsage(
+  report: AssistantCloudRunReport,
+  usage: RunTelemetryUsageInit | undefined,
+): void {
+  if (!usage) return;
+  const normalized = normalizeRunTelemetryUsage(usage);
+  if (!normalized) return;
+  if (normalized.inputTokens !== undefined) {
+    report.input_tokens = normalized.inputTokens;
+  }
+  if (normalized.outputTokens !== undefined) {
+    report.output_tokens = normalized.outputTokens;
+  }
+  if (normalized.reasoningTokens !== undefined) {
+    report.reasoning_tokens = normalized.reasoningTokens;
+  }
+  if (normalized.cachedInputTokens !== undefined) {
+    report.cached_input_tokens = normalized.cachedInputTokens;
+  }
+}
+
+function createRunReportStep(
+  init: RunReportStepInit,
+): NonNullable<AssistantCloudRunReport["steps"]>[number] {
+  const step: NonNullable<AssistantCloudRunReport["steps"]>[number] = {};
+  const usage = init.usage ? normalizeRunTelemetryUsage(init.usage) : undefined;
+  if (usage?.inputTokens !== undefined) step.input_tokens = usage.inputTokens;
+  if (usage?.outputTokens !== undefined)
+    step.output_tokens = usage.outputTokens;
+  if (usage?.reasoningTokens !== undefined) {
+    step.reasoning_tokens = usage.reasoningTokens;
+  }
+  if (usage?.cachedInputTokens !== undefined) {
+    step.cached_input_tokens = usage.cachedInputTokens;
+  }
+  if (init.toolCalls !== undefined) step.tool_calls = init.toolCalls;
+  if (init.startMs !== undefined) step.start_ms = init.startMs;
+  if (init.endMs !== undefined) step.end_ms = init.endMs;
+  if (init.finishReason !== undefined) {
+    step.finish_reason = init.finishReason.slice(0, 32);
+  }
+  return step;
+}
+
+function normalizeRunReportTags(
+  tags: readonly string[] | undefined,
+): string[] | undefined {
+  if (!tags) return undefined;
+  const result: string[] = [];
+  const seen = new Set<string>();
+  for (const tag of tags) {
+    const normalized = tag.trim().slice(0, 64).trim();
+    if (!normalized || seen.has(normalized)) continue;
+    seen.add(normalized);
+    result.push(normalized);
+    if (result.length === 20) break;
+  }
+  return result.length > 0 ? result : undefined;
+}
+
+function normalizeRunReportMilliseconds(
+  value: number | undefined,
+): number | undefined {
+  if (value === undefined || !Number.isFinite(value)) return undefined;
+  return Math.max(0, Math.round(value));
+}
+
+export function createRunReport(init: RunReportInit): AssistantCloudRunReport {
+  const report: AssistantCloudRunReport = {
+    thread_id: init.threadId,
+    status: init.status,
+  };
+  const traceId = init.traceId?.toLowerCase();
+  if (traceId && /^[0-9a-f]{32}$/.test(traceId)) report.trace_id = traceId;
+  if (init.outcome !== undefined) report.outcome_type = init.outcome;
+  if (init.errorCode !== undefined) report.error_code = init.errorCode;
+  if (init.error !== undefined) report.error = init.error;
+  if (init.messageId !== undefined) report.message_id = init.messageId;
+  if (init.modelId !== undefined) report.model_id = init.modelId;
+  if (init.provider !== undefined) {
+    report.provider = init.provider;
+    report.provider_type = init.provider;
+  }
+  assignUsage(report, init.usage);
+  if (init.steps !== undefined) {
+    report.steps = init.steps.map(createRunReportStep);
+    report.total_steps = init.steps.length;
+  } else if (init.totalSteps !== undefined) {
+    report.total_steps = init.totalSteps;
+  }
+  if (init.toolCalls !== undefined) report.tool_calls = init.toolCalls;
+  const durationMs = normalizeRunReportMilliseconds(init.durationMs);
+  if (durationMs !== undefined) report.duration_ms = durationMs;
+  const firstTokenMs = normalizeRunReportMilliseconds(init.firstTokenMs);
+  if (firstTokenMs !== undefined) report.first_token_ms = firstTokenMs;
+  if (init.outputText !== undefined) {
+    report.output_text = truncateRunTelemetryText(init.outputText);
+  }
+  if (init.metadata !== undefined) report.metadata = init.metadata;
+  if (init.telemetry?.environment !== undefined) {
+    report.environment = init.telemetry.environment;
+  }
+  if (init.telemetry?.release !== undefined) {
+    report.release = init.telemetry.release;
+  }
+  const tags = normalizeRunReportTags(init.telemetry?.tags);
+  if (tags !== undefined) report.tags = tags;
+  return report;
+}
+
 /**
  * Resolves the token counts a provider reports under any of the names the AI
  * SDK has used: the current top-level ones, the legacy prompt/completion pair,