@elizaos/plugin-elizacloud 2.0.0-beta.1 → 2.0.11-beta.7

package/src/models/text.ts

@@ -3,19 +3,20 @@ import type {
   IAgentRuntime,
   ModelTypeName,
   TextStreamResult,
+  TokenUsage,
 } from "@elizaos/core";
 import {
   buildCanonicalSystemPrompt,
   logger,
   ModelType,
+  recordInferenceSpan,
   renderChatMessagesForPrompt,
   resolveEffectiveSystemPrompt,
+  Semaphore,
+  timeInferenceSpan,
 } from "@elizaos/core";
-import type { LanguageModel } from "ai";
-import { createOpenAIClient } from "../providers/openai";
 import {
   getActionPlannerModel,
-  getExperimentalTelemetry,
   getLargeModel,
   getMediumModel,
   getMegaModel,
@@ -36,6 +37,150 @@ const RESPONSE_HANDLER_MODEL_TYPE = (ModelType.RESPONSE_HANDLER ??
   "RESPONSE_HANDLER") as ModelTypeName;
 const ACTION_PLANNER_MODEL_TYPE = (ModelType.ACTION_PLANNER ?? "ACTION_PLANNER") as ModelTypeName;
 
+/**
+ * Per-process cap on CONCURRENT native cloud text calls.
+ *
+ * Covers BOTH native cloud text routes that share the one cerebras key:
+ * the `/chat/completions` round-trip (native-transport callers) AND the
+ * `/responses` round-trip (bare-`{ prompt }` callers, incl. the primary reply
+ * action). Same model name -> same shared key -> same concurrency budget, so
+ * both routes must funnel through this one semaphore or a bare-prompt call can
+ * still push the key over its limit.
+ *
+ * The per-turn burst that triggers the 429 comes from the prompt BATCHER
+ * (`dynamicPromptExecFromState`, which always sets providerOptions -> native
+ * `/chat/completions`) and the merged evaluator call — NOT from composeState
+ * providers (no provider calls `useModel` during composeState). Firing those
+ * at once overruns the ONE shared cerebras key's concurrent-request limit
+ * -> 429 -> 3 retries x backoff -> 30-63s of latency. Capping in-flight calls
+ * through a small semaphore keeps each call ~3s with no 429, without needing
+ * more keys or backend changes.
+ *
+ * Default is a SAFETY CEILING, not full serialization: the paid cerebras key
+ * (1000 req/min) and leaner per-turn call counts make the 429 risk small, so
+ * the default of 8 leaves the typical 1-3 concurrent calls/turn untouched while
+ * still bounding a pathological burst. The limiter is process-global and keys
+ * on native transport, not the model, so it also bounds non-cerebras native
+ * calls (e.g. zai-glm-4.7) — a high default avoids serializing those. Set
+ * `ELIZAOS_CLOUD_NATIVE_CONCURRENCY` (positive integer) to tighten it (1 = fully
+ * serialize) on a cerebras-bottlenecked single-key deployment, or raise it for
+ * more parallelism. Embeddings use a SEPARATE `/embeddings` route
+ * (embeddings.ts) and are intentionally NOT gated here.
+ */
+const NATIVE_CONCURRENCY_ENV = "ELIZAOS_CLOUD_NATIVE_CONCURRENCY";
+const DEFAULT_NATIVE_CONCURRENCY = 8;
+
+/**
+ * Client-side timeout for cloud text round-trips. Without this the handler
+ * passes no `timeoutMs`/`signal` to `requestRaw`, so a hung/slow gateway holds
+ * the concurrency permit AND stalls the whole turn until fetch's own (very
+ * long) default. `ELIZAOS_CLOUD_TEXT_TIMEOUT_MS` overrides; `0`/negative opts
+ * out (no client-side timeout).
+ */
+const TEXT_TIMEOUT_ENV = "ELIZAOS_CLOUD_TEXT_TIMEOUT_MS";
+const DEFAULT_TEXT_TIMEOUT_MS = 120_000;
+
+export function resolveTextTimeoutMs(): number | undefined {
+  const raw =
+    typeof process !== "undefined" ? process.env[TEXT_TIMEOUT_ENV] : undefined;
+  if (raw === undefined || raw.trim() === "") return DEFAULT_TEXT_TIMEOUT_MS;
+  const parsed = Number.parseInt(raw, 10);
+  if (!Number.isFinite(parsed)) return DEFAULT_TEXT_TIMEOUT_MS;
+  return parsed <= 0 ? undefined : parsed;
+}
+
+/**
+ * Token-by-token streaming of the native `/chat/completions` round-trip. On by
+ * default so the user-visible reply renders from the first token instead of
+ * waiting for the whole generation. `ELIZAOS_CLOUD_STREAMING=0`/`false`/`off`
+ * forces the buffered path (kill-switch). Streaming only engages when the
+ * runtime actually requests it (`params.stream`), so non-streaming callers
+ * (connectors with no UI stream) are unaffected.
+ */
+const STREAMING_ENV = "ELIZAOS_CLOUD_STREAMING";
+
+export function resolveStreamingEnabled(): boolean {
+  const raw = typeof process !== "undefined" ? process.env[STREAMING_ENV] : undefined;
+  if (raw === undefined) return true;
+  const v = raw.trim().toLowerCase();
+  return v !== "0" && v !== "false" && v !== "off";
+}
+
+/**
+ * Combine the runtime's abort signal with the client-side timeout into one
+ * signal for `requestRaw`. A stream is long-lived, so it should abort on EITHER
+ * a caller cancel OR the timeout — `requestRaw` honors only a single signal, so
+ * merge them here.
+ */
+function buildStreamAbortSignal(
+  abortSignal: AbortSignal | undefined,
+  timeoutMs: number | undefined
+): AbortSignal | undefined {
+  const timeoutSig =
+    typeof timeoutMs === "number" && timeoutMs > 0
+      ? AbortSignal.timeout(timeoutMs)
+      : undefined;
+  if (abortSignal && timeoutSig) return AbortSignal.any([abortSignal, timeoutSig]);
+  return abortSignal ?? timeoutSig;
+}
+
+let nativeChatLimiter: Semaphore | null = null;
+
+function resolveNativeConcurrency(): number {
+  const raw =
+    typeof process !== "undefined" ? process.env[NATIVE_CONCURRENCY_ENV] : undefined;
+  const parsed = raw ? Number.parseInt(raw, 10) : Number.NaN;
+  return Number.isFinite(parsed) && parsed > 0 ? parsed : DEFAULT_NATIVE_CONCURRENCY;
+}
+
+function getNativeChatLimiter(): Semaphore {
+  if (!nativeChatLimiter) {
+    nativeChatLimiter = new Semaphore(resolveNativeConcurrency());
+  }
+  return nativeChatLimiter;
+}
+
+/**
+ * Run a single cerebras-bound network round-trip under the shared per-process
+ * concurrency cap. Hold the permit only across `fn` (the `requestRaw` call);
+ * release the instant the server responds so response-body parsing runs
+ * unguarded. `finally` frees the permit even on throw so a failed call never
+ * starves the queue. Used by BOTH native text routes (`/chat/completions` and
+ * `/responses`) so every cerebras text call shares one budget.
+ *
+ * Exported for unit tests that drive the shared cap directly.
+ *
+ * `label` (e.g. `responses` / `chat/completions`) tags the latency spans this
+ * records on the active per-turn inference timer: `cloud.semaphore-wait` (time
+ * spent queued for a permit — non-zero means the cap is serializing) and
+ * `cloud.http:<label>` (the network round-trip). Both are no-ops when no turn
+ * timer is active.
+ */
+export async function withNativeChatLimit<T>(
+  fn: () => Promise<T>,
+  label = "native"
+): Promise<T> {
+  const limiter = getNativeChatLimiter();
+  const waitStartedAt = Date.now();
+  await limiter.acquire();
+  recordInferenceSpan("cloud.semaphore-wait", Date.now() - waitStartedAt, {
+    route: label,
+  });
+  try {
+    return await timeInferenceSpan(`cloud.http:${label}`, fn, { route: label });
+  } finally {
+    limiter.release();
+  }
+}
+
+/**
+ * Test-only: discard the cached limiter so the next call re-reads the env knob.
+ * Production code never needs this — the knob is read once per process.
+ */
+export function __resetNativeChatLimiterForTests(): void {
+  nativeChatLimiter = null;
+}
+
 type ResponsesApiResponse = Record<string, unknown> & {
   error?: {
     message?: string;
@@ -61,7 +206,6 @@ const REASONING_MODEL_PATTERNS = [
   "claude-opus-4-7",
   "gpt-5",
 ] as const;
-const RESPONSES_ROUTED_PREFIXES = ["openai/", "anthropic/"] as const;
 type ChatAttachment = {
   data: string | Uint8Array | URL;
   mediaType: string;
@@ -121,37 +265,87 @@ type ChatCompletionsResponse = Record<string, unknown> & {
   usage?: Record<string, unknown>;
 };
 
-function buildUserContent(params: GenerateTextParamsWithAttachments) {
-  const content: Array<
-    | { type: "text"; text: string }
-    | {
-        type: "file";
-        data: string | Uint8Array | URL;
-        mediaType: string;
-        filename?: string;
-      }
-  > = [{ type: "text", text: params.prompt ?? "" }];
-
-  for (const attachment of params.attachments ?? []) {
-    content.push({
-      type: "file",
-      data: attachment.data,
-      mediaType: attachment.mediaType,
-      ...(attachment.filename ? { filename: attachment.filename } : {}),
-    });
-  }
+/**
+ * Eliza-Cloud-hosted `eliza-1` model ids that run a fork of llama-server (or
+ * vLLM with the eliza1 parsers) capable of honoring the `x-eliza-span-samplers`
+ * header. Other upstreams (OpenAI / Anthropic / generic OpenRouter) strip
+ * unknown headers safely, but to keep the wire surface narrow we only attach
+ * the per-span sampler plan when the resolved model is one we know honors it.
+ *
+ * The "we know" bound is conservative — extend the prefix list when a new
+ * fork-built deployment lands. The fallback is "do not send the header" which
+ * preserves today's behavior on every other provider.
+ */
+const SPAN_SAMPLER_HONORING_MODEL_PREFIXES = [
+  "vast/eliza-1-",
+  "elizaos/eliza-1-",
+  "eliza-1-",
+] as const;
 
-  return content;
+function isSpanSamplerHonoringModel(modelName: string): boolean {
+  const lower = modelName.toLowerCase();
+  return SPAN_SAMPLER_HONORING_MODEL_PREFIXES.some((prefix) =>
+    lower.startsWith(prefix),
+  );
 }
 
-function isReasoningModel(modelName: string): boolean {
-  const lower = modelName.toLowerCase();
-  return REASONING_MODEL_PATTERNS.some((pattern) => lower.includes(pattern));
+/**
+ * Build the `x-eliza-span-samplers` HTTP header value from a {@link SpanSamplerPlan}.
+ * Returns `undefined` when there is no plan or no overrides — narrow the wire
+ * surface so non-eliza providers never see a stray fork-extension header.
+ *
+ * Wire schema (snake_case):
+ *   { overrides: [{ span_index, temperature, top_k?, top_p? }, ...], strict?: boolean }
+ */
+function buildSpanSamplerHeader(
+  plan: GenerateTextParams["spanSamplerPlan"],
+): string | undefined {
+  if (!plan || plan.overrides.length === 0) return undefined;
+  const overrides = plan.overrides.map((o) => {
+    const wire: Record<string, unknown> = {
+      span_index: o.spanIndex,
+      temperature: o.temperature,
+    };
+    if (typeof o.topK === "number") wire.top_k = o.topK;
+    if (typeof o.topP === "number") wire.top_p = o.topP;
+    return wire;
+  });
+  const body: Record<string, unknown> = { overrides };
+  if (plan.strict === true) body.strict = true;
+  return JSON.stringify(body);
 }
 
-function supportsStopSequences(modelName: string): boolean {
+/**
+ * Extract the authoritative USD cost the metered cloud gateway charged for a
+ * request, when it surfaces one. The gateway is the only honest source of USD
+ * (it owns the model-pricing table + platform markup); we prefer it over any
+ * client-side token estimate. Checks the response body `usage.cost_usd` first,
+ * then the `X-Eliza-Cost-Usd` response header. Returns undefined when neither
+ * is present so consumers fall back to a token-based estimate.
+ */
+function extractCostUsd(
+  usage: unknown,
+  response?: { headers?: { get?: (name: string) => string | null } }
+): number | undefined {
+  const fromBody = firstNumber(
+    asRecord(usage).cost_usd,
+    asRecord(usage).costUsd,
+    asRecord(usage).cost
+  );
+  if (typeof fromBody === "number" && Number.isFinite(fromBody)) {
+    return fromBody;
+  }
+  const header = response?.headers?.get?.("X-Eliza-Cost-Usd");
+  if (header) {
+    const parsed = Number(header);
+    if (Number.isFinite(parsed)) return parsed;
+  }
+  return undefined;
+}
+
+function isReasoningModel(modelName: string): boolean {
   const lower = modelName.toLowerCase();
-  return !RESPONSES_ROUTED_PREFIXES.some((prefix) => lower.startsWith(prefix));
+  return REASONING_MODEL_PATTERNS.some((pattern) => lower.includes(pattern));
 }
 
 function isRecord(value: unknown): value is Record<string, unknown> {
@@ -256,30 +450,60 @@ function unwrapJsonSchema(value: unknown): unknown {
   return record.schema ?? record.jsonSchema ?? value;
 }
 
-function normalizeNativeTools(tools: unknown): unknown[] | undefined {
+// Normalize a single tool entry into the OpenAI `{ type, function }` wire
+// shape. Accepts BOTH the already-nested form (`{ type: "function", function:
+// { name, parameters } }`) and core's FLAT `ToolDefinition` envelope
+// (`{ name, type: "function", parameters }`, e.g. createHandleResponseTool /
+// the action planner). Returning the flat form verbatim made the cloud gateway
+// read `tool.function.name` on an undefined `function` → "Cannot read
+// properties of undefined (reading 'name')". Returns undefined for entries with
+// no resolvable name so they are dropped rather than crashing downstream.
+function normalizeNativeToolEntry(
+  rawTool: unknown,
+  fallbackName?: string
+): Record<string, unknown> | undefined {
+  const tool = asRecord(rawTool);
+  const nested = asRecord(tool.function);
+  const name = firstString(nested.name, tool.name, fallbackName);
+  if (!name) {
+    return undefined;
+  }
+  const description = firstString(nested.description, tool.description);
+  const inputSchema = unwrapJsonSchema(
+    nested.parameters ??
+      tool.inputSchema ??
+      tool.parameters ??
+      tool.schema ?? { type: "object" }
+  );
+  return {
+    type: "function",
+    function: {
+      name,
+      ...(description ? { description } : {}),
+      parameters: inputSchema,
+    },
+  };
+}
+
+export function normalizeNativeTools(tools: unknown): unknown[] | undefined {
   if (!tools) {
     return undefined;
   }
 
   if (Array.isArray(tools)) {
-    return tools;
+    const normalized = tools
+      .map((tool) => normalizeNativeToolEntry(tool))
+      .filter((tool): tool is Record<string, unknown> => tool !== undefined);
+    return normalized.length > 0 ? normalized : undefined;
   }
 
   const toolSet = asRecord(tools);
   const normalized: unknown[] = [];
   for (const [name, rawTool] of Object.entries(toolSet)) {
-    const tool = asRecord(rawTool);
-    const inputSchema = unwrapJsonSchema(
-      tool.inputSchema ?? tool.parameters ?? tool.schema ?? { type: "object" }
-    );
-    normalized.push({
-      type: "function",
-      function: {
-        name,
-        ...(typeof tool.description === "string" ? { description: tool.description } : {}),
-        parameters: inputSchema,
-      },
-    });
+    const entry = normalizeNativeToolEntry(rawTool, name);
+    if (entry) {
+      normalized.push(entry);
+    }
   }
 
   return normalized.length > 0 ? normalized : undefined;
@@ -604,40 +828,24 @@ function getModelNameForType(runtime: IAgentRuntime, modelType: TextModelType):
   }
 }
 
+/**
+ * Resolve the model name, rendered prompt, and effective system prompt for a
+ * cloud text call.
+ *
+ * This used to also construct a Vercel AI-SDK `LanguageModel` (`openai.chat()`)
+ * plus a full `generateParams` object — but the handlers below call the cloud
+ * HTTP API directly (`requestRaw` → `/responses` / `/chat/completions`), so that
+ * AI-SDK client + params object was built and immediately discarded on every
+ * single text generation. Removed: it was pure per-call overhead and a
+ * misleading code path when reasoning about which transport actually runs.
+ */
 function buildGenerateParams(
   runtime: IAgentRuntime,
   modelType: TextModelType,
   params: GenerateTextParams
 ) {
-  const paramsWithAttachments = params as GenerateTextParamsWithAttachments;
   const prompt = params.prompt ?? "";
-  const maxTokens = params.maxTokens ?? 8192;
-
-  const openai = createOpenAIClient(runtime);
   const modelName = getModelNameForType(runtime, modelType);
-  const experimentalTelemetry = getExperimentalTelemetry(runtime);
-  const userContent =
-    (paramsWithAttachments.attachments?.length ?? 0) > 0
-      ? buildUserContent(paramsWithAttachments)
-      : undefined;
-
-  // Use openai.chat() (Chat Completions API) instead of openai.languageModel()
-  // (Responses API). The Responses API unconditionally rejects presencePenalty,
-  // frequencyPenalty, and stopSequences for ALL models, emitting noisy warnings.
-  // The Chat Completions API supports these features natively and handles
-  // reasoning models gracefully when the params are omitted.
-  const model = openai.chat(modelName) as LanguageModel;
-
-  // Reasoning models don't support temperature, frequency/presence penalties,
-  // or stopSequences. Detect via model name patterns.
-  const reasoning = isReasoningModel(modelName);
-  const stopSequences =
-    !reasoning &&
-    supportsStopSequences(modelName) &&
-    Array.isArray(params.stopSequences) &&
-    params.stopSequences.length > 0
-      ? params.stopSequences
-      : undefined;
   const systemPrompt = resolveEffectiveSystemPrompt({
     params,
     fallback: buildCanonicalSystemPrompt({ character: runtime.character }),
@@ -647,20 +855,7 @@ function buildGenerateParams(
       omitDuplicateSystem: systemPrompt,
     }) ?? prompt;
 
-  const generateParams = {
-    model,
-    ...(userContent
-      ? { messages: [{ role: "user" as const, content: userContent }] }
-      : { prompt: promptText }),
-    system: systemPrompt,
-    ...(stopSequences ? { stopSequences } : {}),
-    maxOutputTokens: maxTokens,
-    experimental_telemetry: {
-      isEnabled: experimentalTelemetry,
-    },
-  };
-
-  return { generateParams, modelName, modelType, prompt: promptText, systemPrompt };
+  return { modelName, modelType, prompt: promptText, systemPrompt };
 }
 
 async function generateTextWithModel(
@@ -673,16 +868,33 @@ async function generateTextWithModel(
 
   logger.debug(`[ELIZAOS_CLOUD] Generating text with ${modelType} model: ${modelName}`);
 
-  if (params.stream) {
-    logger.debug(
-      "[ELIZAOS_CLOUD] Streaming text disabled for responses compatibility; falling back to buffered response."
-    );
-  }
+  // Stream the user-visible reply token-by-token. Gated to the structured
+  // reply path (`streamStructured`, set only by the RESPONSE_HANDLER stage-1
+  // call): that call carries a responseSkeleton, so the runtime's field
+  // extractor surfaces `replyText` incrementally to the UI. Planner/other
+  // native calls (no responseSkeleton) stay buffered — streaming their raw
+  // envelope would leak internals to the UI stream. The bare `/responses`
+  // route stays buffered too (different SSE schema, not on the reply path).
+  const paramsStreaming = params as {
+    stream?: boolean;
+    streamStructured?: boolean;
+  };
+  const wantsStream =
+    Boolean(paramsStreaming.stream) &&
+    paramsStreaming.streamStructured === true &&
+    resolveStreamingEnabled();
 
   logger.log(`[ELIZAOS_CLOUD] Using ${modelType} model: ${modelName}`);
   logger.log(prompt);
 
   if (hasNativeTransportOptions(paramsWithNative)) {
+    if (wantsStream) {
+      return streamNativeChatCompletion(runtime, modelType, paramsWithNative, {
+        modelName,
+        prompt,
+        systemPrompt,
+      });
+    }
     const nativeResult = await generateNativeChatCompletion(runtime, modelType, paramsWithNative, {
       modelName,
       prompt,
@@ -718,13 +930,27 @@ async function generateTextWithModel(
     requestBody.temperature = params.temperature;
   }
 
-  const response = await createCloudApiClient(runtime).requestRaw("POST", "/responses", {
-    headers: {
-      "X-Eliza-Llm-Purpose": getPurposeForModelType(modelType),
-      "X-Eliza-Model-Type": modelType,
-    },
-    json: requestBody,
-  });
+  const responsesHeaders: Record<string, string> = {
+    "X-Eliza-Llm-Purpose": getPurposeForModelType(modelType),
+    "X-Eliza-Model-Type": modelType,
+  };
+  if (isSpanSamplerHonoringModel(modelName)) {
+    const samplerHeader = buildSpanSamplerHeader(params.spanSamplerPlan);
+    if (samplerHeader) {
+      responsesHeaders["x-eliza-span-samplers"] = samplerHeader;
+    }
+  }
+  // Same shared cerebras key as the /chat/completions route, so gate this
+  // bare-prompt round-trip through the SAME limiter (parsing stays unguarded).
+  const response = await withNativeChatLimit(
+    () =>
+      createCloudApiClient(runtime).requestRaw("POST", "/responses", {
+        headers: responsesHeaders,
+        json: requestBody,
+        timeoutMs: resolveTextTimeoutMs(),
+      }),
+    "responses"
+  );
   const responseText = await response.text();
   let data: ResponsesApiResponse = {};
   if (responseText) {
@@ -757,11 +983,23 @@ async function generateTextWithModel(
   }
 
   if (data.usage) {
-    emitModelUsageEvent(runtime, modelType, prompt, {
-      inputTokens: data.usage.input_tokens ?? 0,
-      outputTokens: data.usage.output_tokens ?? 0,
-      totalTokens: data.usage.total_tokens ?? 0,
-    });
+    emitModelUsageEvent(
+      runtime,
+      modelType,
+      prompt,
+      {
+        inputTokens: data.usage.input_tokens ?? 0,
+        outputTokens: data.usage.output_tokens ?? 0,
+        totalTokens: data.usage.total_tokens ?? 0,
+      },
+      {
+        modelName: getModelNameForType(runtime, modelType),
+        ...(() => {
+          const costUsd = extractCostUsd(data.usage, response);
+          return typeof costUsd === "number" ? { costUsd } : {};
+        })(),
+      }
+    );
   }
 
   const text = extractResponsesOutputText(data);
@@ -772,7 +1010,9 @@ async function generateTextWithModel(
   return text;
 }
 
-async function generateNativeChatCompletion(
+// Exported for unit tests (the concurrency limiter wrapper). Not part of the
+// plugin's public model-handler surface.
+export async function generateNativeChatCompletion(
   runtime: IAgentRuntime,
   modelType: TextModelType,
   params: GenerateTextParamsWithNativeOptions,
@@ -788,13 +1028,35 @@ async function generateNativeChatCompletion(
     context.prompt,
     context.systemPrompt
   );
-  const response = await createCloudApiClient(runtime).requestRaw("POST", "/chat/completions", {
-    headers: {
-      "X-Eliza-Llm-Purpose": getPurposeForModelType(modelType),
-      "X-Eliza-Model-Type": modelType,
-    },
-    json: requestBody,
-  });
+  const headers: Record<string, string> = {
+    "X-Eliza-Llm-Purpose": getPurposeForModelType(modelType),
+    "X-Eliza-Model-Type": modelType,
+  };
+  // Per-span sampler overrides only ride along when the resolved model is a
+  // fork-built eliza-1 deployment that knows how to honor the header. Other
+  // upstreams (OpenAI / Anthropic / generic OpenRouter) strip unknown headers
+  // safely, but we keep the wire surface narrow until the cloud honor path
+  // lands in Wave 3.
+  if (isSpanSamplerHonoringModel(context.modelName)) {
+    const samplerHeader = buildSpanSamplerHeader(params.spanSamplerPlan);
+    if (samplerHeader) {
+      headers["x-eliza-span-samplers"] = samplerHeader;
+    }
+  }
+  // Serialize the per-turn batcher/evaluator burst through the SAME shared
+  // semaphore the /responses route uses, so N simultaneous native cloud text
+  // calls don't overrun the one shared cerebras key's concurrent limit (-> 429
+  // -> retries -> 30-63s). The permit is held only across the network
+  // round-trip; the text()/JSON parse below runs unguarded.
+  const response = await withNativeChatLimit(
+    () =>
+      createCloudApiClient(runtime).requestRaw("POST", "/chat/completions", {
+        headers,
+        json: requestBody,
+        timeoutMs: resolveTextTimeoutMs(),
+      }),
+    "chat/completions"
+  );
   const responseText = await response.text();
   let data: ChatCompletionsResponse = {};
   if (responseText) {
@@ -828,7 +1090,13 @@ async function generateNativeChatCompletion(
 
   const usage = convertNativeUsage(data.usage);
   if (usage) {
-    emitModelUsageEvent(runtime, modelType, context.prompt, usage);
+    emitModelUsageEvent(runtime, modelType, context.prompt, usage, {
+      modelName: context.modelName,
+      ...(() => {
+        const costUsd = extractCostUsd(data.usage, response);
+        return typeof costUsd === "number" ? { costUsd } : {};
+      })(),
+    });
   }
 
   const text = extractChatCompletionText(data);
@@ -849,6 +1117,332 @@ async function generateNativeChatCompletion(
   };
 }
 
+// ---------------------------------------------------------------------------
+// Streaming native /chat/completions (token-by-token, OpenAI-compatible SSE)
+// ---------------------------------------------------------------------------
+
+interface Deferred<T> {
+  promise: Promise<T>;
+  resolve: (value: T) => void;
+}
+
+function deferred<T>(): Deferred<T> {
+  let resolve!: (value: T) => void;
+  const promise = new Promise<T>((r) => {
+    resolve = r;
+  });
+  return { promise, resolve };
+}
+
+/**
+ * Parse an OpenAI-compatible SSE byte stream into the decoded JSON frame of
+ * each `data:` line. Yields one object per frame; stops at `data: [DONE]`.
+ * Tolerates partial reads (buffers across chunk boundaries) and ignores
+ * non-`data:` lines (comments, blank separators). Exported for unit tests.
+ */
+export async function* parseOpenAiSseStream(
+  body: ReadableStream<Uint8Array>
+): AsyncGenerator<Record<string, unknown>> {
+  const reader = body.getReader();
+  const decoder = new TextDecoder();
+  let buffer = "";
+  const handle = (line: string): Record<string, unknown> | "DONE" | null => {
+    const trimmed = line.trimStart();
+    if (!trimmed.startsWith("data:")) return null;
+    const payload = trimmed.slice(5).trim();
+    if (payload === "") return null;
+    if (payload === "[DONE]") return "DONE";
+    try {
+      return JSON.parse(payload) as Record<string, unknown>;
+    } catch {
+      return null;
+    }
+  };
+  try {
+    for (;;) {
+      const { value, done } = await reader.read();
+      if (done) break;
+      buffer += decoder.decode(value, { stream: true });
+      let nl: number;
+      while ((nl = buffer.indexOf("\n")) >= 0) {
+        const line = buffer.slice(0, nl);
+        buffer = buffer.slice(nl + 1);
+        const frame = handle(line);
+        if (frame === "DONE") return;
+        if (frame) yield frame;
+      }
+    }
+    const tail = handle(buffer);
+    if (tail && tail !== "DONE") yield tail;
+  } finally {
+    // cancel() (not just releaseLock()) tears down the underlying connection,
+    // so an EARLY consumer break (runtime abort / turn-supersede / a downstream
+    // throw closes this generator via .return()) stops the upstream generation
+    // instead of letting it run to its natural end and bill tokens nobody reads.
+    // On natural completion the stream is already done, so this is a no-op; it
+    // also releases the lock. Not threading the abort signal into the fetch on
+    // purpose — cancel() gets the teardown without rejecting an in-flight read
+    // with AbortError and changing the runtime's quiet-stop semantics.
+    try {
+      await reader.cancel();
+    } catch {
+      // Reader already cancelled/released by an upstream abort — nothing to do.
+    }
+  }
+}
+
+interface StreamingToolCallAcc {
+  id?: string;
+  name?: string;
+  args: string;
+}
+
+/** Fold one SSE `delta.tool_calls[]` array into the per-index accumulator. */
+export function accumulateToolCallDeltas(
+  acc: Map<number, StreamingToolCallAcc>,
+  deltas: unknown
+): void {
+  if (!Array.isArray(deltas)) return;
+  for (const raw of deltas) {
+    const d = asRecord(raw);
+    const index = typeof d.index === "number" ? d.index : 0;
+    const cur = acc.get(index) ?? { args: "" };
+    const id = firstString(d.id);
+    if (id) cur.id = id;
+    const fn = recordAt(d, "function");
+    const name = firstString(fn.name);
+    if (name) cur.name = name;
+    if (typeof fn.arguments === "string") cur.args += fn.arguments;
+    acc.set(index, cur);
+  }
+}
+
+/** Materialize accumulated tool-call deltas into the buffered-path shape. */
+export function finalizeStreamedToolCalls(
+  acc: Map<number, StreamingToolCallAcc>
+): NativeToolCall[] {
+  const out: NativeToolCall[] = [];
+  for (const [index, c] of [...acc.entries()].sort((a, b) => a[0] - b[0])) {
+    if (!c.name) continue;
+    out.push({
+      type: "tool-call",
+      toolCallId: c.id ?? `call_${c.name}_${index}`,
+      toolName: c.name,
+      input: parseJsonIfPossible(c.args.trim() === "" ? "{}" : c.args),
+    });
+  }
+  return out;
+}
+
+/**
+ * Streaming variant of {@link generateNativeChatCompletion}: returns a
+ * {@link TextStreamResult} whose `textStream` yields `delta.content` as it
+ * arrives, so `useModel`'s for-await loop streams it to the UI from the first
+ * token. Falls back to a single-chunk buffered result if the gateway answers
+ * non-SSE (self-healing). The shared concurrency permit is held for the whole
+ * stream lifetime (released in the generator's `finally`), not just until
+ * headers arrive — otherwise the cap would under-count in-flight requests.
+ */
+export async function streamNativeChatCompletion(
+  runtime: IAgentRuntime,
+  modelType: TextModelType,
+  params: GenerateTextParamsWithNativeOptions,
+  context: { modelName: string; prompt: string; systemPrompt?: string }
+): Promise<TextStreamResult> {
+  const requestBody = buildNativeRequestBody(
+    params,
+    context.modelName,
+    context.prompt,
+    context.systemPrompt
+  );
+  requestBody.stream = true;
+  // OpenAI-compatible: ask the server to include a final usage-only frame so we
+  // can meter the streamed call accurately.
+  requestBody.stream_options = { include_usage: true };
+
+  const headers: Record<string, string> = {
+    "X-Eliza-Llm-Purpose": getPurposeForModelType(modelType),
+    "X-Eliza-Model-Type": modelType,
+  };
+  if (isSpanSamplerHonoringModel(context.modelName)) {
+    const samplerHeader = buildSpanSamplerHeader(params.spanSamplerPlan);
+    if (samplerHeader) {
+      headers["x-eliza-span-samplers"] = samplerHeader;
+    }
+  }
+
+  const abortSignal = (params as { signal?: AbortSignal }).signal;
+  const signal = buildStreamAbortSignal(abortSignal, resolveTextTimeoutMs());
+
+  const limiter = getNativeChatLimiter();
+  const waitStartedAt = Date.now();
+  await limiter.acquire();
+  recordInferenceSpan("cloud.semaphore-wait", Date.now() - waitStartedAt, {
+    route: "chat/completions:stream",
+  });
+  let permitReleased = false;
+  const releasePermit = (): void => {
+    if (!permitReleased) {
+      permitReleased = true;
+      limiter.release();
+    }
+  };
+
+  let response: Response;
+  try {
+    response = await createCloudApiClient(runtime).requestRaw("POST", "/chat/completions", {
+      headers,
+      json: requestBody,
+      ...(signal ? { signal } : {}),
+    });
+  } catch (err) {
+    releasePermit();
+    throw err;
+  }
+
+  if (!response.ok) {
+    let errorBody: { message?: string } | undefined;
+    try {
+      const errText = await response.text();
+      if (errText) {
+        errorBody = (JSON.parse(errText) as ChatCompletionsResponse).error;
+      }
+    } catch {
+      // Non-JSON error body — fall through to the status-coded message.
+    }
+    releasePermit();
+    const message =
+      typeof errorBody?.message === "string" && errorBody.message.trim()
+        ? errorBody.message.trim()
+        : `elizaOS Cloud error ${response.status}`;
+    const requestError = new Error(message) as Error & {
+      status?: number;
+      error?: unknown;
+    };
+    requestError.status = response.status;
+    if (errorBody) requestError.error = errorBody;
+    throw requestError;
+  }
+
+  const contentType = response.headers.get("content-type") ?? "";
+  const isSse = contentType.includes("text/event-stream") && response.body !== null;
+
+  // Self-healing fallback: gateway answered with a buffered JSON body despite
+  // the stream request. Yield it as a single chunk so the streaming contract
+  // (and the structured-field extractor downstream) still works.
+  if (!isSse) {
+    const bufferedText = await response.text();
+    releasePermit();
+    let data: ChatCompletionsResponse = {};
+    if (bufferedText) {
+      try {
+        data = JSON.parse(bufferedText) as ChatCompletionsResponse;
+      } catch (parseErr) {
+        logger.error(
+          `[ELIZAOS_CLOUD] Failed to parse buffered chat completions JSON: ${
+            parseErr instanceof Error ? parseErr.message : String(parseErr)
+          }`
+        );
+      }
+    }
+    const text = extractChatCompletionText(data);
+    const toolCalls = extractNativeToolCalls(data);
+    const usage = convertNativeUsage(data.usage);
+    if (usage) {
+      emitModelUsageEvent(runtime, modelType, context.prompt, usage, {
+        modelName: context.modelName,
+        ...(() => {
+          const costUsd = extractCostUsd(data.usage, response);
+          return typeof costUsd === "number" ? { costUsd } : {};
+        })(),
+      });
+    }
+    if (!text.trim() && toolCalls.length === 0) {
+      throw new Error("elizaOS Cloud returned no text or tool calls");
+    }
+    async function* single(): AsyncGenerator<string> {
+      if (text) yield text;
+    }
+    return {
+      textStream: single(),
+      text: Promise.resolve(text),
+      usage: Promise.resolve(usage),
+      finishReason: Promise.resolve(data.choices?.[0]?.finish_reason),
+      toolCalls: Promise.resolve(toolCalls),
+      providerMetadata: { modelName: context.modelName, usage: data.usage },
+    };
+  }
+
+  const body = response.body as ReadableStream<Uint8Array>;
+  const toolAcc = new Map<number, StreamingToolCallAcc>();
+  let accumulated = "";
+  let nativeUsage: NativeTokenUsage | undefined;
+  let rawUsage: unknown;
+  let finishReason: string | undefined;
+
+  const textD = deferred<string>();
+  const usageD = deferred<TokenUsage | undefined>();
+  const finishD = deferred<string | undefined>();
+  const toolCallsD = deferred<NativeToolCall[]>();
+
+  async function* generate(): AsyncGenerator<string> {
+    try {
+      for await (const frame of parseOpenAiSseStream(body)) {
+        if (frame.error) {
+          const message = asRecord(frame.error).message;
+          throw new Error(
+            typeof message === "string" && message.trim()
+              ? message.trim()
+              : "elizaOS Cloud stream error"
+          );
+        }
+        const choices = Array.isArray(frame.choices) ? frame.choices : [];
+        const choice = asRecord(choices[0]);
+        const delta = recordAt(choice, "delta");
+        // Raw (un-trimmed) content — inter-token whitespace is significant.
+        if (typeof delta.content === "string" && delta.content.length > 0) {
+          accumulated += delta.content;
+          yield delta.content;
+        }
+        if (delta.tool_calls) {
+          accumulateToolCallDeltas(toolAcc, delta.tool_calls);
+        }
+        const fr = firstString(choice.finish_reason);
+        if (fr) finishReason = fr;
+        if (frame.usage) {
+          rawUsage = frame.usage;
+          nativeUsage = convertNativeUsage(frame.usage);
+        }
+      }
+    } finally {
+      releasePermit();
+      const toolCalls = finalizeStreamedToolCalls(toolAcc);
+      textD.resolve(accumulated);
+      usageD.resolve(nativeUsage);
+      finishD.resolve(finishReason);
+      toolCallsD.resolve(toolCalls);
+      if (nativeUsage) {
+        emitModelUsageEvent(runtime, modelType, context.prompt, nativeUsage, {
+          modelName: context.modelName,
+          ...(() => {
+            const costUsd = extractCostUsd(rawUsage, response);
+            return typeof costUsd === "number" ? { costUsd } : {};
+          })(),
+        });
+      }
+    }
+  }
+
+  return {
+    textStream: generate(),
+    text: textD.promise,
+    usage: usageD.promise,
+    finishReason: finishD.promise,
+    toolCalls: toolCallsD.promise,
+    providerMetadata: { modelName: context.modelName },
+  };
+}
+
 export async function handleTextSmall(
   runtime: IAgentRuntime,
   params: GenerateTextParams