@elizaos/plugin-elizacloud 2.0.0-alpha.8 → 2.0.11-beta.7

package/src/models/image.ts

@@ -0,0 +1,234 @@
+import type { IAgentRuntime, ImageDescriptionParams, ImageGenerationParams } from "@elizaos/core";
+import { logger, ModelType } from "@elizaos/core";
+import {
+  getImageDescriptionModel,
+  getImageGenerationModel,
+  getSetting,
+  resolveCloudTimeoutMs,
+} from "../utils/config";
+import { emitModelUsageEvent } from "../utils/events";
+import { parseImageDescriptionResponse } from "../utils/helpers";
+import { createElizaCloudClient } from "../utils/sdk-client";
+
+export async function handleImageGeneration(
+  runtime: IAgentRuntime,
+  params: ImageGenerationParams
+): Promise<{ url: string }[]> {
+  const numImages = params.count || 1;
+  const size = params.size || "1024x1024";
+  const prompt = params.prompt;
+  const modelName = getImageGenerationModel(runtime);
+  logger.log(`[ELIZAOS_CLOUD] Using IMAGE model: ${modelName}`);
+
+  const aspectRatioMap: Record<string, string> = {
+    "1024x1024": "1:1",
+    "1792x1024": "16:9",
+    "1024x1792": "9:16",
+  };
+  const aspectRatio = aspectRatioMap[size] || "1:1";
+
+  try {
+    const requestBody = {
+      prompt: prompt,
+      numImages: numImages,
+      aspectRatio: aspectRatio,
+      model: modelName,
+    };
+
+    const typedData = await createElizaCloudClient(runtime).generateImage(requestBody);
+
+    const result = typedData.images.map((img: { url?: string; image?: string }) => ({
+      url: img.url ?? img.image ?? "",
+    }));
+    return result;
+  } catch (error) {
+    const message = error instanceof Error ? error.message : String(error);
+    logger.error(`[ELIZAOS_CLOUD] Image generation error: ${message}`);
+    throw error;
+  }
+}
+
+export async function handleImageDescription(
+  runtime: IAgentRuntime,
+  params: ImageDescriptionParams | string
+): Promise<{ title: string; description: string }> {
+  // Honour `DISABLE_IMAGE_DESCRIPTION` (set by the runtime when
+  // `features.vision === false`). The runtime exposes it via getSetting; some
+  // hosts only set it in process.env. Check both before burning a quota slot.
+  // The docs (`docs/runtime/core.md`) already promise this behaviour, but
+  // historically only `plugin-discord` honoured it at the call site, leaving
+  // every other caller (agent-orchestrator's task validator, vision, lifeops,
+  // farcaster, telegram) free to spend the rate-limit budget.
+  const disableSetting = getSetting(runtime, "DISABLE_IMAGE_DESCRIPTION", "");
+  const disabled = [disableSetting, process.env.DISABLE_IMAGE_DESCRIPTION].some((value) => {
+    const normalized = value?.trim().toLowerCase();
+    return normalized === "1" || normalized === "true" || normalized === "yes" || normalized === "on";
+  });
+  if (disabled) {
+    logger.debug("[ELIZAOS_CLOUD] IMAGE_DESCRIPTION skipped — DISABLE_IMAGE_DESCRIPTION is set");
+    return {
+      title: "Image description disabled",
+      description: "Image description is disabled by configuration.",
+    };
+  }
+
+  let imageUrl: string;
+  let promptText: string | undefined;
+  const modelName = getImageDescriptionModel(runtime);
+  logger.log(`[ELIZAOS_CLOUD] Using IMAGE_DESCRIPTION model: ${modelName}`);
+  const maxTokens = Number.parseInt(
+    getSetting(runtime, "ELIZAOS_CLOUD_IMAGE_DESCRIPTION_MAX_TOKENS", "8192") || "8192",
+    10
+  );
+
+  if (typeof params === "string") {
+    imageUrl = params;
+    promptText = "Please analyze this image and provide a title and detailed description.";
+  } else {
+    imageUrl = params.imageUrl;
+    promptText =
+      params.prompt || "Please analyze this image and provide a title and detailed description.";
+  }
+
+  const messages = [
+    {
+      role: "user",
+      content: [
+        { type: "text", text: promptText },
+        { type: "image_url", image_url: { url: imageUrl } },
+      ],
+    },
+  ];
+
+  const client = createElizaCloudClient(runtime);
+
+  try {
+    const requestBody: Record<string, unknown> = {
+      model: modelName,
+      messages: messages,
+      max_tokens: maxTokens,
+    };
+
+    // On 429, honour the upstream's `retryAfter` instead of retrying on a
+    // hardcoded backoff. Hardcoded retries inside the rate-limit window add
+    // wasted requests to the same bucket and make the problem worse — see
+    // #7374's billing render-loop fix and S33's dashboard 429-aware UX.
+    // Strategy: only retry once, only if the upstream signals a short wait
+    // (≤5s, i.e. transient burst). Anything longer, bail immediately and let
+    // the caller fail fast.
+    let response: Response | null = null;
+    let attemptedRetry = false;
+    for (let attempt = 0; attempt < 2; attempt++) {
+      const attemptResponse = await client.routes.postApiV1ChatCompletionsRaw({
+        json: requestBody,
+        timeoutMs: resolveCloudTimeoutMs("ELIZAOS_CLOUD_IMAGE_TIMEOUT_MS", 120_000),
+      });
+      if (!attemptResponse) {
+        continue;
+      }
+      response = attemptResponse;
+      if (attemptResponse.status !== 429 || attemptedRetry) break;
+
+      // `Number(null) === 0`, so guard against a missing header before
+      // calling `Number(...)` — otherwise the header path always wins with a
+      // bogus `0` and the body fallback becomes unreachable.
+      const headerValue = attemptResponse.headers.get("retry-after");
+      const headerRetryAfter =
+        headerValue !== null && Number.isFinite(Number(headerValue))
+          ? Number(headerValue)
+          : undefined;
+      let bodyRetryAfter: number | undefined;
+      try {
+        const peek = (await attemptResponse.clone().json()) as {
+          retryAfter?: unknown;
+        };
+        bodyRetryAfter =
+          typeof peek?.retryAfter === "number" && Number.isFinite(peek.retryAfter)
+            ? peek.retryAfter
+            : undefined;
+      } catch {
+        // Body wasn't JSON — fall through to header value.
+      }
+      const retryAfter = headerRetryAfter ?? bodyRetryAfter ?? 0;
+
+      if (retryAfter > 0 && retryAfter <= 5) {
+        logger.warn(
+          `[ELIZAOS_CLOUD] Image analysis rate-limited (429), retrying once after ${retryAfter}s...`
+        );
+        await new Promise((r) => setTimeout(r, retryAfter * 1000));
+        attemptedRetry = true;
+        continue;
+      }
+      // Long rate-limit window: don't burn another bucket slot retrying inside it.
+      logger.warn(
+        `[ELIZAOS_CLOUD] Image analysis rate-limited (429); upstream retryAfter=${retryAfter || "unknown"}s — failing fast`
+      );
+      break;
+    }
+
+    if (!response) {
+      throw new Error("ElizaOS Cloud API did not return a response");
+    }
+
+    const finalResponse = response;
+
+    if (!finalResponse.ok) {
+      const status = finalResponse.status;
+      if (status === 402) {
+        throw new Error(
+          "Eliza Cloud credits exhausted — top up at https://www.elizacloud.ai/dashboard/settings?tab=billing"
+        );
+      }
+      if (status === 429) {
+        throw new Error(
+          "Eliza Cloud rate limit exceeded for image description — try again in a minute"
+        );
+      }
+      throw new Error(`ElizaOS Cloud API error: ${status}`);
+    }
+
+    type OpenAIResponseType = {
+      choices?: Array<{
+        message?: { content?: string };
+        finish_reason?: string;
+      }>;
+      usage?: {
+        prompt_tokens: number;
+        completion_tokens: number;
+        total_tokens: number;
+      };
+    };
+
+    const typedResult = (await finalResponse.json()) as OpenAIResponseType;
+    const content = typedResult.choices?.[0]?.message?.content;
+
+    if (typedResult.usage) {
+      emitModelUsageEvent(
+        runtime,
+        ModelType.IMAGE_DESCRIPTION,
+        typeof params === "string" ? params : params.prompt || "",
+        {
+          inputTokens: typedResult.usage.prompt_tokens,
+          outputTokens: typedResult.usage.completion_tokens,
+          totalTokens: typedResult.usage.total_tokens,
+        }
+      );
+    }
+
+    if (!content) {
+      return {
+        title: "Failed to analyze image",
+        description: "No response from API",
+      };
+    }
+
+    return parseImageDescriptionResponse(content);
+  } catch (error) {
+    const message = error instanceof Error ? error.message : String(error);
+    logger.error(`Error analyzing image: ${message}`);
+    return {
+      title: "Failed to analyze image",
+      description: `Error: ${message}`,
+    };
+  }
+}

package/src/models/index.ts

@@ -0,0 +1,16 @@
+export type { BatchEmbeddingResult } from "./embeddings";
+export { handleBatchTextEmbedding, handleTextEmbedding } from "./embeddings";
+export { handleImageDescription, handleImageGeneration } from "./image";
+export { handleResearch } from "./research";
+export { fetchTextToSpeech, handleTextToSpeech } from "./speech";
+export {
+  handleActionPlanner,
+  handleResponseHandler,
+  handleTextLarge,
+  handleTextMedium,
+  handleTextMega,
+  handleTextNano,
+  handleTextSmall,
+} from "./text";
+export { handleTokenizerDecode, handleTokenizerEncode } from "./tokenization";
+export { handleTranscription } from "./transcription";

package/src/models/research.ts

@@ -0,0 +1,275 @@
+/** Research model handler — calls /responses for deep-research models. */
+
+import type {
+  IAgentRuntime,
+  ResearchAnnotation,
+  ResearchOutputItem,
+  ResearchParams,
+  ResearchResult,
+} from "@elizaos/core";
+import { logger, ModelType } from "@elizaos/core";
+import { getResearchModel, resolveCloudTimeoutMs } from "../utils/config";
+import { emitModelUsageEvent } from "../utils/events";
+import { createCloudApiClient } from "../utils/sdk-client";
+
+// Deep research is a long-running, TURN-BLOCKING call; without a timeout a
+// stalled gateway hangs the turn forever (cloud-sdk applies no default). The
+// default is deliberately generous (10 min) so a legitimately slow run isn't
+// aborted; `ELIZAOS_CLOUD_RESEARCH_TIMEOUT_MS=0` opts out.
+const DEFAULT_RESEARCH_TIMEOUT_MS = 600_000;
+
+interface ResponsesAPIOutput {
+  id: string;
+  status: string;
+  output: Array<{
+    type: string;
+    id?: string;
+    status?: string;
+    content?: Array<{
+      type: string;
+      text?: string;
+      annotations?: Array<{
+        type: string;
+        url?: string;
+        title?: string;
+        start_index?: number;
+        end_index?: number;
+      }>;
+    }>;
+    action?: {
+      type: string;
+      query?: string;
+      url?: string;
+    };
+    query?: string;
+    results?: Array<{
+      file_id: string;
+      file_name: string;
+      score: number;
+    }>;
+    code?: string;
+    output?: string;
+    server_label?: string;
+    tool_name?: string;
+    arguments?: Record<string, unknown>;
+    result?: unknown;
+  }>;
+  usage?: {
+    input_tokens: number;
+    output_tokens: number;
+    total_tokens: number;
+  };
+}
+
+type ResponsesAPIInput =
+  | string
+  | Array<{
+      role: "user" | "system" | "assistant";
+      content: Array<{
+        type: "input_text";
+        text: string;
+      }>;
+    }>;
+
+function normalizeInput(input: ResearchParams["input"]): ResponsesAPIInput {
+  if (typeof input !== "string") {
+    return input as ResponsesAPIInput;
+  }
+
+  return [
+    {
+      role: "user",
+      content: [
+        {
+          type: "input_text",
+          text: input,
+        },
+      ],
+    },
+  ];
+}
+
+function buildResearchApiError(status: number, errorText: string): Error {
+  try {
+    const parsed = JSON.parse(errorText) as {
+      error?: { message?: string; param?: string; code?: string };
+    };
+    const message = parsed.error?.message;
+    const param = parsed.error?.param;
+    if (param === "tools.0.type" && message?.includes('expected "function"')) {
+      return new Error(
+        `Research API error: ${status} Eliza Cloud /responses rejected deep-research tool types; the provider currently only accepts function tools on this route`
+      );
+    }
+  } catch {
+    // Fall through to the raw error text.
+  }
+
+  return new Error(`Research API error: ${status} ${errorText}`);
+}
+
+function parseAnnotations(
+  raw: Array<{
+    type: string;
+    url?: string;
+    title?: string;
+    start_index?: number;
+    end_index?: number;
+  }>
+): ResearchAnnotation[] {
+  return raw
+    .filter((a) => a.url !== undefined)
+    .map((a) => ({
+      url: a.url as string,
+      title: a.title ?? "",
+      startIndex: a.start_index ?? 0,
+      endIndex: a.end_index ?? 0,
+    }));
+}
+
+function parseOutputItems(raw: ResponsesAPIOutput["output"]): ResearchOutputItem[] {
+  const items: ResearchOutputItem[] = [];
+
+  for (const item of raw) {
+    switch (item.type) {
+      case "web_search_call":
+        items.push({
+          id: item.id ?? "",
+          type: "web_search_call",
+          status: (item.status as "completed" | "failed") ?? "completed",
+          action: {
+            type: (item.action?.type as "search" | "open_page" | "find_in_page") ?? "search",
+            query: item.action?.query,
+            url: item.action?.url,
+          },
+        });
+        break;
+      case "file_search_call":
+        items.push({
+          id: item.id ?? "",
+          type: "file_search_call",
+          status: (item.status as "completed" | "failed") ?? "completed",
+          query: item.query ?? "",
+          results: item.results?.map((r) => ({
+            fileId: r.file_id,
+            fileName: r.file_name,
+            score: r.score,
+          })),
+        });
+        break;
+      case "code_interpreter_call":
+        items.push({
+          id: item.id ?? "",
+          type: "code_interpreter_call",
+          status: (item.status as "completed" | "failed") ?? "completed",
+          code: item.code ?? "",
+          output: item.output,
+        });
+        break;
+      case "mcp_tool_call":
+        items.push({
+          id: item.id ?? "",
+          type: "mcp_tool_call",
+          status: (item.status as "completed" | "failed") ?? "completed",
+          serverLabel: item.server_label ?? "",
+          toolName: item.tool_name ?? "",
+          arguments: (item.arguments ?? {}) as Record<string, import("@elizaos/core").JsonValue>,
+          result: item.result as import("@elizaos/core").JsonValue | undefined,
+        });
+        break;
+      case "message": {
+        const content = item.content ?? [];
+        items.push({
+          type: "message",
+          content: content
+            .filter((c) => c.type === "output_text")
+            .map((c) => ({
+              type: "output_text" as const,
+              text: c.text ?? "",
+              annotations: parseAnnotations(c.annotations ?? []),
+            })),
+        });
+        break;
+      }
+    }
+  }
+
+  return items;
+}
+
+export async function handleResearch(
+  runtime: IAgentRuntime,
+  params: ResearchParams
+): Promise<ResearchResult> {
+  const modelName = params.model ?? getResearchModel(runtime);
+  logger.log(`[ELIZAOS_CLOUD] Using RESEARCH model: ${modelName}`);
+
+  const tools = params.tools ?? [{ type: "web_search_preview" }];
+
+  const requestBody: Record<string, unknown> = {
+    model: modelName,
+    input: normalizeInput(params.input),
+    tools: tools,
+  };
+
+  if (params.instructions) {
+    requestBody.instructions = params.instructions;
+  }
+  if (params.background !== undefined) {
+    requestBody.background = params.background;
+  }
+  if (params.maxToolCalls !== undefined) {
+    requestBody.max_tool_calls = params.maxToolCalls;
+  }
+  if (params.reasoningSummary) {
+    requestBody.reasoning = { summary: params.reasoningSummary };
+  }
+
+  const response = await createCloudApiClient(runtime).requestRaw("POST", "/responses", {
+    json: requestBody,
+    timeoutMs: resolveCloudTimeoutMs(
+      "ELIZAOS_CLOUD_RESEARCH_TIMEOUT_MS",
+      DEFAULT_RESEARCH_TIMEOUT_MS
+    ),
+  });
+
+  if (!response.ok) {
+    const errorText = await response.text();
+    throw buildResearchApiError(response.status, errorText);
+  }
+
+  const data = (await response.json()) as ResponsesAPIOutput;
+
+  if (data.usage) {
+    emitModelUsageEvent(runtime, ModelType.RESEARCH, params.input, {
+      inputTokens: data.usage.input_tokens,
+      outputTokens: data.usage.output_tokens,
+      totalTokens: data.usage.total_tokens,
+    });
+  }
+
+  const outputItems = parseOutputItems(data.output);
+
+  // Extract final text and annotations from the last message output
+  let text = "";
+  const annotations: ResearchAnnotation[] = [];
+
+  for (const item of outputItems) {
+    if (item.type === "message") {
+      for (const content of item.content) {
+        if (content.type === "output_text") {
+          text += content.text;
+          annotations.push(...content.annotations);
+        }
+      }
+    }
+  }
+
+  return {
+    id: data.id,
+    text,
+    annotations,
+    outputItems,
+    status: data.status as ResearchResult["status"],
+  };
+}