@elizaos/plugin-elizacloud 2.0.0-alpha.7 → 2.0.0-beta.1

package/src/models/embeddings.ts

@@ -0,0 +1,234 @@
+import type { IAgentRuntime, TextEmbeddingParams } from "@elizaos/core";
+import { logger, ModelType, VECTOR_DIMS } from "@elizaos/core";
+import { getSetting } from "../utils/config";
+import { emitModelUsageEvent } from "../utils/events";
+import { createCloudApiClient } from "../utils/sdk-client";
+
+const MAX_BATCH_SIZE = 100;
+function extractRateLimitInfo(response: Response): {
+  remainingRequests?: number;
+  remainingTokens?: number;
+  limitRequests?: number;
+  limitTokens?: number;
+  resetRequests?: string;
+  resetTokens?: string;
+  retryAfter?: number;
+} {
+  return {
+    remainingRequests:
+      parseInt(response.headers.get("x-ratelimit-remaining-requests") || "", 10) || undefined,
+    remainingTokens:
+      parseInt(response.headers.get("x-ratelimit-remaining-tokens") || "", 10) || undefined,
+    limitRequests:
+      parseInt(response.headers.get("x-ratelimit-limit-requests") || "", 10) || undefined,
+    limitTokens: parseInt(response.headers.get("x-ratelimit-limit-tokens") || "", 10) || undefined,
+    resetRequests: response.headers.get("x-ratelimit-reset-requests") || undefined,
+    resetTokens: response.headers.get("x-ratelimit-reset-tokens") || undefined,
+    retryAfter: parseInt(response.headers.get("retry-after") || "", 10) || undefined,
+  };
+}
+
+function getEmbeddingConfig(runtime: IAgentRuntime) {
+  const embeddingModelName = getSetting(
+    runtime,
+    "ELIZAOS_CLOUD_EMBEDDING_MODEL",
+    "text-embedding-3-small"
+  );
+  const embeddingDimension = Number.parseInt(
+    getSetting(runtime, "ELIZAOS_CLOUD_EMBEDDING_DIMENSIONS", "1536") || "1536",
+    10
+  ) as (typeof VECTOR_DIMS)[keyof typeof VECTOR_DIMS];
+
+  if (!Object.values(VECTOR_DIMS).includes(embeddingDimension)) {
+    const errorMsg = `Invalid embedding dimension: ${embeddingDimension}. Must be one of: ${Object.values(VECTOR_DIMS).join(", ")}`;
+    logger.error(errorMsg);
+    throw new Error(errorMsg);
+  }
+
+  return { embeddingModelName, embeddingDimension };
+}
+
+function createErrorVector(dimension: number, marker: number): number[] {
+  const vector = Array(dimension).fill(0);
+  vector[0] = marker;
+  return vector;
+}
+
+export interface BatchEmbeddingParams {
+  texts: string[];
+}
+
+export async function handleTextEmbedding(
+  runtime: IAgentRuntime,
+  params: TextEmbeddingParams | string | null
+): Promise<number[]> {
+  const { embeddingDimension } = getEmbeddingConfig(runtime);
+
+  if (params === null) {
+    logger.debug("Creating test embedding for initialization");
+    return createErrorVector(embeddingDimension, 0.1);
+  }
+
+  let text: string;
+  if (typeof params === "string") {
+    text = params;
+  } else if (typeof params === "object" && params.text) {
+    text = params.text;
+  } else {
+    logger.warn("Invalid input format for embedding");
+    return createErrorVector(embeddingDimension, 0.2);
+  }
+
+  if (!text.trim()) {
+    logger.warn("Empty text for embedding");
+    return createErrorVector(embeddingDimension, 0.3);
+  }
+
+  const results = await handleBatchTextEmbedding(runtime, [text]);
+  return results[0];
+}
+
+export interface BatchEmbeddingResult {
+  embedding: number[];
+  index: number;
+  success: boolean;
+  error?: string;
+}
+
+export async function handleBatchTextEmbedding(
+  runtime: IAgentRuntime,
+  texts: string[]
+): Promise<number[][]> {
+  const { embeddingModelName, embeddingDimension } = getEmbeddingConfig(runtime);
+  const client = createCloudApiClient(runtime, true);
+
+  if (!texts || texts.length === 0) {
+    logger.warn("[BatchEmbeddings] Empty texts array");
+    return [];
+  }
+
+  const validTexts: { text: string; originalIndex: number }[] = [];
+  const results: number[][] = new Array(texts.length);
+
+  for (let i = 0; i < texts.length; i++) {
+    const text = texts[i]?.trim();
+    if (text) {
+      validTexts.push({ text, originalIndex: i });
+    } else {
+      results[i] = createErrorVector(embeddingDimension, 0.3);
+    }
+  }
+
+  if (validTexts.length === 0) {
+    logger.warn("[BatchEmbeddings] All texts were empty");
+    return results;
+  }
+
+  for (let batchStart = 0; batchStart < validTexts.length; batchStart += MAX_BATCH_SIZE) {
+    const batchEnd = Math.min(batchStart + MAX_BATCH_SIZE, validTexts.length);
+    const batch = validTexts.slice(batchStart, batchEnd);
+    const batchTexts = batch.map((b) => b.text);
+
+    logger.info(
+      `[BatchEmbeddings] Processing batch ${Math.floor(batchStart / MAX_BATCH_SIZE) + 1}/${Math.ceil(validTexts.length / MAX_BATCH_SIZE)}: ${batch.length} texts`
+    );
+
+    try {
+      const response = await client.requestRaw("POST", "/embeddings", {
+        json: {
+          model: embeddingModelName,
+          input: batchTexts,
+        },
+      });
+
+      const rateLimitInfo = extractRateLimitInfo(response);
+
+      if (rateLimitInfo.remainingRequests !== undefined && rateLimitInfo.remainingRequests < 50) {
+        logger.warn(
+          `[BatchEmbeddings] Rate limit: ${rateLimitInfo.remainingRequests}/${rateLimitInfo.limitRequests} requests remaining`
+        );
+      }
+
+      if (response.status === 429) {
+        const retryAfter = rateLimitInfo.retryAfter || 30;
+        logger.warn(`[BatchEmbeddings] Rate limited, waiting ${retryAfter}s...`);
+        await new Promise((resolve) => setTimeout(resolve, retryAfter * 1000));
+
+        const retryResponse = await client.requestRaw("POST", "/embeddings", {
+          json: {
+            model: embeddingModelName,
+            input: batchTexts,
+          },
+        });
+
+        if (!retryResponse.ok) {
+          logger.error(`[BatchEmbeddings] Retry failed: ${retryResponse.status}`);
+          for (const item of batch) {
+            results[item.originalIndex] = createErrorVector(embeddingDimension, 0.4);
+          }
+          continue;
+        }
+
+        const retryData = (await retryResponse.json()) as {
+          data: Array<{ embedding: number[]; index: number }>;
+        };
+
+        if (retryData?.data) {
+          for (const item of retryData.data) {
+            const originalIndex = batch[item.index].originalIndex;
+            results[originalIndex] = item.embedding;
+          }
+          logger.info(`[BatchEmbeddings] Retry successful for ${batch.length} embeddings`);
+        }
+        continue;
+      }
+
+      if (!response.ok) {
+        logger.error(`[BatchEmbeddings] API error: ${response.status} - ${response.statusText}`);
+        for (const item of batch) {
+          results[item.originalIndex] = createErrorVector(embeddingDimension, 0.4);
+        }
+        continue;
+      }
+
+      const data = (await response.json()) as {
+        data: Array<{ embedding: number[]; index: number }>;
+        usage?: { prompt_tokens: number; total_tokens: number };
+      };
+
+      if (!data?.data || !Array.isArray(data.data)) {
+        logger.error("[BatchEmbeddings] API returned invalid structure");
+        for (const item of batch) {
+          results[item.originalIndex] = createErrorVector(embeddingDimension, 0.5);
+        }
+        continue;
+      }
+
+      for (const item of data.data) {
+        const originalIndex = batch[item.index].originalIndex;
+        results[originalIndex] = item.embedding;
+      }
+
+      if (data.usage) {
+        const usage = {
+          inputTokens: data.usage.prompt_tokens,
+          outputTokens: 0,
+          totalTokens: data.usage.total_tokens,
+        };
+        emitModelUsageEvent(runtime, ModelType.TEXT_EMBEDDING, `batch:${batch.length}`, usage);
+      }
+
+      logger.debug(
+        `[BatchEmbeddings] Got ${batch.length} embeddings (${embeddingDimension}d), remaining: ${rateLimitInfo.remainingRequests ?? "unknown"}`
+      );
+    } catch (error) {
+      const message = error instanceof Error ? error.message : String(error);
+      logger.error(`[BatchEmbeddings] Error: ${message}`);
+      for (const item of batch) {
+        results[item.originalIndex] = createErrorVector(embeddingDimension, 0.6);
+      }
+    }
+  }
+
+  return results;
+}

package/src/models/image.ts

@@ -0,0 +1,219 @@
+import type { IAgentRuntime, ImageDescriptionParams, ImageGenerationParams } from "@elizaos/core";
+import { logger, ModelType } from "@elizaos/core";
+import { getImageDescriptionModel, getImageGenerationModel, getSetting } from "../utils/config";
+import { emitModelUsageEvent } from "../utils/events";
+import { parseImageDescriptionResponse } from "../utils/helpers";
+import { createElizaCloudClient } from "../utils/sdk-client";
+
+export async function handleImageGeneration(
+  runtime: IAgentRuntime,
+  params: ImageGenerationParams
+): Promise<{ url: string }[]> {
+  const numImages = params.count || 1;
+  const size = params.size || "1024x1024";
+  const prompt = params.prompt;
+  const modelName = getImageGenerationModel(runtime);
+  logger.log(`[ELIZAOS_CLOUD] Using IMAGE model: ${modelName}`);
+
+  const aspectRatioMap: Record<string, string> = {
+    "1024x1024": "1:1",
+    "1792x1024": "16:9",
+    "1024x1792": "9:16",
+  };
+  const aspectRatio = aspectRatioMap[size] || "1:1";
+
+  try {
+    const requestBody = {
+      prompt: prompt,
+      numImages: numImages,
+      aspectRatio: aspectRatio,
+      model: modelName,
+    };
+
+    const typedData = await createElizaCloudClient(runtime).generateImage(requestBody);
+
+    const result = typedData.images.map((img) => ({
+      url: img.url ?? img.image ?? "",
+    }));
+    return result;
+  } catch (error) {
+    const message = error instanceof Error ? error.message : String(error);
+    logger.error(`[ELIZAOS_CLOUD] Image generation error: ${message}`);
+    throw error;
+  }
+}
+
+export async function handleImageDescription(
+  runtime: IAgentRuntime,
+  params: ImageDescriptionParams | string
+): Promise<{ title: string; description: string }> {
+  // Honour `DISABLE_IMAGE_DESCRIPTION` (set by the runtime when
+  // `features.vision === false`). The runtime exposes it via getSetting; some
+  // hosts only set it in process.env. Check both before burning a quota slot.
+  // The docs (`docs/runtime/core.md`) already promise this behaviour, but
+  // historically only `plugin-discord` honoured it at the call site, leaving
+  // every other caller (agent-orchestrator's task validator, vision, lifeops,
+  // farcaster, telegram) free to spend the rate-limit budget.
+  const disableSetting = getSetting(runtime, "DISABLE_IMAGE_DESCRIPTION", "");
+  const disabled =
+    disableSetting === "true" ||
+    disableSetting === "1" ||
+    process.env.DISABLE_IMAGE_DESCRIPTION === "true" ||
+    process.env.DISABLE_IMAGE_DESCRIPTION === "1";
+  if (disabled) {
+    logger.debug("[ELIZAOS_CLOUD] IMAGE_DESCRIPTION skipped — DISABLE_IMAGE_DESCRIPTION is set");
+    return {
+      title: "Image description disabled",
+      description: "Image description is disabled by configuration.",
+    };
+  }
+
+  let imageUrl: string;
+  let promptText: string | undefined;
+  const modelName = getImageDescriptionModel(runtime);
+  logger.log(`[ELIZAOS_CLOUD] Using IMAGE_DESCRIPTION model: ${modelName}`);
+  const maxTokens = Number.parseInt(
+    getSetting(runtime, "ELIZAOS_CLOUD_IMAGE_DESCRIPTION_MAX_TOKENS", "8192") || "8192",
+    10
+  );
+
+  if (typeof params === "string") {
+    imageUrl = params;
+    promptText = "Please analyze this image and provide a title and detailed description.";
+  } else {
+    imageUrl = params.imageUrl;
+    promptText =
+      params.prompt || "Please analyze this image and provide a title and detailed description.";
+  }
+
+  const messages = [
+    {
+      role: "user",
+      content: [
+        { type: "text", text: promptText },
+        { type: "image_url", image_url: { url: imageUrl } },
+      ],
+    },
+  ];
+
+  const client = createElizaCloudClient(runtime);
+
+  try {
+    const requestBody: Record<string, unknown> = {
+      model: modelName,
+      messages: messages,
+      max_tokens: maxTokens,
+    };
+
+    // On 429, honour the upstream's `retryAfter` instead of retrying on a
+    // hardcoded backoff. Hardcoded retries inside the rate-limit window add
+    // wasted requests to the same bucket and make the problem worse — see
+    // #7374's billing render-loop fix and S33's dashboard 429-aware UX.
+    // Strategy: only retry once, only if the upstream signals a short wait
+    // (≤5s, i.e. transient burst). Anything longer, bail immediately and let
+    // the caller fail fast.
+    let response: Response | null = null;
+    let attemptedRetry = false;
+    for (let attempt = 0; attempt < 2; attempt++) {
+      response = await client.routes.postApiV1ChatCompletionsRaw({
+        json: requestBody,
+      });
+      if (response.status !== 429 || attemptedRetry) break;
+
+      // `Number(null) === 0`, so guard against a missing header before
+      // calling `Number(...)` — otherwise the header path always wins with a
+      // bogus `0` and the body fallback becomes unreachable.
+      const headerValue = response.headers.get("retry-after");
+      const headerRetryAfter =
+        headerValue !== null && Number.isFinite(Number(headerValue))
+          ? Number(headerValue)
+          : undefined;
+      let bodyRetryAfter: number | undefined;
+      try {
+        const peek = (await response.clone().json()) as {
+          retryAfter?: unknown;
+        };
+        bodyRetryAfter =
+          typeof peek?.retryAfter === "number" && Number.isFinite(peek.retryAfter)
+            ? peek.retryAfter
+            : undefined;
+      } catch {
+        // Body wasn't JSON — fall through to header value.
+      }
+      const retryAfter = headerRetryAfter ?? bodyRetryAfter ?? 0;
+
+      if (retryAfter > 0 && retryAfter <= 5) {
+        logger.warn(
+          `[ELIZAOS_CLOUD] Image analysis rate-limited (429), retrying once after ${retryAfter}s...`
+        );
+        await new Promise((r) => setTimeout(r, retryAfter * 1000));
+        attemptedRetry = true;
+        continue;
+      }
+      // Long rate-limit window: don't burn another bucket slot retrying inside it.
+      logger.warn(
+        `[ELIZAOS_CLOUD] Image analysis rate-limited (429); upstream retryAfter=${retryAfter || "unknown"}s — failing fast`
+      );
+      break;
+    }
+
+    if (!response?.ok) {
+      const status = response?.status ?? 0;
+      if (status === 402) {
+        throw new Error(
+          "Eliza Cloud credits exhausted — top up at https://www.elizacloud.ai/dashboard/settings?tab=billing"
+        );
+      }
+      if (status === 429) {
+        throw new Error(
+          "Eliza Cloud rate limit exceeded for image description — try again in a minute"
+        );
+      }
+      throw new Error(`ElizaOS Cloud API error: ${status}`);
+    }
+
+    type OpenAIResponseType = {
+      choices?: Array<{
+        message?: { content?: string };
+        finish_reason?: string;
+      }>;
+      usage?: {
+        prompt_tokens: number;
+        completion_tokens: number;
+        total_tokens: number;
+      };
+    };
+
+    const typedResult = (await response.json()) as OpenAIResponseType;
+    const content = typedResult.choices?.[0]?.message?.content;
+
+    if (typedResult.usage) {
+      emitModelUsageEvent(
+        runtime,
+        ModelType.IMAGE_DESCRIPTION,
+        typeof params === "string" ? params : params.prompt || "",
+        {
+          inputTokens: typedResult.usage.prompt_tokens,
+          outputTokens: typedResult.usage.completion_tokens,
+          totalTokens: typedResult.usage.total_tokens,
+        }
+      );
+    }
+
+    if (!content) {
+      return {
+        title: "Failed to analyze image",
+        description: "No response from API",
+      };
+    }
+
+    return parseImageDescriptionResponse(content);
+  } catch (error) {
+    const message = error instanceof Error ? error.message : String(error);
+    logger.error(`Error analyzing image: ${message}`);
+    return {
+      title: "Failed to analyze image",
+      description: `Error: ${message}`,
+    };
+  }
+}

package/src/models/index.ts

@@ -0,0 +1,16 @@
+export type { BatchEmbeddingResult } from "./embeddings";
+export { handleBatchTextEmbedding, handleTextEmbedding } from "./embeddings";
+export { handleImageDescription, handleImageGeneration } from "./image";
+export { handleResearch } from "./research";
+export { fetchTextToSpeech, handleTextToSpeech } from "./speech";
+export {
+  handleActionPlanner,
+  handleResponseHandler,
+  handleTextLarge,
+  handleTextMedium,
+  handleTextMega,
+  handleTextNano,
+  handleTextSmall,
+} from "./text";
+export { handleTokenizerDecode, handleTokenizerEncode } from "./tokenization";
+export { handleTranscription } from "./transcription";