@fondation-io/ai 7.0.0-beta.45

package/docs/09-troubleshooting/70-high-memory-usage-with-images.mdx

@@ -0,0 +1,108 @@
+---
+title: High memory usage when processing many images
+description: Troubleshooting high memory usage when using generateText or streamText with many images
+---
+
+# High memory usage when processing many images
+
+## Issue
+
+When using `generateText` or `streamText` with many images (e.g., in a loop or batch processing), you may notice:
+
+- Memory usage grows continuously and doesn't decrease
+- Application eventually runs out of memory
+- Memory is not reclaimed even after garbage collection
+
+This is especially noticeable when using `experimental_download` to process images from URLs, or when sending base64-encoded images in prompts.
+
+## Background
+
+By default, the AI SDK includes the full request and response bodies in the step results. When processing images, the request body contains the base64-encoded image data, which can be very large (a single image can be 1MB+ when base64 encoded). If you process many images and keep references to the results, this data accumulates in memory.
+
+For example, processing 100 images of 500KB each would include ~50MB+ of request body data in memory.
+
+## Solution
+
+Use the `experimental_include` option to disable inclusion of request and/or response bodies:
+
+```ts
+import { generateText } from 'ai';
+import { openai } from '@ai-sdk/openai';
+
+const result = await generateText({
+  model: openai('gpt-4o'),
+  messages: [
+    {
+      role: 'user',
+      content: [
+        { type: 'text', text: 'Describe this image' },
+        { type: 'image', image: imageUrl },
+      ],
+    },
+  ],
+  // Disable inclusion of request body to reduce memory usage
+  experimental_include: {
+    requestBody: false,
+    responseBody: false,
+  },
+});
+```
+
+### Options
+
+The `experimental_include` option accepts:
+
+- `requestBody`: Set to `false` to exclude the request body from step results. This is where base64-encoded images are stored. Default: `true`. Available in both `generateText` and `streamText`.
+- `responseBody`: Set to `false` to exclude the response body from step results. Default: `true`. Only available in `generateText`.
+
+### When to use
+
+- **Batch processing images**: When processing many images in a loop
+- **Long-running agents**: When an agent may process many images over its lifetime
+- **Memory-constrained environments**: When running in environments with limited memory
+
+### Trade-offs
+
+When you disable body inclusion:
+
+- You won't have access to `result.request.body` or `result.response.body`
+- Debugging may be harder since you can't inspect the raw request/response
+- If you need the bodies for logging or debugging, consider extracting the data you need before the next iteration
+
+## Example: Processing multiple images
+
+```ts
+import { generateText } from 'ai';
+import { openai } from '@ai-sdk/openai';
+
+const imageUrls = [
+  /* array of image URLs */
+];
+const results = [];
+
+for (const imageUrl of imageUrls) {
+  const result = await generateText({
+    model: openai('gpt-4o'),
+    messages: [
+      {
+        role: 'user',
+        content: [
+          { type: 'text', text: 'Describe this image' },
+          { type: 'image', image: imageUrl },
+        ],
+      },
+    ],
+    experimental_include: {
+      requestBody: false,
+    },
+  });
+
+  // Only store the text result, not the full result object
+  results.push(result.text);
+}
+```
+
+## Learn more
+
+- [`generateText` API Reference](/docs/reference/ai-sdk-core/generate-text)
+- [`streamText` API Reference](/docs/reference/ai-sdk-core/stream-text)

package/docs/09-troubleshooting/index.mdx

@@ -0,0 +1,11 @@
+---
+title: Troubleshooting
+description: Troubleshooting information for common issues encountered with the AI SDK.
+collapsed: true
+---
+
+# Troubleshooting
+
+This section is designed to help you quickly identify and resolve common issues encountered with the AI SDK, ensuring a smoother and more efficient development experience.
+
+<Support />

package/internal.d.ts

@@ -0,0 +1 @@
+export * from './dist/internal';

package/package.json

@@ -0,0 +1,120 @@
+{
+  "name": "@fondation-io/ai",
+  "version": "7.0.0-beta.45",
+  "description": "AI SDK by Vercel - The AI Toolkit for TypeScript and JavaScript",
+  "license": "Apache-2.0",
+  "sideEffects": false,
+  "main": "./dist/index.js",
+  "module": "./dist/index.mjs",
+  "types": "./dist/index.d.ts",
+  "source": "./src/index.ts",
+  "files": [
+    "dist/**/*",
+    "docs/**/*",
+    "src",
+    "!src/**/*.test.ts",
+    "!src/**/*.test-d.ts",
+    "!src/**/__snapshots__",
+    "CHANGELOG.md",
+    "internal.d.ts",
+    "README.md",
+    "test.d.ts"
+  ],
+  "directories": {
+    "doc": "./docs"
+  },
+  "scripts": {
+    "build": "pnpm clean && tsup --tsconfig tsconfig.build.json",
+    "build:watch": "pnpm clean && tsup --watch --tsconfig tsconfig.build.json",
+    "clean": "del-cli dist docs *.tsbuildinfo",
+    "prepack": "cp -r ../../content/docs ./docs",
+    "postpack": "del-cli docs",
+    "type-check": "tsc --build",
+    "test": "pnpm test:node && pnpm test:edge",
+    "test:update": "pnpm test:node -u",
+    "test:watch": "vitest --config vitest.node.config.js",
+    "test:edge": "vitest --config vitest.edge.config.js --run",
+    "test:node": "vitest --config vitest.node.config.js --run",
+    "check-bundle-size": "tsx scripts/check-bundle-size.ts"
+  },
+  "exports": {
+    "./package.json": "./package.json",
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.mjs",
+      "require": "./dist/index.js"
+    },
+    "./internal": {
+      "types": "./dist/internal/index.d.ts",
+      "import": "./dist/internal/index.mjs",
+      "module": "./dist/internal/index.mjs",
+      "require": "./dist/internal/index.js"
+    },
+    "./test": {
+      "types": "./dist/test/index.d.ts",
+      "import": "./dist/test/index.mjs",
+      "module": "./dist/test/index.mjs",
+      "require": "./dist/test/index.js"
+    }
+  },
+  "dependencies": {
+    "@ai-sdk/gateway": "4.0.0-beta.25",
+    "@ai-sdk/provider": "4.0.0-beta.5",
+    "@ai-sdk/provider-utils": "5.0.0-beta.7",
+    "@opentelemetry/api": "1.9.0"
+  },
+  "devDependencies": {
+    "@ai-sdk/test-server": "workspace:*",
+    "@edge-runtime/vm": "^5.0.0",
+    "@types/json-schema": "7.0.15",
+    "@types/node": "20.17.24",
+    "@vercel/ai-tsconfig": "workspace:*",
+    "esbuild": "^0.24.2",
+    "tsup": "^7.2.0",
+    "tsx": "^4.19.2",
+    "typescript": "5.8.3",
+    "zod": "3.25.76"
+  },
+  "peerDependencies": {
+    "zod": "^3.25.76 || ^4.1.8"
+  },
+  "engines": {
+    "node": ">=18"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "homepage": "https://ai-sdk.dev/docs",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/vercel/ai.git"
+  },
+  "bugs": {
+    "url": "https://github.com/vercel/ai/issues"
+  },
+  "keywords": [
+    "ai",
+    "vercel",
+    "sdk",
+    "llm",
+    "mcp",
+    "tool-calling",
+    "tools",
+    "structured-output",
+    "agent",
+    "agentic",
+    "generative",
+    "genai",
+    "chatbot",
+    "prompt",
+    "inference",
+    "language-model",
+    "streaming",
+    "openai",
+    "anthropic",
+    "claude",
+    "gemini",
+    "xai",
+    "grok"
+  ]
+}

package/src/agent/agent.ts

@@ -0,0 +1,156 @@
+import { ModelMessage } from '@ai-sdk/provider-utils';
+import { GenerateTextResult } from '../generate-text/generate-text-result';
+import { Output } from '../generate-text/output';
+import { StreamTextTransform } from '../generate-text/stream-text';
+import { StreamTextResult } from '../generate-text/stream-text-result';
+import { ToolSet } from '../generate-text/tool-set';
+import { TimeoutConfiguration } from '../prompt/call-settings';
+import type {
+  ToolLoopAgentOnFinishCallback,
+  ToolLoopAgentOnStartCallback,
+  ToolLoopAgentOnStepFinishCallback,
+  ToolLoopAgentOnStepStartCallback,
+  ToolLoopAgentOnToolCallFinishCallback,
+  ToolLoopAgentOnToolCallStartCallback,
+} from './tool-loop-agent-settings';
+
+/**
+ * Parameters for calling an agent.
+ */
+export type AgentCallParameters<CALL_OPTIONS, TOOLS extends ToolSet = {}> = ([
+  CALL_OPTIONS,
+] extends [never]
+  ? { options?: never }
+  : { options: CALL_OPTIONS }) &
+  (
+    | {
+        /**
+         * A prompt. It can be either a text prompt or a list of messages.
+         *
+         * You can either use `prompt` or `messages` but not both.
+         */
+        prompt: string | Array<ModelMessage>;
+
+        /**
+         * A list of messages.
+         *
+         * You can either use `prompt` or `messages` but not both.
+         */
+        messages?: never;
+      }
+    | {
+        /**
+         * A list of messages.
+         *
+         * You can either use `prompt` or `messages` but not both.
+         */
+        messages: Array<ModelMessage>;
+
+        /**
+         * A prompt. It can be either a text prompt or a list of messages.
+         *
+         * You can either use `prompt` or `messages` but not both.
+         */
+        prompt?: never;
+      }
+  ) & {
+    /**
+     * Abort signal.
+     */
+    abortSignal?: AbortSignal;
+
+    /**
+     * Timeout in milliseconds. Can be specified as a number or as an object with `totalMs`.
+     */
+    timeout?: TimeoutConfiguration<TOOLS>;
+
+    /**
+     * Callback that is called when the agent operation begins, before any LLM calls.
+     */
+    experimental_onStart?: ToolLoopAgentOnStartCallback<TOOLS>;
+
+    /**
+     * Callback that is called when a step (LLM call) begins, before the provider is called.
+     */
+    experimental_onStepStart?: ToolLoopAgentOnStepStartCallback<TOOLS>;
+
+    /**
+     * Callback that is called before each tool execution begins.
+     */
+    experimental_onToolCallStart?: ToolLoopAgentOnToolCallStartCallback<TOOLS>;
+
+    /**
+     * Callback that is called after each tool execution completes.
+     */
+    experimental_onToolCallFinish?: ToolLoopAgentOnToolCallFinishCallback<TOOLS>;
+
+    /**
+     * Callback that is called when each step (LLM call) is finished, including intermediate steps.
+     */
+    onStepFinish?: ToolLoopAgentOnStepFinishCallback<TOOLS>;
+
+    /**
+     * Callback that is called when all steps are finished and the response is complete.
+     */
+    onFinish?: ToolLoopAgentOnFinishCallback<TOOLS>;
+  };
+
+/**
+ * Parameters for streaming an output from an agent.
+ */
+export type AgentStreamParameters<
+  CALL_OPTIONS,
+  TOOLS extends ToolSet,
+> = AgentCallParameters<CALL_OPTIONS, TOOLS> & {
+  /**
+   * Optional stream transformations.
+   * They are applied in the order they are provided.
+   * The stream transformations must maintain the stream structure for streamText to work correctly.
+   */
+  experimental_transform?:
+    | StreamTextTransform<TOOLS>
+    | Array<StreamTextTransform<TOOLS>>;
+};
+
+/**
+ * An Agent receives a prompt (text or messages) and generates or streams an output
+ * that consists of steps, tool calls, data parts, etc.
+ *
+ * You can implement your own Agent by implementing the `Agent` interface,
+ * or use the `ToolLoopAgent` class.
+ */
+export interface Agent<
+  CALL_OPTIONS = never,
+  TOOLS extends ToolSet = {},
+  OUTPUT extends Output = never,
+> {
+  /**
+   * The specification version of the agent interface. This will enable
+   * us to evolve the agent interface and retain backwards compatibility.
+   */
+  readonly version: 'agent-v1';
+
+  /**
+   * The id of the agent.
+   */
+  readonly id: string | undefined;
+
+  /**
+   * The tools that the agent can use.
+   */
+  readonly tools: TOOLS;
+
+  /**
+   * Generates an output from the agent (non-streaming).
+   */
+  generate(
+    options: AgentCallParameters<CALL_OPTIONS, TOOLS>,
+  ): PromiseLike<GenerateTextResult<TOOLS, OUTPUT>>;
+
+  /**
+   * Streams an output from the agent (streaming).
+   */
+  stream(
+    options: AgentStreamParameters<CALL_OPTIONS, TOOLS>,
+  ): PromiseLike<StreamTextResult<TOOLS, OUTPUT>>;
+}

package/src/agent/create-agent-ui-stream-response.ts

@@ -0,0 +1,61 @@
+import { StreamTextTransform, UIMessageStreamOptions } from '../generate-text';
+import { Output } from '../generate-text/output';
+import { ToolSet } from '../generate-text/tool-set';
+import { TimeoutConfiguration } from '../prompt/call-settings';
+import { createUIMessageStreamResponse } from '../ui-message-stream';
+import { UIMessageStreamResponseInit } from '../ui-message-stream/ui-message-stream-response-init';
+import { InferUITools, UIMessage } from '../ui/ui-messages';
+import { Agent } from './agent';
+import { createAgentUIStream } from './create-agent-ui-stream';
+import type { ToolLoopAgentOnStepFinishCallback } from './tool-loop-agent-settings';
+
+/**
+ * Runs the agent and returns a response object with a UI message stream.
+ *
+ * @param agent - The agent to run.
+ * @param uiMessages - The input UI messages.
+ * @param abortSignal - Abort signal. Optional.
+ * @param timeout - Timeout in milliseconds. Optional.
+ * @param options - The options for the agent. Optional.
+ * @param experimental_transform - Stream transformations. Optional.
+ * @param onStepFinish - Callback that is called when each step is finished. Optional.
+ * @param headers - Additional headers for the response. Optional.
+ * @param status - The status code for the response. Optional.
+ * @param statusText - The status text for the response. Optional.
+ * @param consumeSseStream - Whether to consume the SSE stream. Optional.
+ *
+ * @returns The response object.
+ */
+export async function createAgentUIStreamResponse<
+  CALL_OPTIONS = never,
+  TOOLS extends ToolSet = {},
+  OUTPUT extends Output = never,
+  MESSAGE_METADATA = unknown,
+>({
+  headers,
+  status,
+  statusText,
+  consumeSseStream,
+  ...options
+}: {
+  agent: Agent<CALL_OPTIONS, TOOLS, OUTPUT>;
+  uiMessages: unknown[];
+  abortSignal?: AbortSignal;
+  timeout?: TimeoutConfiguration<TOOLS>;
+  options?: CALL_OPTIONS;
+  experimental_transform?:
+    | StreamTextTransform<TOOLS>
+    | Array<StreamTextTransform<TOOLS>>;
+  onStepFinish?: ToolLoopAgentOnStepFinishCallback<TOOLS>;
+} & UIMessageStreamResponseInit &
+  UIMessageStreamOptions<
+    UIMessage<MESSAGE_METADATA, never, InferUITools<TOOLS>>
+  >): Promise<Response> {
+  return createUIMessageStreamResponse({
+    headers,
+    status,
+    statusText,
+    consumeSseStream,
+    stream: await createAgentUIStream(options),
+  });
+}

package/src/agent/create-agent-ui-stream.ts

@@ -0,0 +1,84 @@
+import { StreamTextTransform, UIMessageStreamOptions } from '../generate-text';
+import { Output } from '../generate-text/output';
+import { ToolSet } from '../generate-text/tool-set';
+import { TimeoutConfiguration } from '../prompt/call-settings';
+import { InferUIMessageChunk } from '../ui-message-stream';
+import { convertToModelMessages } from '../ui/convert-to-model-messages';
+import { InferUITools, UIMessage } from '../ui/ui-messages';
+import { validateUIMessages } from '../ui/validate-ui-messages';
+import { AsyncIterableStream } from '../util/async-iterable-stream';
+import { Agent } from './agent';
+import type { ToolLoopAgentOnStepFinishCallback } from './tool-loop-agent-settings';
+
+/**
+ * Runs the agent and stream the output as a UI message stream.
+ *
+ * @param agent - The agent to run.
+ * @param uiMessages - The input UI messages.
+ * @param abortSignal - The abort signal. Optional.
+ * @param timeout - Timeout in milliseconds. Optional.
+ * @param options - The options for the agent.
+ * @param experimental_transform - The stream transformations. Optional.
+ * @param onStepFinish - Callback that is called when each step is finished. Optional.
+ *
+ * @returns The UI message stream.
+ */
+export async function createAgentUIStream<
+  CALL_OPTIONS = never,
+  TOOLS extends ToolSet = {},
+  OUTPUT extends Output = never,
+  MESSAGE_METADATA = unknown,
+>({
+  agent,
+  uiMessages,
+  options,
+  abortSignal,
+  timeout,
+  experimental_transform,
+  onStepFinish,
+  ...uiMessageStreamOptions
+}: {
+  agent: Agent<CALL_OPTIONS, TOOLS, OUTPUT>;
+  uiMessages: unknown[];
+  abortSignal?: AbortSignal;
+  timeout?: TimeoutConfiguration<TOOLS>;
+  options?: CALL_OPTIONS;
+  experimental_transform?:
+    | StreamTextTransform<TOOLS>
+    | Array<StreamTextTransform<TOOLS>>;
+  onStepFinish?: ToolLoopAgentOnStepFinishCallback<TOOLS>;
+  // TODO `originalMessages` is part of this for bc, omit in v7
+} & UIMessageStreamOptions<
+  UIMessage<MESSAGE_METADATA, never, InferUITools<TOOLS>>
+>): Promise<
+  AsyncIterableStream<
+    InferUIMessageChunk<UIMessage<MESSAGE_METADATA, never, InferUITools<TOOLS>>>
+  >
+> {
+  const validatedMessages = await validateUIMessages<
+    UIMessage<MESSAGE_METADATA, never, InferUITools<TOOLS>>
+  >({
+    messages: uiMessages,
+    tools: agent.tools,
+  });
+
+  const modelMessages = await convertToModelMessages(validatedMessages, {
+    tools: agent.tools,
+  });
+
+  const result = await agent.stream({
+    prompt: modelMessages,
+    options: options as CALL_OPTIONS,
+    abortSignal,
+    timeout,
+    experimental_transform,
+    onStepFinish,
+  });
+
+  return result.toUIMessageStream({
+    ...uiMessageStreamOptions,
+    // TODO reading `originalMessages` is here for bc, always use `validatedMessages` in v7
+    originalMessages:
+      uiMessageStreamOptions.originalMessages ?? validatedMessages,
+  });
+}

package/src/agent/index.ts

@@ -0,0 +1,37 @@
+export {
+  type Agent,
+  type AgentCallParameters,
+  type AgentStreamParameters,
+} from './agent';
+export {
+  type ToolLoopAgentOnFinishCallback,
+  type ToolLoopAgentOnStartCallback,
+  type ToolLoopAgentOnStepFinishCallback,
+  type ToolLoopAgentOnStepStartCallback,
+  type ToolLoopAgentOnToolCallFinishCallback,
+  type ToolLoopAgentOnToolCallStartCallback,
+  type ToolLoopAgentSettings,
+
+  /**
+   * @deprecated Use `ToolLoopAgentSettings` instead.
+   */
+  type ToolLoopAgentSettings as Experimental_AgentSettings,
+} from './tool-loop-agent-settings';
+export {
+  ToolLoopAgent,
+
+  /**
+   * @deprecated Use `ToolLoopAgent` instead.
+   */
+  ToolLoopAgent as Experimental_Agent,
+} from './tool-loop-agent';
+export {
+  /**
+   * @deprecated Use `InferAgentUIMessage` instead.
+   */
+  type InferAgentUIMessage as Experimental_InferAgentUIMessage,
+  type InferAgentUIMessage,
+} from './infer-agent-ui-message';
+export { createAgentUIStreamResponse } from './create-agent-ui-stream-response';
+export { createAgentUIStream } from './create-agent-ui-stream';
+export { pipeAgentUIStreamToResponse } from './pipe-agent-ui-stream-to-response';

package/src/agent/infer-agent-tools.ts

@@ -0,0 +1,7 @@
+import { Agent } from './agent';
+
+/**
+ * Infer the type of the tools of an agent.
+ */
+export type InferAgentTools<AGENT> =
+  AGENT extends Agent<any, infer TOOLS, any> ? TOOLS : never;

package/src/agent/infer-agent-ui-message.ts

@@ -0,0 +1,11 @@
+import { InferUITools, UIMessage } from '../ui/ui-messages';
+import { InferAgentTools } from './infer-agent-tools';
+
+/**
+ * Infer the UI message type of an agent.
+ */
+export type InferAgentUIMessage<AGENT, MESSAGE_METADATA = unknown> = UIMessage<
+  MESSAGE_METADATA,
+  never,
+  InferUITools<InferAgentTools<AGENT>>
+>;

package/src/agent/pipe-agent-ui-stream-to-response.ts

@@ -0,0 +1,64 @@
+import { ServerResponse } from 'node:http';
+import { StreamTextTransform, UIMessageStreamOptions } from '../generate-text';
+import { Output } from '../generate-text/output';
+import { ToolSet } from '../generate-text/tool-set';
+import { TimeoutConfiguration } from '../prompt/call-settings';
+import { pipeUIMessageStreamToResponse } from '../ui-message-stream';
+import { UIMessageStreamResponseInit } from '../ui-message-stream/ui-message-stream-response-init';
+import { InferUITools, UIMessage } from '../ui/ui-messages';
+import { Agent } from './agent';
+import { createAgentUIStream } from './create-agent-ui-stream';
+import type { ToolLoopAgentOnStepFinishCallback } from './tool-loop-agent-settings';
+
+/**
+ * Pipes the agent UI message stream to a Node.js ServerResponse object.
+ *
+ * @param response - The Node.js ServerResponse object to pipe to.
+ * @param agent - The agent to run.
+ * @param uiMessages - The input UI messages.
+ * @param abortSignal - Abort signal. Optional.
+ * @param timeout - Timeout in milliseconds. Optional.
+ * @param options - The options for the agent. Optional.
+ * @param experimental_transform - Stream transformations. Optional.
+ * @param onStepFinish - Callback that is called when each step is finished. Optional.
+ * @param headers - Additional headers for the response. Optional.
+ * @param status - The status code for the response. Optional.
+ * @param statusText - The status text for the response. Optional.
+ * @param consumeSseStream - Whether to consume the SSE stream. Optional.
+ */
+export async function pipeAgentUIStreamToResponse<
+  CALL_OPTIONS = never,
+  TOOLS extends ToolSet = {},
+  OUTPUT extends Output = never,
+  MESSAGE_METADATA = unknown,
+>({
+  response,
+  headers,
+  status,
+  statusText,
+  consumeSseStream,
+  ...options
+}: {
+  response: ServerResponse;
+  agent: Agent<CALL_OPTIONS, TOOLS, OUTPUT>;
+  uiMessages: unknown[];
+  abortSignal?: AbortSignal;
+  timeout?: TimeoutConfiguration<TOOLS>;
+  options?: CALL_OPTIONS;
+  experimental_transform?:
+    | StreamTextTransform<TOOLS>
+    | Array<StreamTextTransform<TOOLS>>;
+  onStepFinish?: ToolLoopAgentOnStepFinishCallback<TOOLS>;
+} & UIMessageStreamResponseInit &
+  UIMessageStreamOptions<
+    UIMessage<MESSAGE_METADATA, never, InferUITools<TOOLS>>
+  >): Promise<void> {
+  pipeUIMessageStreamToResponse({
+    response,
+    headers,
+    status,
+    statusText,
+    consumeSseStream,
+    stream: await createAgentUIStream(options),
+  });
+}