agent-accelerator 0.6.0 → 0.6.5

package/src/tools/architecture.md

@@ -2,12 +2,13 @@
 
 Defining tools and running them.
 
-| File                 | What it is                                                                                                                                                     |
-| :------------------- | :------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| `tool.ts`            | `tool()` factory, name validation, `toStandardToolDeclarations`, option normalizers.                                                                           |
-| `schema.ts`          | Zod to JSON Schema, `$ref` resolution, `stripSchemaForGoogle`.                                                                                                 |
-| `executor.ts`        | `executeToolCalls`: validation, concurrency (8 global + per-tool), attempts, timeout, result cap, error scrubbing, `ctx.stop`, `tool.*` events, per-call logs. |
-| `subagent-result.ts` | The private-Symbol brand that marks a result as made by the SDK's delegation tools.                                                                            |
+| File                 | What it is                                                                                                                                                                                               |
+| :------------------- | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `tool.ts`            | `tool()` factory, name validation, `toStandardToolDeclarations`, option normalizers.                                                                                                                     |
+| `schema.ts`          | Zod to JSON Schema, `$ref` resolution, `stripSchemaForGoogle`.                                                                                                                                           |
+| `executor.ts`        | `executeToolCalls`: validation, concurrency (8 global + per-tool), attempts, timeout, result cap, error scrubbing, `ctx.stop`, `tool.*` events, per-call logs.                                           |
+| `tool-content.ts`    | `toolContent([...])`: a branded (private Symbol) tool result of text and media. The executor turns it into `result` (text) plus `parts` (inline data URLs), so only a tool's own code can produce media. |
+| `subagent-result.ts` | The private-Symbol brand that marks a result as made by the SDK's delegation tools.                                                                                                                      |
 
 ## Invariants
 

package/src/tools/executor.ts

@@ -3,6 +3,10 @@ import { toJsonSafe } from "../utils/serialization.ts";
 import { ConfigError } from "../types/errors.ts";
 import { isTransientError } from "../utils/errors.ts";
 import { isSubagentResult } from "./subagent-result.ts";
+import { isToolContent, type ToolContent } from "./tool-content.ts";
+import { normalizeMediaInput } from "../utils/media.ts";
+import type { ContentPart } from "../types/message.ts";
+import { DEFAULT_PROMPTS, fillPrompt, type Prompts } from "../prompts/prompts.ts";
 import { isTruncatedArguments } from "../types/tool.ts";
 import { telemetryBus } from "../telemetry.ts";
 import type { ToolLogWriter } from "../session/tool-log.ts";
@@ -14,6 +18,8 @@ export interface ExecuteToolsOptions {
   parallel?: boolean;
   signal?: AbortSignal;
   sessionId?: string;
+  /** The refusal texts shown to the model (`AgentConfig.prompts`); the shipped ones when unset. */
+  prompts?: Pick<Prompts, "toolTruncatedArguments" | "toolNotFound" | "toolSchemaError">;
   /** Called the moment EACH call finishes (before the whole batch does), so a caller can save a result before the next tool is done. Never throws into the run. */
   onResult?: (result: ToolResultRecord) => void;
   /** Opens the log file of one call (only when the Agent persists to a folder); the executor writes its steps into it and `ctx.log` writes through it. */
@@ -232,10 +238,11 @@ export function shouldBlockRepeat(
  * machine-readable code lives at `(result as any).errorCode` while the human
  * prefix keeps plain-string consumers working.
  */
-export function makeRepeatBlockedResult(call: ToolCallRecord): ToolResultRecord {
-  const message =
-    `repeat_blocked: Tool '${call.name}' was called again with identical arguments. ` +
-    "The previous result is already available; reuse it or call the tool with different arguments.";
+export function makeRepeatBlockedResult(
+  call: ToolCallRecord,
+  template = DEFAULT_PROMPTS.toolRepeatBlocked
+): ToolResultRecord {
+  const message = fillPrompt(template, { tool: call.name });
   return {
     id: call.id,
     name: call.name,
@@ -477,6 +484,36 @@ async function executeAttempt(
   }
 }
 
+// Turns a `toolContent` result into its text (what every provider sees) and its media as inline data URLs (what Google also sends); in: runSingle for a branded tool result, out: the result record's `result` and `parts`.
+async function resolveToolContent(
+  content: ToolContent,
+  charLimit: number
+): Promise<{ text: string; parts: ContentPart[] }> {
+  const texts: string[] = [];
+  const parts: ContentPart[] = [];
+  const kinds: string[] = [];
+  for (const raw of content.parts) {
+    const part = typeof raw === "string" ? ({ type: "text", text: raw } as const) : raw;
+    if (part.type === "text") {
+      texts.push(part.text);
+      continue;
+    }
+    const value = (part as unknown as Record<string, string | Uint8Array | ArrayBuffer>)[part.type]!;
+    const media = await normalizeMediaInput(value, (part as { mimeType?: string }).mimeType);
+    const filename = (part as { filename?: string }).filename;
+    parts.push({
+      type: part.type,
+      [part.type]: media.dataUrl,
+      mimeType: media.mimeType,
+      ...(filename ? { filename } : {}),
+      ...((part.type === "image" || part.type === "video") && part.resolution ? { resolution: part.resolution } : {}),
+    } as unknown as ContentPart);
+    kinds.push(media.mimeType);
+  }
+  const text = texts.join("\n") || (kinds.length > 0 ? `[media result: ${kinds.join(", ")}]` : "null");
+  return { text: truncateResultString(text, charLimit), parts };
+}
+
 /**
  * Executes one or more tool calls in parallel or sequence, handling errors safely
  */
@@ -496,6 +533,7 @@ export async function executeToolCalls(options: ExecuteToolsOptions): Promise<To
     onSubagentEvent,
     onResult,
     openToolLog,
+    prompts = DEFAULT_PROMPTS,
   } = options;
   assertNoToolCollisions(tools);
   const globalSemaphore = new Semaphore(DEFAULT_TOOL_CONCURRENCY);
@@ -534,9 +572,7 @@ export async function executeToolCalls(options: ExecuteToolsOptions): Promise<To
       return {
         id: call.id,
         name: call.name,
-        result:
-          `Error: Tool '${call.name}' arguments are truncated or incomplete and could not be parsed. ` +
-          "Call the tool again with the complete JSON arguments. Do not explain the error, just retry the call.",
+        result: fillPrompt(prompts.toolTruncatedArguments, { tool: call.name }),
         isError: true,
         durationMs: elapsedMs(startTime),
       };
@@ -547,11 +583,11 @@ export async function executeToolCalls(options: ExecuteToolsOptions): Promise<To
 
     if (!toolDef) {
       trace("call_blocked", "warn", { reason: "tool_not_found", suggestion: resolved.suggestion });
-      const suffix = resolved.suggestion ? ` Did you mean '${resolved.suggestion}'?` : "";
+      const suggestion = resolved.suggestion ? ` Did you mean '${resolved.suggestion}'?` : "";
       return {
         id: call.id,
         name: call.name,
-        result: `Error: Tool '${call.name}' not found.${suffix}`,
+        result: fillPrompt(prompts.toolNotFound, { tool: call.name, suggestion }),
         isError: true,
         durationMs: elapsedMs(startTime),
       };
@@ -614,16 +650,12 @@ export async function executeToolCalls(options: ExecuteToolsOptions): Promise<To
               ? JSON.stringify(toolDef.parameters).slice(0, 800)
               : undefined;
           const detail = parseRes.error.message;
-          const hint = expectedParams ? ` Expected parameters: ${expectedParams}.` : "";
+          const expected = expectedParams ? ` Expected parameters: ${expectedParams}.` : "";
           trace("call_blocked", "warn", { reason: "schema_validation", detail: parseRes.error.message });
           return {
             id: call.id,
             name: call.name,
-            result:
-              `Schema validation error for tool '${call.name}': ${detail}` +
-              ` Received argument keys: [${receivedKeys}].` +
-              hint +
-              ` Fix the arguments and call the tool again with corrected JSON. Do not explain the error, just retry the call.`,
+            result: fillPrompt(prompts.toolSchemaError, { tool: call.name, detail, keys: receivedKeys, expected }),
             isError: true,
             durationMs: elapsedMs(startTime),
           };
@@ -692,6 +724,19 @@ export async function executeToolCalls(options: ExecuteToolsOptions): Promise<To
       log?.write({ phase: "result", result: isSubagentResult(rawResult) ? rawResult.xml : toJsonSafe(rawResult) });
       const finishOk = (resultChars: number): void =>
         trace("call_succeeded", "info", { attempts: attempt, durationMs: elapsedMs(executionStartTime), resultChars });
+      // Text and media from `toolContent`: the text is the result, the media rides in `parts` (inline data, set only here).
+      if (isToolContent(rawResult)) {
+        const { text, parts } = await resolveToolContent(rawResult, resultCharLimit(toolDef));
+        finishOk(text.length);
+        return {
+          id: call.id,
+          name: resolved.name,
+          result: text,
+          ...(parts.length > 0 ? { parts } : {}),
+          isError: false,
+          durationMs: elapsedMs(executionStartTime),
+        };
+      }
       if (rawResult === undefined) {
         finishOk(4);
         return {

package/src/tools/tool-content.ts

@@ -0,0 +1,32 @@
+/**
+ * A tool result made of text and media. A tool returns `toolContent([...])` when the model should SEE a file (a chart
+ * image, a scanned page, a recording) and not only read text about it. The brand is a private Symbol, so only the
+ * tool's own code can produce it: JSON a tool fetched from the web can never become media for the provider.
+ */
+import type { ContentPart } from "../types/message.ts";
+
+/** Parts a tool may return: plain strings (text) and the usual text, image, audio, video and file parts. */
+export type ToolContentPart = string | Extract<ContentPart, { type: "text" | "image" | "audio" | "video" | "file" }>;
+
+const TOOL_CONTENT_BRAND: unique symbol = Symbol("agent-accelerator.tool-content") as never;
+
+/** What `toolContent` returns; the executor turns it into a tool result with `parts`. */
+export interface ToolContent {
+  readonly parts: ToolContentPart[];
+  readonly [TOOL_CONTENT_BRAND]: true;
+}
+
+/**
+ * Wraps text and media for a tool's return value, e.g.
+ * `return toolContent(["instrument.jpg", { type: "image", image: bytes, mimeType: "image/jpeg" }])`.
+ * Media may be bytes, a data URL, a path or a URL (the tool's own code is trusted with these). Google passes the media
+ * to the model inside the function result; other providers get the text only, with one warning.
+ */
+export function toolContent(parts: ToolContentPart[]): ToolContent {
+  return { parts: [...parts], [TOOL_CONTENT_BRAND]: true };
+}
+
+/** True only for values made by {@link toolContent}. */
+export function isToolContent(value: unknown): value is ToolContent {
+  return !!value && typeof value === "object" && (value as Record<symbol, unknown>)[TOOL_CONTENT_BRAND] === true;
+}

package/src/types/agent.ts

@@ -4,6 +4,7 @@ import type { ThinkingLevel, CacheOption, ServiceTier } from "./core.ts";
 import type { ModelProviderInstance } from "../providers/registry.ts";
 import type { Agent } from "../agent/agent.ts";
 import type { z } from "zod";
+import type { PromptOverrides } from "../prompts/prompts.ts";
 
 // ---------------------------------------------------------------------------
 // Canonical option types (all additive; nothing renamed).
@@ -43,7 +44,12 @@ export interface AgentLogger {
 }
 
 /** Canonical tool-selection control. String form is the portable subset. */
-export type ToolChoiceOption = "auto" | "none" | "required" | { mode?: "auto" | "none" | "required"; tool?: string };
+export type ToolChoiceOption =
+  | "auto"
+  | "none"
+  | "required"
+  | "validated"
+  | { mode?: "auto" | "none" | "required" | "validated"; tool?: string };
 
 /** Canonical structured-output input. Zod, JSON Schema, or explicit form. */
 export type StructuredOutputOption =
@@ -149,7 +155,10 @@ export interface AgentConfig {
    * Canonical tool-selection control (`"auto"` default). `"none"` applies to
    * every turn of a run; `"required"` and a pinned tool (`{ tool: "name" }`)
    * apply to the FIRST model turn only, since forcing a tool call on every
-   * turn would make a run unable to finish.
+   * turn would make a run unable to finish. `"validated"` (Google only) lets the
+   * model decide whether to call a tool but makes it keep every call valid against
+   * the tool's schema; it holds for the whole run. Other providers ignore it
+   * with one warning.
    */
   toolChoice?: ToolChoiceOption;
   /** Google Interactions storage tier for the chain-TTL gate: `"free"` (23h, default) or `"paid"` (54d). */
@@ -215,12 +224,29 @@ export interface AgentConfig {
   budget?: Budget;
   /** Run + network limits. Turn/budget caps unlimited by default; network timeouts finite. */
   limits?: AgentRunLimits;
+  /**
+   * A spending cap for the WHOLE chat, for its whole life: what the agent, all its workers (the `subagents` tool,
+   * `agent.workers` at any depth, fixed `subagents`) and their model calls cost, added up. It survives restarts: a
+   * persisted agent starts from what its saved sessions already spent. Once the total reaches `maxCostUsd` or
+   * `maxTotalTokens`, the next model call anywhere in the chat throws a `BudgetExceededError` ("Usage limit reached").
+   * Unlike `budget` (per run) it never resets. Scheduled actions are separate and never count. Models with unknown
+   * pricing add tokens but $0 to the cost. Unset = no cap.
+   */
+  usageLimit?: Pick<Budget, "maxCostUsd" | "maxTotalTokens">;
   /**
    * The only models this agent may use, as `provider/model` strings compared as written. Unset (default) = any model.
    * Checked when the Agent is built (also for a config an agent edited through {@link Agent.fromConfig}); a model
    * outside the list throws a `ConfigError`. Workers and `subagentModel` are not affected.
    */
   allowedModels?: string[];
+  /**
+   * Replaces single prompts the SDK puts in front of the model (the reasoning directive, tool refusals, the worker
+   * headers, the tool manuals ...). Every prompt you leave out keeps its default; see `DEFAULT_PROMPTS` for the names, the
+   * shipped text and the `{{placeholders}}` the SDK fills in. Keeping the original text is recommended: it is what the SDK
+   * is tested with, and a good reference for writing your own. Workers inherit it. A prompt is part of the cached prompt
+   * prefix when it is in the system prompt or a tool description, so change it once, not per run.
+   */
+  prompts?: PromptOverrides;
 }
 
 /**
@@ -301,12 +327,20 @@ export interface DynamicSubagentsConfig {
   /** Fixed reasoning level for all dynamic workers. The Main Agent cannot override it. Falls back to the parent level when omitted. */
   thinkingLevel?: ThinkingLevel;
   /**
-   * Spending cap of EACH worker (the `subagents` tool, `agent.workers` and scheduled actions), set as that worker's own
-   * per-run `budget`: a spawn, a message or a scheduled run is one run, and the usage of the worker's own workers counts
-   * toward it. Exceeding it fails that call with a budget error and leaves the worker open. Default: unlimited. Models
-   * with unknown pricing count $0 toward `maxCostUsd`. Passed down to the workers' own workers.
+   * Spending cap of EACH chat worker (the `subagents` tool and `agent.workers`), set as that worker's own per-run
+   * `budget`: a spawn or a message is one run, and the usage of the worker's own workers counts toward it. Exceeding it
+   * fails that call with a budget error and leaves the worker open. Default: unlimited. Models with unknown pricing count
+   * $0 toward `maxCostUsd`. Passed down to the workers' own workers. Scheduled actions have their own cap:
+   * `scheduledActionBudget`.
    */
   workerBudget?: Pick<Budget, "maxCostUsd" | "maxTotalTokens">;
+  /**
+   * Spending cap of EACH scheduled-action run, set as that run's own per-run `budget` (attempts included: a retry is part
+   * of the same run). Scheduled actions are detached from the chat, so neither `workerBudget` nor `usageLimit` applies to
+   * them. Exceeding it records a failed run. Default: unlimited. Models with unknown pricing count $0 toward
+   * `maxCostUsd`.
+   */
+  scheduledActionBudget?: Pick<Budget, "maxCostUsd" | "maxTotalTokens">;
   /**
    * How many levels of workers may exist below this agent. Default: no limit. `1` = workers cannot start workers of their
    * own; `3` = a worker may start workers that start workers. A worker with levels left gets its own `subagents` tool, and

package/src/types/core.ts

@@ -42,6 +42,10 @@ export interface TokenUsage {
   cacheReadTokens?: number;
   cacheWriteTokens?: number;
   thinkingTokens?: number;
+  /** Media part of `inputTokens` (a subset, report only: pricing never reads it). OpenRouter reports audio and video. */
+  inputDetails?: { audioTokens?: number; videoTokens?: number };
+  /** Media part of `outputTokens` (a subset, report only). OpenRouter reports audio and image. */
+  outputDetails?: { audioTokens?: number; imageTokens?: number };
   cost?: {
     inputCost?: number;
     outputCost?: number;

package/src/types/message.ts

@@ -12,6 +12,9 @@ export interface TextPart {
 /** Detail level a provider spends on one image or video (more detail = more input tokens). Google (images and videos) and OpenRouter (images, as `image_url.detail`) honour it; other providers ignore it with a warning. */
 export type MediaResolution = "low" | "medium" | "high" | "ultra_high";
 
+/** How a provider reads a video: `"static"` (fixed-rate frames, the default) or `"agentic"` (the model navigates the timeline and loads only what the prompt needs). Only Google honours it, and agentic mode only on some Gemini models; other providers ignore it with a warning. */
+export type VideoProcessing = "static" | "agentic";
+
 /** Image content accepted as a URL, data, path, or binary value. */
 export interface ImagePart {
   type: "image";
@@ -55,6 +58,8 @@ export interface VideoPart {
   mimeType?: string;
   /** Detail level per frame (Google: `low`/`medium` 70, `high` 280 tokens). */
   resolution?: MediaResolution;
+  /** How the video is read (Google: `"agentic"` lets the model explore a long video instead of loading every frame). */
+  processing?: VideoProcessing;
 }
 
 /** Assistant-generated structured tool invocation. */
@@ -89,6 +94,8 @@ export interface ToolResultPart {
   logFile?: string;
   /** The tool ended the run with `ctx.stop(reason)` (see `ToolExecutionContext.stop`). */
   stopped?: boolean;
+  /** Media the tool returned with `toolContent` (inline data URLs; `result` holds its text). Google sends it to the model. */
+  parts?: ContentPart[];
 }
 
 /** Provider reasoning/thinking content. */

package/src/types/model.ts

@@ -2,6 +2,7 @@ import type { ThinkingConfig, CacheOption, ServiceTier, TokenUsage } from "./cor
 import type { ProviderContext, ProviderSteps } from "./message.ts";
 import type { StandardToolDeclaration, ToolCallRecord } from "./tool.ts";
 import type { AssistantMessageEventStream } from "../streaming/event-stream.ts";
+import type { Prompts } from "../prompts/prompts.ts";
 
 // ---------------------------------------------------------------------------
 // Provider identity — battle-tested: matches models.dev provider keys
@@ -169,7 +170,7 @@ export interface ProviderRequestOptions {
   cache?: CacheOption;
   serviceTier?: ServiceTier;
   tools?: StandardToolDeclaration[];
-  toolChoice?: "auto" | "none" | "required" | { type: "function"; function: { name: string } };
+  toolChoice?: "auto" | "none" | "required" | "validated" | { type: "function"; function: { name: string } };
   signal?: AbortSignal;
   sessionId?: string;
   env?: Record<string, string>;
@@ -187,6 +188,8 @@ export interface ProviderRequestOptions {
   maxOutputTokens?: number;
   /** `fetch` implementation used for the provider request (defaults to global `fetch`). */
   fetch?: typeof fetch;
+  /** The SDK prompts adapters put in a request (`AgentConfig.prompts`); defaults when unset. */
+  prompts?: Pick<Prompts, "structuredOutput" | "toolBridge">;
   /** Base backoff delay in ms for provider retries (default 250). */
   retryBaseDelayMs?: number;
   /** Total time budget in ms for provider retries (default 30000). */

package/src/types/scheduler.ts

@@ -18,12 +18,12 @@ export interface ScheduleSpec {
   /**
    * What happens when the host program was not running at a delivery time. `"skip"`: move on to the next time (a one-shot
    * is done). `"run"`: run it as soon as the program is back. Omitted: the action is marked `missed` and waits for
-   * `runScheduledAction` or `skipScheduledAction`.
+   * `agent.scheduled.run` or `agent.scheduled.skip`.
    */
   onMissed?: "skip" | "run";
 }
 
-/** What `createScheduledAction` receives: dynamic sub-agent fields plus the schedule. */
+/** What `agent.scheduled.create` receives: dynamic sub-agent fields plus the schedule. */
 export interface ScheduledActionInput {
   role?: string;
   instructions: string;
@@ -40,7 +40,7 @@ export interface ScheduledActionInput {
   };
 }
 
-/** What `updateScheduledAction` may change. `id` and the creating agent are immutable. */
+/** What `agent.scheduled.update` may change. `id` and the creating agent are immutable. */
 export type ScheduledActionPatch = Partial<Omit<ScheduledActionInput, "scheduled_action">> & {
   scheduled_action?: Partial<Omit<ScheduledActionInput["scheduled_action"], "id">>;
 };
@@ -81,7 +81,7 @@ export interface ScheduledDelivery {
   late: boolean;
 }
 
-/** Result of `deleteScheduledAction`. */
+/** Result of `agent.scheduled.delete`. */
 export type DeleteResult =
   | { ok: true }
   | { ok: false; reason: "not_found" }

package/src/types/tool.ts

@@ -153,6 +153,8 @@ export interface ToolResultRecord {
   logFile?: string;
   /** The tool ended the run with `ctx.stop(reason)`; `result` is the reason. */
   stopped?: boolean;
+  /** Media returned with `toolContent`, as inline data URLs (`result` is its text). Set only by the executor. */
+  parts?: import("./message.ts").ContentPart[];
 }
 
 /**

package/src/utils/errors.ts

@@ -181,9 +181,9 @@ export function assertModalitiesSupported(context: ProviderContext, providerId:
  * That wire has no video shape — only `input_text`/`input_image`/`input_file`
  * exist in either vendor's Responses docs — so a video part sent as
  * `input_file` lands in the router's document parser and 400s confusingly
- * (`Failed to parse the file ... Provide a PDF document`, observed live on
- * `openrouter/stealth/space-bunny-alpha` under the old Responses skin despite
- * its catalog `video` flag, which is chat-transport oriented). Fail fast with
+ * (`Failed to parse the file ... Provide a PDF document`, observed live under
+ * the old OpenRouter Responses skin despite the catalog `video` flag, which is
+ * chat-transport oriented). Fail fast with
  * a clear one-liner instead; video stays Gemini-only, and OpenRouter Chat
  * Completions carries `video_url` natively (no guard there). Throws a `ValidationError`.
  */

package/src/utils/headers.ts

@@ -24,7 +24,7 @@ export function getPackageVersion(): string {
     const v = (globalThis as any).process?.env?.npm_package_version;
     if (typeof v === "string" && v) return v;
   } catch {}
-  return "0.6.0";
+  return "0.6.5";
 }
 
 // User-Agent string with version and platform; in: AGENT_ACCEL_USER_AGENT, out: a string.