convex-durable-agents 0.2.3 → 0.2.5

package/dist/component/tool_calls.d.ts

@@ -1,5 +1,6 @@
 import type { UIMessagePart } from "ai";
 import type { Doc, Id } from "./_generated/dataModel.js";
+export declare function shouldContinueAfterToolCompletion(status: Doc<"threads">["status"]): boolean;
 export type ToolCallDoc = {
     _id: Id<"tool_calls">;
     _creationTime: number;
@@ -10,12 +11,30 @@ export type ToolCallDoc = {
     args: unknown;
     result?: unknown;
     error?: string;
+    status: "pending" | "completed" | "failed";
+    callbackAttempt?: number;
+    callbackLastError?: string;
+    handler?: string;
+    executionAttempt?: number;
+    executionMaxAttempts?: number;
+    executionLastError?: string;
+    executionRetryPolicy?: unknown;
+    nextRetryAt?: number;
 };
 export declare const vToolCallDoc: import("convex/values").VObject<{
     error?: string | undefined;
+    nextRetryAt?: number | undefined;
     result?: any;
+    handler?: string | undefined;
+    callbackAttempt?: number | undefined;
+    callbackLastError?: string | undefined;
+    executionAttempt?: number | undefined;
+    executionMaxAttempts?: number | undefined;
+    executionLastError?: string | undefined;
+    executionRetryPolicy?: any;
     threadId: import("convex/values").GenericId<"threads">;
     msgId: string;
+    status: "completed" | "failed" | "pending";
     toolCallId: string;
     args: any;
     toolName: string;
@@ -31,9 +50,20 @@ export declare const vToolCallDoc: import("convex/values").VObject<{
     args: import("convex/values").VAny<any, "required", string>;
     result: import("convex/values").VAny<any, "optional", string>;
     error: import("convex/values").VString<string | undefined, "optional">;
-}, "required", "threadId" | "msgId" | "error" | "toolCallId" | "result" | "args" | "toolName" | "_creationTime" | `result.${string}` | `args.${string}` | "_id">;
+    status: import("convex/values").VUnion<"completed" | "failed" | "pending", [import("convex/values").VLiteral<"pending", "required">, import("convex/values").VLiteral<"completed", "required">, import("convex/values").VLiteral<"failed", "required">], "required", never>;
+    callbackAttempt: import("convex/values").VFloat64<number | undefined, "optional">;
+    callbackLastError: import("convex/values").VString<string | undefined, "optional">;
+    handler: import("convex/values").VString<string | undefined, "optional">;
+    executionAttempt: import("convex/values").VFloat64<number | undefined, "optional">;
+    executionMaxAttempts: import("convex/values").VFloat64<number | undefined, "optional">;
+    executionLastError: import("convex/values").VString<string | undefined, "optional">;
+    executionRetryPolicy: import("convex/values").VAny<any, "optional", string>;
+    nextRetryAt: import("convex/values").VFloat64<number | undefined, "optional">;
+}, "required", "threadId" | "msgId" | "status" | "error" | "nextRetryAt" | "toolCallId" | "result" | "args" | "handler" | "toolName" | "_creationTime" | "callbackAttempt" | "callbackLastError" | "executionAttempt" | "executionMaxAttempts" | "executionLastError" | "executionRetryPolicy" | `result.${string}` | `args.${string}` | `executionRetryPolicy.${string}` | "_id">;
 export declare const create: import("convex/server").RegisteredMutation<"public", {
     callback?: string | undefined;
+    handler?: string | undefined;
+    retry?: any;
     threadId: import("convex/values").GenericId<"threads">;
     msgId: string;
     toolCallId: string;
@@ -68,13 +98,14 @@ export declare const list: import("convex/server").RegisteredQuery<"public", {
  * Schedule a tool call for execution
  */
 export declare const scheduleToolCall: import("convex/server").RegisteredMutation<"public", {
+    retry?: any;
     threadId: import("convex/values").GenericId<"threads">;
     msgId: string;
     toolCallId: string;
     args: any;
+    handler: string;
     saveDelta: boolean;
     toolName: string;
-    handler: string;
 }, Promise<null>>;
 /**
  * Schedule an async tool call - creates the record and notifies the callback,
@@ -89,6 +120,27 @@ export declare const scheduleAsyncToolCall: import("convex/server").RegisteredMu
     saveDelta: boolean;
     toolName: string;
 }, Promise<null>>;
+export declare const updateExecutionRetryState: import("convex/server").RegisteredMutation<"internal", {
+    nextRetryAt?: number | undefined;
+    executionLastError?: string | undefined;
+    executionRetryFnId?: import("convex/values").GenericId<"_scheduled_functions"> | undefined;
+    clearNextRetryAt?: boolean | undefined;
+    clearExecutionRetryFnId?: boolean | undefined;
+    threadId: import("convex/values").GenericId<"threads">;
+    toolCallId: string;
+    executionAttempt: number;
+}, Promise<null>>;
+export declare const scheduleExecutionRetry: import("convex/server").RegisteredMutation<"internal", {
+    threadId: import("convex/values").GenericId<"threads">;
+    nextRetryAt: number;
+    toolCallId: string;
+    handler: string;
+    executionAttempt: number;
+    executionLastError: string;
+}, Promise<null>>;
+export declare const resumePendingSyncToolExecutions: import("convex/server").RegisteredMutation<"public", {
+    limit?: number | undefined;
+}, Promise<number>>;
 /**
  * Execute a tool call
  */
@@ -111,9 +163,9 @@ export declare const executeAsyncToolCallback: import("convex/server").Registere
 }, Promise<null>>;
 export declare const onAsyncCallbackFailure: import("convex/server").RegisteredMutation<"internal", {
     threadId: import("convex/values").GenericId<"threads">;
+    attempt: number;
     error: string;
     toolCallId: string;
-    attempt: number;
 }, Promise<null>>;
 export declare const failPendingToolCall: import("convex/server").RegisteredMutation<"internal", {
     threadId: import("convex/values").GenericId<"threads">;

package/dist/component/tool_calls.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"tool_calls.d.ts","sourceRoot":"","sources":["../../src/component/tool_calls.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,IAAI,CAAC;AAKxC,OAAO,KAAK,EAAE,GAAG,EAAE,EAAE,EAAE,MAAM,2BAA2B,CAAC;AAmFzD,MAAM,MAAM,WAAW,GAAG;IACxB,GAAG,EAAE,EAAE,CAAC,YAAY,CAAC,CAAC;IACtB,aAAa,EAAE,MAAM,CAAC;IACtB,QAAQ,EAAE,EAAE,CAAC,SAAS,CAAC,CAAC;IACxB,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,OAAO,CAAC;IACd,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB,CAAC;AAiBF,eAAO,MAAM,YAAY;;;;;;;;;;;;;;;;;;;;gKAUvB,CAAC;AAEH,eAAO,MAAM,MAAM;;;;;;;;wBA6CjB,CAAC;AAEH,eAAO,MAAM,SAAS;;;oBAoBpB,CAAC;AAEH,eAAO,MAAM,QAAQ;;;oBAoBnB,CAAC;AAEH,eAAO,MAAM,kBAAkB;;;;iBA6C7B,CAAC;AAEH,eAAO,MAAM,WAAW;;0BAYtB,CAAC;AAEH,eAAO,MAAM,eAAe;;;+BAgB1B,CAAC;AAEH,eAAO,MAAM,IAAI;;0BAYf,CAAC;AAEH;;GAEG;AACH,eAAO,MAAM,gBAAgB;;;;;;;;iBA2C3B,CAAC;AAEH;;;GAGG;AACH,eAAO,MAAM,qBAAqB;;;;;;;;iBAiDhC,CAAC;AAEH;;GAEG;AACH,eAAO,MAAM,eAAe;;;;iBAgD1B,CAAC;AAEH;;;GAGG;AACH,eAAO,MAAM,wBAAwB;;;;;;;iBAqCnC,CAAC;AAEH,eAAO,MAAM,sBAAsB;;;;;iBAqDjC,CAAC;AAEH,eAAO,MAAM,mBAAmB;;;iBA6B9B,CAAC;AAEH;;GAEG;AACH,eAAO,MAAM,cAAc;;iBAqEzB,CAAC;AAEH,KAAK,iBAAiB,GAAG,GAAG,CAAC,YAAY,CAAC,CAAC;AAE3C,wBAAgB,qBAAqB,CACnC,QAAQ,EAAE,iBAAiB,EAC3B,WAAW,CAAC,EAAE;IAAE,UAAU,EAAE,MAAM,CAAC;IAAC,oBAAoB,CAAC,EAAE,OAAO,CAAA;CAAE,GACnE,aAAa,CAAC,GAAG,EAAE,GAAG,CAAC,GAAG,IAAI,CAyBhC;AAED;;;GAGG;AACH,eAAO,MAAM,aAAa;;;;iBAuDxB,CAAC;AAEH;;;GAGG;AACH,eAAO,MAAM,YAAY;;;;iBA2DvB,CAAC"}
+{"version":3,"file":"tool_calls.d.ts","sourceRoot":"","sources":["../../src/component/tool_calls.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAoB,aAAa,EAAE,MAAM,IAAI,CAAC;AAM1D,OAAO,KAAK,EAAE,GAAG,EAAE,EAAE,EAAE,MAAM,2BAA2B,CAAC;AAiGzD,wBAAgB,iCAAiC,CAAC,MAAM,EAAE,GAAG,CAAC,SAAS,CAAC,CAAC,QAAQ,CAAC,GAAG,OAAO,CAE3F;AAmED,MAAM,MAAM,WAAW,GAAG;IACxB,GAAG,EAAE,EAAE,CAAC,YAAY,CAAC,CAAC;IACtB,aAAa,EAAE,MAAM,CAAC;IACtB,QAAQ,EAAE,EAAE,CAAC,SAAS,CAAC,CAAC;IACxB,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,OAAO,CAAC;IACd,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,SAAS,GAAG,WAAW,GAAG,QAAQ,CAAC;IAC3C,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,oBAAoB,CAAC,EAAE,OAAO,CAAC;IAC/B,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB,CAAC;AA0BF,eAAO,MAAM,YAAY;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kXAmBvB,CAAC;AAwDH,eAAO,MAAM,MAAM;;;;;;;;;;wBA2BjB,CAAC;AAEH,eAAO,MAAM,SAAS;;;oBA0BpB,CAAC;AAEH,eAAO,MAAM,QAAQ;;;oBA2BnB,CAAC;AAEH,eAAO,MAAM,kBAAkB;;;;iBA6C7B,CAAC;AAEH,eAAO,MAAM,WAAW;;0BAYtB,CAAC;AAEH,eAAO,MAAM,eAAe;;;+BAgB1B,CAAC;AAEH,eAAO,MAAM,IAAI;;0BAYf,CAAC;AAEH;;GAEG;AACH,eAAO,MAAM,gBAAgB;;;;;;;;;iBA8C3B,CAAC;AAEH;;;GAGG;AACH,eAAO,MAAM,qBAAqB;;;;;;;;iBAiDhC,CAAC;AAEH,eAAO,MAAM,yBAAyB;;;;;;;;;iBA+CpC,CAAC;AAEH,eAAO,MAAM,sBAAsB;;;;;;;iBAmCjC,CAAC;AAEH,eAAO,MAAM,+BAA+B;;mBA6C1C,CAAC;AAEH;;GAEG;AACH,eAAO,MAAM,eAAe;;;;iBAoI1B,CAAC;AAEH;;;GAGG;AACH,eAAO,MAAM,wBAAwB;;;;;;;iBAqCnC,CAAC;AAEH,eAAO,MAAM,sBAAsB;;;;;iBAqDjC,CAAC;AAEH,eAAO,MAAM,mBAAmB;;;iBA6B9B,CAAC;AAEH;;GAEG;AACH,eAAO,MAAM,cAAc;;iBA2EzB,CAAC;AAEH,KAAK,iBAAiB,GAAG,GAAG,CAAC,YAAY,CAAC,CAAC;AAE3C,wBAAgB,qBAAqB,CACnC,QAAQ,EAAE,iBAAiB,EAC3B,WAAW,CAAC,EAAE;IAAE,UAAU,EAAE,MAAM,CAAC;IAAC,oBAAoB,CAAC,EAAE,OAAO,CAAA;CAAE,GACnE,aAAa,CAAC,GAAG,EAAE,GAAG,CAAC,GAAG,IAAI,CAyBhC;AAED;;;GAGG;AACH,eAAO,MAAM,aAAa;;;;iBAuDxB,CAAC;AAEH;;;GAGG;AACH,eAAO,MAAM,YAAY;;;;iBA2DvB,CAAC"}

package/dist/component/tool_calls.js

@@ -1,5 +1,6 @@
 import { v } from "convex/values";
-import { Logger } from "../logger.js";
+import { Logger } from "../utils/logger.js";
+import { extractToolErrorInfo, isRetryableDecision, isRetryableToolErrorDefault } from "../utils/retry.js";
 import { api, internal } from "./_generated/api.js";
 import { internalAction, internalMutation, mutation, query } from "./_generated/server.js";
 import { enqueueAction } from "./agent.js";
@@ -10,6 +11,51 @@ const MINUTE = 60 * SECOND;
 const TOOL_CALL_TIMEOUT_MS = 30 * MINUTE;
 const ASYNC_CALLBACK_MAX_ATTEMPTS = 3;
 const ASYNC_CALLBACK_RETRY_BASE_MS = 5 * SECOND;
+const SYNC_TOOL_MAX_ATTEMPTS = 3;
+const SYNC_TOOL_RETRY_INITIAL_BACKOFF_MS = 500;
+function normalizeSyncToolRetryPolicy(value) {
+    if (value == null || typeof value !== "object") {
+        return undefined;
+    }
+    const obj = value;
+    if (obj.enabled !== true) {
+        return undefined;
+    }
+    return {
+        enabled: true,
+        maxAttempts: typeof obj.maxAttempts === "number" ? obj.maxAttempts : undefined,
+        backoff: obj.backoff ?? undefined,
+        shouldRetryError: typeof obj.shouldRetryError === "string" ? obj.shouldRetryError : undefined,
+    };
+}
+function clampDelayMs(value) {
+    if (!Number.isFinite(value) || value < 0)
+        return 0;
+    return Math.floor(value);
+}
+function computeRetryDelayMs(attempt, backoff) {
+    const policy = backoff ?? {
+        strategy: "exponential",
+        initialDelayMs: SYNC_TOOL_RETRY_INITIAL_BACKOFF_MS,
+        multiplier: 2,
+        maxDelayMs: 10_000,
+        jitter: true,
+    };
+    if ("delayMs" in policy) {
+        const delayMs = clampDelayMs(policy.delayMs);
+        if (!policy.jitter)
+            return delayMs;
+        return Math.floor(Math.random() * (delayMs + 1));
+    }
+    const initialDelayMs = clampDelayMs(policy.initialDelayMs);
+    const multiplier = Number.isFinite(policy.multiplier ?? 2) ? (policy.multiplier ?? 2) : 2;
+    const unbounded = initialDelayMs * multiplier ** Math.max(0, attempt - 1);
+    const maxDelayMs = policy.maxDelayMs == null ? unbounded : clampDelayMs(policy.maxDelayMs);
+    const delayMs = Math.min(unbounded, maxDelayMs);
+    if (!policy.jitter)
+        return delayMs;
+    return Math.floor(Math.random() * (delayMs + 1));
+}
 function normalizeToolCallTimeoutMs(timeoutMs) {
     if (!Number.isFinite(timeoutMs) || timeoutMs <= 0) {
         throw new Error(`Invalid tool call timeout: ${timeoutMs}`);
@@ -27,6 +73,9 @@ function formatTimeoutMs(timeoutMs) {
     }
     return `${timeoutMs}ms`;
 }
+export function shouldContinueAfterToolCompletion(status) {
+    return status === "streaming" || status === "awaiting_tool_results";
+}
 async function cleanupTimeoutFn(ctx, toolCall) {
     if (!toolCall.timeoutFnId) {
         return;
@@ -36,16 +85,28 @@ async function cleanupTimeoutFn(ctx, toolCall) {
         await ctx.scheduler.cancel(toolCall.timeoutFnId);
     }
 }
+async function cleanupExecutionRetryFn(ctx, toolCall) {
+    if (!toolCall.executionRetryFnId) {
+        return;
+    }
+    const retryFn = await ctx.db.system.get(toolCall.executionRetryFnId);
+    if (retryFn?.state.kind === "pending") {
+        await ctx.scheduler.cancel(toolCall.executionRetryFnId);
+    }
+}
 async function failToolCallIfPending(ctx, toolCall, error) {
     const latest = await ctx.db.get(toolCall._id);
     if (!latest || latest.status !== "pending") {
         return false;
     }
     await cleanupTimeoutFn(ctx, latest);
+    await cleanupExecutionRetryFn(ctx, latest);
     await ctx.db.patch(latest._id, {
         error,
         status: "failed",
         callbackLastError: error,
+        executionRetryFnId: undefined,
+        nextRetryAt: undefined,
     });
     if (latest.saveDelta) {
         logger.debug(`failPendingToolCall: inserting tool outcome delta for callId=${latest.toolCallId}`);
@@ -81,6 +142,15 @@ function publicToolCall(toolCall) {
         args: toolCall.args,
         result: toolCall.result,
         error: toolCall.error,
+        status: toolCall.status,
+        callbackAttempt: toolCall.callbackAttempt,
+        callbackLastError: toolCall.callbackLastError,
+        handler: toolCall.handler,
+        executionAttempt: toolCall.executionAttempt,
+        executionMaxAttempts: toolCall.executionMaxAttempts,
+        executionLastError: toolCall.executionLastError,
+        executionRetryPolicy: toolCall.executionRetryPolicy,
+        nextRetryAt: toolCall.nextRetryAt,
     };
 }
 // Tool call doc validator for return types
@@ -94,7 +164,54 @@ export const vToolCallDoc = v.object({
     args: v.any(),
     result: v.optional(v.any()),
     error: v.optional(v.string()),
+    status: v.union(v.literal("pending"), v.literal("completed"), v.literal("failed")),
+    callbackAttempt: v.optional(v.number()),
+    callbackLastError: v.optional(v.string()),
+    handler: v.optional(v.string()),
+    executionAttempt: v.optional(v.number()),
+    executionMaxAttempts: v.optional(v.number()),
+    executionLastError: v.optional(v.string()),
+    executionRetryPolicy: v.optional(v.any()),
+    nextRetryAt: v.optional(v.number()),
 });
+async function createToolCallRecord(ctx, args) {
+    const existingToolCall = await ctx.db
+        .query("tool_calls")
+        .withIndex("by_thread_tool_call_id", (q) => q.eq("threadId", args.threadId).eq("toolCallId", args.toolCallId))
+        .first();
+    if (existingToolCall) {
+        throw new Error(`Tool call ${args.toolCallId} already exists`);
+    }
+    logger.debug(`create: tool=${args.toolName}, callId=${args.toolCallId}, thread=${args.threadId}, msgId=${args.msgId}`);
+    const expiresAt = Date.now() + TOOL_CALL_TIMEOUT_MS;
+    const toolCallId = await ctx.db.insert("tool_calls", {
+        threadId: args.threadId,
+        msgId: args.msgId,
+        toolCallId: args.toolCallId,
+        toolName: args.toolName,
+        callback: args.callback,
+        handler: args.handler,
+        callbackAttempt: args.callback ? 0 : undefined,
+        executionAttempt: args.retry ? 0 : undefined,
+        executionMaxAttempts: args.retry?.maxAttempts,
+        executionRetryPolicy: args.retry,
+        args: args.args,
+        saveDelta: args.saveDelta,
+        timeoutMs: TOOL_CALL_TIMEOUT_MS,
+        expiresAt,
+        status: "pending",
+    });
+    const timeoutFnId = await ctx.scheduler.runAfter(TOOL_CALL_TIMEOUT_MS, internal.tool_calls.failPendingToolCall, {
+        threadId: args.threadId,
+        toolCallId: args.toolCallId,
+    });
+    await ctx.db.patch(toolCallId, { timeoutFnId });
+    const toolCall = await ctx.db.get(toolCallId);
+    if (!toolCall) {
+        throw new Error(`Tool call ${toolCallId} not found after creation`);
+    }
+    return toolCall;
+}
 export const create = mutation({
     args: {
         threadId: v.id("threads"),
@@ -102,39 +219,24 @@ export const create = mutation({
         toolCallId: v.string(),
         toolName: v.string(),
         callback: v.optional(v.string()),
+        handler: v.optional(v.string()),
+        retry: v.optional(v.any()),
         args: v.any(),
         saveDelta: v.boolean(),
     },
     returns: vToolCallDoc,
     handler: async (ctx, args) => {
-        const existingToolCall = await ctx.db
-            .query("tool_calls")
-            .withIndex("by_thread_tool_call_id", (q) => q.eq("threadId", args.threadId).eq("toolCallId", args.toolCallId))
-            .first();
-        if (existingToolCall) {
-            throw new Error(`Tool call ${args.toolCallId} already exists`);
-        }
-        logger.debug(`create: tool=${args.toolName}, callId=${args.toolCallId}, thread=${args.threadId}, msgId=${args.msgId}`);
-        const expiresAt = Date.now() + TOOL_CALL_TIMEOUT_MS;
-        const toolCallId = await ctx.db.insert("tool_calls", {
+        const toolCall = await createToolCallRecord(ctx, {
             threadId: args.threadId,
             msgId: args.msgId,
             toolCallId: args.toolCallId,
             toolName: args.toolName,
             callback: args.callback,
-            callbackAttempt: args.callback ? 0 : undefined,
+            handler: args.handler,
+            retry: normalizeSyncToolRetryPolicy(args.retry),
             args: args.args,
             saveDelta: args.saveDelta,
-            timeoutMs: TOOL_CALL_TIMEOUT_MS,
-            expiresAt,
-            status: "pending",
-        });
-        const timeoutFnId = await ctx.scheduler.runAfter(TOOL_CALL_TIMEOUT_MS, internal.tool_calls.failPendingToolCall, {
-            threadId: args.threadId,
-            toolCallId: args.toolCallId,
         });
-        await ctx.db.patch(toolCallId, { timeoutFnId });
-        const toolCall = await ctx.db.get(toolCallId);
         return publicToolCall(toolCall);
     },
 });
@@ -155,7 +257,13 @@ export const setResult = mutation({
         }
         logger.debug(`setResult: callId=${toolCall.toolCallId}, tool=${toolCall.toolName}`);
         await cleanupTimeoutFn(ctx, toolCall);
-        await ctx.db.patch(args.id, { result: args.result, status: "completed" });
+        await cleanupExecutionRetryFn(ctx, toolCall);
+        await ctx.db.patch(args.id, {
+            result: args.result,
+            status: "completed",
+            executionRetryFnId: undefined,
+            nextRetryAt: undefined,
+        });
         return true;
     },
 });
@@ -176,7 +284,14 @@ export const setError = mutation({
         }
         logger.debug(`setError: callId=${toolCall.toolCallId}, tool=${toolCall.toolName}, error=${args.error}`);
         await cleanupTimeoutFn(ctx, toolCall);
-        await ctx.db.patch(args.id, { error: args.error, status: "failed", callbackLastError: args.error });
+        await cleanupExecutionRetryFn(ctx, toolCall);
+        await ctx.db.patch(args.id, {
+            error: args.error,
+            status: "failed",
+            callbackLastError: args.error,
+            executionRetryFnId: undefined,
+            nextRetryAt: undefined,
+        });
         return true;
     },
 });
@@ -276,6 +391,7 @@ export const scheduleToolCall = mutation({
         toolName: v.string(),
         args: v.any(),
         handler: v.string(),
+        retry: v.optional(v.any()),
         saveDelta: v.boolean(),
     },
     returns: v.null(),
@@ -287,11 +403,13 @@ export const scheduleToolCall = mutation({
         }
         logger.debug(`scheduleToolCall: tool=${args.toolName}, callId=${args.toolCallId}, thread=${args.threadId}`);
         // Create the tool call record
-        await ctx.runMutation(api.tool_calls.create, {
+        await createToolCallRecord(ctx, {
             threadId: args.threadId,
             msgId: args.msgId,
             toolCallId: args.toolCallId,
             toolName: args.toolName,
+            handler: args.handler,
+            retry: normalizeSyncToolRetryPolicy(args.retry),
             args: args.args,
             saveDelta: args.saveDelta,
         });
@@ -330,7 +448,7 @@ export const scheduleAsyncToolCall = mutation({
         }
         logger.debug(`scheduleAsyncToolCall: tool=${args.toolName}, callId=${args.toolCallId}, thread=${args.threadId}`);
         // Create the tool call record (will remain pending until addToolResult is called)
-        await ctx.runMutation(api.tool_calls.create, {
+        await createToolCallRecord(ctx, {
             threadId: args.threadId,
             toolCallId: args.toolCallId,
             msgId: args.msgId,
@@ -354,6 +472,126 @@ export const scheduleAsyncToolCall = mutation({
         return null;
     },
 });
+export const updateExecutionRetryState = internalMutation({
+    args: {
+        threadId: v.id("threads"),
+        toolCallId: v.string(),
+        executionAttempt: v.number(),
+        executionLastError: v.optional(v.string()),
+        nextRetryAt: v.optional(v.number()),
+        executionRetryFnId: v.optional(v.id("_scheduled_functions")),
+        clearNextRetryAt: v.optional(v.boolean()),
+        clearExecutionRetryFnId: v.optional(v.boolean()),
+    },
+    returns: v.null(),
+    handler: async (ctx, args) => {
+        const toolCall = await getToolCallByScope(ctx, {
+            threadId: args.threadId,
+            toolCallId: args.toolCallId,
+        });
+        if (!toolCall || toolCall.status !== "pending") {
+            return null;
+        }
+        const patch = {
+            executionAttempt: args.executionAttempt,
+        };
+        if (args.executionLastError !== undefined) {
+            patch.executionLastError = args.executionLastError;
+        }
+        if (args.nextRetryAt !== undefined) {
+            patch.nextRetryAt = args.nextRetryAt;
+        }
+        if (args.executionRetryFnId !== undefined) {
+            patch.executionRetryFnId = args.executionRetryFnId;
+        }
+        if (args.clearNextRetryAt) {
+            patch.nextRetryAt = undefined;
+        }
+        if (args.clearExecutionRetryFnId) {
+            patch.executionRetryFnId = undefined;
+        }
+        await ctx.db.patch(toolCall._id, patch);
+        return null;
+    },
+});
+export const scheduleExecutionRetry = internalMutation({
+    args: {
+        threadId: v.id("threads"),
+        toolCallId: v.string(),
+        handler: v.string(),
+        executionAttempt: v.number(),
+        executionLastError: v.string(),
+        nextRetryAt: v.number(),
+    },
+    returns: v.null(),
+    handler: async (ctx, args) => {
+        const toolCall = await getToolCallByScope(ctx, {
+            threadId: args.threadId,
+            toolCallId: args.toolCallId,
+        });
+        if (!toolCall || toolCall.status !== "pending") {
+            return null;
+        }
+        await cleanupExecutionRetryFn(ctx, toolCall);
+        const delayMs = Math.max(0, args.nextRetryAt - Date.now());
+        const executionRetryFnId = await ctx.scheduler.runAfter(delayMs, internal.tool_calls.executeToolCall, {
+            threadId: args.threadId,
+            toolCallId: args.toolCallId,
+            handler: args.handler,
+        });
+        await ctx.db.patch(toolCall._id, {
+            executionAttempt: args.executionAttempt,
+            executionLastError: args.executionLastError,
+            nextRetryAt: args.nextRetryAt,
+            executionRetryFnId,
+        });
+        return null;
+    },
+});
+export const resumePendingSyncToolExecutions = mutation({
+    args: {
+        limit: v.optional(v.number()),
+    },
+    returns: v.number(),
+    handler: async (ctx, args) => {
+        const limit = Math.max(1, Math.floor(args.limit ?? 100));
+        const pending = await ctx.db
+            .query("tool_calls")
+            .withIndex("by_status_only", (q) => q.eq("status", "pending"))
+            .take(limit * 2);
+        let resumed = 0;
+        const now = Date.now();
+        for (const toolCall of pending) {
+            if (resumed >= limit) {
+                break;
+            }
+            if (!toolCall.handler) {
+                continue;
+            }
+            if (toolCall.executionRetryFnId) {
+                const retryFn = await ctx.db.system.get(toolCall.executionRetryFnId);
+                if (retryFn?.state.kind === "pending") {
+                    continue;
+                }
+            }
+            const nextRetryAt = toolCall.nextRetryAt ?? now;
+            const delayMs = Math.max(0, nextRetryAt - now);
+            const executionRetryFnId = await ctx.scheduler.runAfter(delayMs, internal.tool_calls.executeToolCall, {
+                threadId: toolCall.threadId,
+                toolCallId: toolCall.toolCallId,
+                handler: toolCall.handler,
+            });
+            await ctx.db.patch(toolCall._id, {
+                executionRetryFnId,
+            });
+            resumed += 1;
+        }
+        if (resumed > 0) {
+            logger.warn(`resumePendingSyncToolExecutions: resumed ${resumed} pending sync tool call(s)`);
+        }
+        return resumed;
+    },
+});
 /**
  * Execute a tool call
  */
@@ -365,7 +603,6 @@ export const executeToolCall = internalAction({
     },
     returns: v.null(),
     handler: async (ctx, args) => {
-        logger.debug(`executeToolCall: callId=${args.toolCallId}, thread=${args.threadId}`);
         // Get the tool call record
         const toolCall = await ctx.runQuery(api.tool_calls.getByToolCallId, {
             threadId: args.threadId,
@@ -374,32 +611,107 @@ export const executeToolCall = internalAction({
         if (!toolCall) {
             throw new Error(`Tool call ${args.toolCallId} not found`);
         }
+        if (toolCall.status !== "pending") {
+            logger.debug(`executeToolCall: skipping callId=${args.toolCallId}, status already terminal (${toolCall.status})`);
+            return null;
+        }
         logger.debug(`executeToolCall: tool=${toolCall.toolName}`);
+        const thread = await ctx.runQuery(api.threads.get, {
+            threadId: args.threadId,
+        });
+        if (!thread) {
+            throw new Error(`Thread ${args.threadId} not found`);
+        }
+        if (thread.stopSignal || thread.status === "stopped") {
+            await ctx.runMutation(api.tool_calls.addToolError, {
+                threadId: args.threadId,
+                toolCallId: toolCall.toolCallId,
+                error: "Tool execution cancelled because the thread was stopped",
+            });
+            return null;
+        }
+        const handler = toolCall.handler ?? args.handler;
+        const retryPolicy = normalizeSyncToolRetryPolicy(toolCall.executionRetryPolicy);
+        const retryEnabled = retryPolicy?.enabled === true;
+        const maxAttempts = retryEnabled ? Math.max(1, retryPolicy.maxAttempts ?? SYNC_TOOL_MAX_ATTEMPTS) : 1;
+        const toolArgs = typeof toolCall.args === "object" && toolCall.args !== null ? toolCall.args : {};
+        const attempt = Math.max(1, (toolCall.executionAttempt ?? 0) + 1);
+        await ctx.runMutation(internal.tool_calls.updateExecutionRetryState, {
+            threadId: args.threadId,
+            toolCallId: args.toolCallId,
+            executionAttempt: attempt,
+            clearNextRetryAt: true,
+            clearExecutionRetryFnId: true,
+        });
+        logger.debug(`executeToolCall: callId=${args.toolCallId}, thread=${args.threadId}, attempt=${attempt}/${maxAttempts}`);
         try {
-            // Execute the tool handler
-            // The handler string is passed from the client and we need to resolve it
-            // For now, we'll use ctx.runAction with a dynamic reference
-            // This requires the handler to be a proper function reference string
-            const toolArgs = typeof toolCall.args === "object" && toolCall.args !== null ? toolCall.args : {};
             logger.debug(`executeToolCall: invoking handler for callId=${args.toolCallId}`);
-            const result = await ctx.runAction(args.handler, toolArgs);
+            const result = await ctx.runAction(handler, toolArgs);
             logger.debug(`executeToolCall: handler succeeded for callId=${args.toolCallId}`);
             await ctx.runMutation(api.tool_calls.addToolResult, {
                 threadId: args.threadId,
                 result,
                 toolCallId: toolCall.toolCallId,
             });
+            return null;
         }
         catch (e) {
-            const error = e instanceof Error ? e.message : String(e);
-            logger.debug(`executeToolCall: handler failed for callId=${args.toolCallId}: ${error}`);
+            const errorInfo = extractToolErrorInfo(e);
+            const error = errorInfo.message;
+            logger.debug(`executeToolCall: handler failed for callId=${args.toolCallId} (attempt=${attempt}): ${error}`);
+            let retryable = false;
+            if (retryEnabled) {
+                if (retryPolicy.shouldRetryError) {
+                    try {
+                        const decision = await ctx.runAction(retryPolicy.shouldRetryError, {
+                            threadId: args.threadId,
+                            toolCallId: args.toolCallId,
+                            toolName: toolCall.toolName,
+                            args: toolCall.args,
+                            error,
+                            attempt,
+                            maxAttempts,
+                        });
+                        retryable = isRetryableDecision(decision);
+                    }
+                    catch (classifierError) {
+                        logger.warn(`executeToolCall: shouldRetryError failed for callId=${args.toolCallId}, falling back to default classifier: ${classifierError instanceof Error ? classifierError.message : String(classifierError)}`);
+                        retryable = isRetryableToolErrorDefault(errorInfo);
+                    }
+                }
+                else {
+                    retryable = isRetryableToolErrorDefault(errorInfo);
+                }
+            }
+            if (retryEnabled && retryable && attempt < maxAttempts) {
+                const delayMs = computeRetryDelayMs(attempt, retryPolicy.backoff);
+                const nextRetryAt = Date.now() + delayMs;
+                await ctx.runMutation(internal.tool_calls.scheduleExecutionRetry, {
+                    threadId: args.threadId,
+                    toolCallId: args.toolCallId,
+                    handler,
+                    executionAttempt: attempt,
+                    executionLastError: error,
+                    nextRetryAt,
+                });
+                logger.warn(`executeToolCall: scheduled retry for callId=${args.toolCallId} in ${delayMs}ms (attempt ${attempt + 1}/${maxAttempts})`);
+                return null;
+            }
+            await ctx.runMutation(internal.tool_calls.updateExecutionRetryState, {
+                threadId: args.threadId,
+                toolCallId: args.toolCallId,
+                executionAttempt: attempt,
+                executionLastError: error,
+                clearNextRetryAt: true,
+                clearExecutionRetryFnId: true,
+            });
             await ctx.runMutation(api.tool_calls.addToolError, {
                 threadId: args.threadId,
                 error,
                 toolCallId: toolCall.toolCallId,
             });
+            return null;
         }
-        return null;
     },
 });
 /**
@@ -543,6 +855,7 @@ export const onToolComplete = internalMutation({
                 status: "stopped",
                 activeStream: null,
                 continue: false,
+                retryState: undefined,
             });
             if (thread.onStatusChangeHandle && previousStatus !== "stopped") {
                 await ctx.runMutation(thread.onStatusChangeHandle, {
@@ -562,6 +875,10 @@ export const onToolComplete = internalMutation({
             }
             return null;
         }
+        if (!shouldContinueAfterToolCompletion(thread.status)) {
+            logger.debug(`onToolComplete: thread status=${thread.status}, skipping continuation`);
+            return null;
+        }
         // Check for pending tool calls
         const pending = await ctx.runQuery(api.tool_calls.listPending, {
             threadId: args.threadId,