convex-durable-agents 0.2.3 → 0.2.4

package/src/client/types.ts

@@ -33,6 +33,17 @@ export type PaginationResult<T> = {
 
 export type ThreadStatus = "streaming" | "awaiting_tool_results" | "completed" | "failed" | "stopped";
 
+export type RetryState = {
+  scope: "stream";
+  attempt: number;
+  maxAttempts: number;
+  nextRetryAt: number;
+  error: string;
+  kind?: string;
+  retryable: boolean;
+  requiresExplicitHandling: boolean;
+};
+
 export type ThreadDoc = {
   _id: string;
   _creationTime: number;
@@ -40,6 +51,7 @@ export type ThreadDoc = {
   stopSignal: boolean;
   streamId?: string | null;
   streamFnHandle: string;
+  retryState?: RetryState;
 };
 
 const vThreadStatus = v.union(
@@ -59,6 +71,19 @@ export const _vClientThreadDoc = v.object({
   streamFnHandle: v.string(),
   workpoolEnqueueAction: v.optional(v.string()),
   toolExecutionWorkpoolEnqueueAction: v.optional(v.string()),
+  retryState: v.optional(
+    v.object({
+      scope: v.literal("stream"),
+      attempt: v.number(),
+      maxAttempts: v.number(),
+      nextRetryAt: v.number(),
+      error: v.string(),
+      kind: v.optional(v.string()),
+      retryable: v.boolean(),
+      requiresExplicitHandling: v.boolean(),
+      retryFnId: v.optional(v.string()),
+    }),
+  ),
 });
 
 export type MessageDoc = UIMessage<any> & {
@@ -77,12 +102,47 @@ export function messageDocToUIMessage(message: MessageDoc): UIMessage {
   };
 }
 
+export type RetryBackoffConfig =
+  | {
+      strategy?: "fixed";
+      delayMs: number;
+      jitter?: boolean;
+    }
+  | {
+      strategy: "exponential";
+      initialDelayMs: number;
+      multiplier?: number;
+      maxDelayMs?: number;
+      jitter?: boolean;
+    };
+
+export type SyncToolRetryOptions = {
+  /**
+   * Opt-in to retry for this sync tool.
+   */
+  enabled: true;
+  /**
+   * Maximum execution attempts including the initial attempt.
+   */
+  maxAttempts?: number;
+  /**
+   * Retry backoff policy.
+   */
+  backoff?: RetryBackoffConfig;
+  /**
+   * Optional function to classify whether an error is retryable.
+   * Receives { threadId, toolCallId, toolName, args, error, attempt, maxAttempts }.
+   */
+  shouldRetryError?: FunctionReference<"action", "internal" | "public">;
+};
+
 // Sync durable tool definition - handler returns the result directly
 export type SyncTool<INPUT = unknown, OUTPUT = unknown> = {
   type: "sync";
   description: string;
   parameters: unknown; // JSON Schema
   handler: FunctionReference<"action", "internal" | "public">;
+  retry?: SyncToolRetryOptions;
   _inputType?: INPUT;
   _outputType?: OUTPUT;
 };

package/src/component/_generated/component.ts

@@ -163,6 +163,13 @@ export type ComponentApi<Name extends string | undefined = string | undefined> =
       >;
     };
     threads: {
+      clearRetryState: FunctionReference<
+        "mutation",
+        "internal",
+        { threadId: string },
+        null,
+        Name
+      >;
       clearStreamId: FunctionReference<
         "mutation",
         "internal",
@@ -182,6 +189,17 @@ export type ComponentApi<Name extends string | undefined = string | undefined> =
         {
           _creationTime: number;
           _id: string;
+          retryState?: {
+            attempt: number;
+            error: string;
+            kind?: string;
+            maxAttempts: number;
+            nextRetryAt: number;
+            requiresExplicitHandling: boolean;
+            retryFnId?: string;
+            retryable: boolean;
+            scope: "stream";
+          };
           status:
             | "streaming"
             | "awaiting_tool_results"
@@ -220,6 +238,17 @@ export type ComponentApi<Name extends string | undefined = string | undefined> =
         {
           _creationTime: number;
           _id: string;
+          retryState?: {
+            attempt: number;
+            error: string;
+            kind?: string;
+            maxAttempts: number;
+            nextRetryAt: number;
+            requiresExplicitHandling: boolean;
+            retryFnId?: string;
+            retryable: boolean;
+            scope: "stream";
+          };
           status:
             | "streaming"
             | "awaiting_tool_results"
@@ -241,6 +270,17 @@ export type ComponentApi<Name extends string | undefined = string | undefined> =
         Array<{
           _creationTime: number;
           _id: string;
+          retryState?: {
+            attempt: number;
+            error: string;
+            kind?: string;
+            maxAttempts: number;
+            nextRetryAt: number;
+            requiresExplicitHandling: boolean;
+            retryFnId?: string;
+            retryable: boolean;
+            scope: "stream";
+          };
           status:
             | "streaming"
             | "awaiting_tool_results"
@@ -276,6 +316,23 @@ export type ComponentApi<Name extends string | undefined = string | undefined> =
         null,
         Name
       >;
+      scheduleRetry: FunctionReference<
+        "mutation",
+        "internal",
+        {
+          attempt: number;
+          error: string;
+          kind?: string;
+          maxAttempts: number;
+          nextRetryAt: number;
+          requiresExplicitHandling: boolean;
+          retryable: boolean;
+          scope: "stream";
+          threadId: string;
+        },
+        null,
+        Name
+      >;
       setStatus: FunctionReference<
         "mutation",
         "internal",
@@ -321,7 +378,9 @@ export type ComponentApi<Name extends string | undefined = string | undefined> =
         {
           args: any;
           callback?: string;
+          handler?: string;
           msgId: string;
+          retry?: any;
           saveDelta: boolean;
           threadId: string;
           toolCallId: string;
@@ -331,9 +390,18 @@ export type ComponentApi<Name extends string | undefined = string | undefined> =
           _creationTime: number;
           _id: string;
           args: any;
+          callbackAttempt?: number;
+          callbackLastError?: string;
           error?: string;
+          executionAttempt?: number;
+          executionLastError?: string;
+          executionMaxAttempts?: number;
+          executionRetryPolicy?: any;
+          handler?: string;
           msgId: string;
+          nextRetryAt?: number;
           result?: any;
+          status: "pending" | "completed" | "failed";
           threadId: string;
           toolCallId: string;
           toolName: string;
@@ -348,9 +416,18 @@ export type ComponentApi<Name extends string | undefined = string | undefined> =
           _creationTime: number;
           _id: string;
           args: any;
+          callbackAttempt?: number;
+          callbackLastError?: string;
           error?: string;
+          executionAttempt?: number;
+          executionLastError?: string;
+          executionMaxAttempts?: number;
+          executionRetryPolicy?: any;
+          handler?: string;
           msgId: string;
+          nextRetryAt?: number;
           result?: any;
+          status: "pending" | "completed" | "failed";
           threadId: string;
           toolCallId: string;
           toolName: string;
@@ -365,9 +442,18 @@ export type ComponentApi<Name extends string | undefined = string | undefined> =
           _creationTime: number;
           _id: string;
           args: any;
+          callbackAttempt?: number;
+          callbackLastError?: string;
           error?: string;
+          executionAttempt?: number;
+          executionLastError?: string;
+          executionMaxAttempts?: number;
+          executionRetryPolicy?: any;
+          handler?: string;
           msgId: string;
+          nextRetryAt?: number;
           result?: any;
+          status: "pending" | "completed" | "failed";
           threadId: string;
           toolCallId: string;
           toolName: string;
@@ -382,15 +468,31 @@ export type ComponentApi<Name extends string | undefined = string | undefined> =
           _creationTime: number;
           _id: string;
           args: any;
+          callbackAttempt?: number;
+          callbackLastError?: string;
           error?: string;
+          executionAttempt?: number;
+          executionLastError?: string;
+          executionMaxAttempts?: number;
+          executionRetryPolicy?: any;
+          handler?: string;
           msgId: string;
+          nextRetryAt?: number;
           result?: any;
+          status: "pending" | "completed" | "failed";
           threadId: string;
           toolCallId: string;
           toolName: string;
         }>,
         Name
       >;
+      resumePendingSyncToolExecutions: FunctionReference<
+        "mutation",
+        "internal",
+        { limit?: number },
+        number,
+        Name
+      >;
       scheduleAsyncToolCall: FunctionReference<
         "mutation",
         "internal",
@@ -413,6 +515,7 @@ export type ComponentApi<Name extends string | undefined = string | undefined> =
           args: any;
           handler: string;
           msgId: string;
+          retry?: any;
           saveDelta: boolean;
           threadId: string;
           toolCallId: string;

package/src/component/agent.ts

@@ -1,7 +1,7 @@
 import type { FunctionHandle, FunctionReference, GenericDataModel, GenericMutationCtx } from "convex/server";
 import { v } from "convex/values";
-import { Logger } from "../logger.js";
-import { STREAM_LIVENESS_THRESHOLD_MS } from "../streaming.js";
+import { Logger } from "../utils/logger.js";
+import { STREAM_LIVENESS_THRESHOLD_MS } from "../utils/streaming.js";
 import { api, internal } from "./_generated/api.js";
 import { action, mutation } from "./_generated/server.js";
 import { cancelStream, isAlive } from "./streams.js";
@@ -69,6 +69,7 @@ export const continueStream = mutation({
         status: "stopped",
         activeStream: null,
         continue: false,
+        retryState: undefined,
       });
       if (thread.onStatusChangeHandle && previousStatus !== "stopped") {
         await ctx.runMutation(thread.onStatusChangeHandle as FunctionHandle<"mutation">, {
@@ -93,6 +94,27 @@ export const continueStream = mutation({
       return null;
     }
 
+    if (thread.retryState?.scope === "stream") {
+      const now = Date.now();
+      if (thread.retryState.nextRetryAt > now) {
+        const delayMs = Math.max(0, thread.retryState.nextRetryAt - now);
+        logger.debug(
+          `Retry pending for thread=${args.threadId}; nextRetryAt=${thread.retryState.nextRetryAt}, now=${now}`,
+        );
+        const retryFnId = await ctx.scheduler.runAfter(delayMs, api.agent.continueStream, {
+          threadId: args.threadId,
+        });
+        await ctx.db.patch(thread._id, {
+          retryState: {
+            ...thread.retryState,
+            retryFnId,
+          },
+        });
+        return null;
+      }
+      logger.debug(`Retry window reached for thread=${args.threadId}; continuing with persisted retryState`);
+    }
+
     // Check for pending tool calls
     const pendingToolCalls = await ctx.runQuery(api.tool_calls.listPending, {
       threadId: args.threadId,

package/src/component/schema.ts

@@ -10,6 +10,18 @@ export const vThreadStatus = v.union(
   v.literal("stopped"),
 );
 
+export const vRetryState = v.object({
+  scope: v.literal("stream"),
+  attempt: v.number(),
+  maxAttempts: v.number(),
+  nextRetryAt: v.number(),
+  error: v.string(),
+  kind: v.optional(v.string()),
+  retryable: v.boolean(),
+  requiresExplicitHandling: v.boolean(),
+  retryFnId: v.optional(v.id("_scheduled_functions")),
+});
+
 // AI SDK message content - supports both string and array of parts
 export const vMessageContent = v.union(
   v.string(),
@@ -70,6 +82,8 @@ const schema = defineSchema({
     onStatusChangeHandle: v.optional(v.string()),
     // Monotonically increasing sequence number for streams of this thread
     seq: v.number(),
+    // Pending retry metadata for stream retries.
+    retryState: v.optional(vRetryState),
   }).index("by_status", ["status"]),
 
   // AI SDK compatible message storage
@@ -95,6 +109,13 @@ const schema = defineSchema({
     callback: v.optional(v.string()),
     callbackAttempt: v.optional(v.number()),
     callbackLastError: v.optional(v.string()),
+    executionAttempt: v.optional(v.number()),
+    executionMaxAttempts: v.optional(v.number()),
+    executionLastError: v.optional(v.string()),
+    executionRetryPolicy: v.optional(v.any()),
+    nextRetryAt: v.optional(v.number()),
+    executionRetryFnId: v.optional(v.id("_scheduled_functions")),
+    handler: v.optional(v.string()),
     timeoutMs: v.optional(v.union(v.number(), v.null())),
     expiresAt: v.optional(v.union(v.number(), v.null())),
     timeoutFnId: v.optional(v.id("_scheduled_functions")),
@@ -105,6 +126,7 @@ const schema = defineSchema({
     saveDelta: v.optional(v.boolean()),
   })
     .index("by_thread", ["threadId"])
+    .index("by_status_only", ["status"])
     .index("by_status", ["threadId", "status"])
     .index("by_thread_tool_call_id", ["threadId", "toolCallId"])
     .index("by_tool_call_id", ["toolCallId"]),

package/src/component/streams.ts

@@ -1,7 +1,7 @@
 import type { UIMessageChunk } from "ai";
 import { type Infer, v } from "convex/values";
-import { Logger } from "../logger.js";
-import { STREAM_LIVENESS_THRESHOLD_MS } from "../streaming.js";
+import { Logger } from "../utils/logger.js";
+import { STREAM_LIVENESS_THRESHOLD_MS } from "../utils/streaming.js";
 import { api, internal } from "./_generated/api";
 import type { Doc, Id } from "./_generated/dataModel";
 import { internalMutation, type MutationCtx, mutation, query } from "./_generated/server";

package/src/component/threads.ts

@@ -1,9 +1,10 @@
 import type { FunctionHandle } from "convex/server";
 import { v } from "convex/values";
-import { Logger } from "../logger.js";
+import { Logger } from "../utils/logger.js";
+import { api } from "./_generated/api.js";
 import type { Doc, Id } from "./_generated/dataModel.js";
-import { internalQuery, mutation, query } from "./_generated/server.js";
-import { vThreadStatus } from "./schema.js";
+import { internalQuery, type MutationCtx, mutation, query } from "./_generated/server.js";
+import { vRetryState, vThreadStatus } from "./schema.js";
 
 const logger = new Logger("threads");
 const FINALIZER_MISMATCH_ALERT_WINDOW_MS = 5 * 60 * 1000;
@@ -64,6 +65,7 @@ export type ThreadDoc = {
   streamFnHandle: string;
   workpoolEnqueueAction?: string;
   toolExecutionWorkpoolEnqueueAction?: string;
+  retryState?: Doc<"threads">["retryState"];
 };
 
 function publicThread(thread: Doc<"threads">): ThreadDoc {
@@ -76,6 +78,7 @@ function publicThread(thread: Doc<"threads">): ThreadDoc {
     streamFnHandle: thread.streamFnHandle,
     workpoolEnqueueAction: thread.workpoolEnqueueAction,
     toolExecutionWorkpoolEnqueueAction: thread.toolExecutionWorkpoolEnqueueAction,
+    retryState: thread.retryState,
   };
 }
 
@@ -89,6 +92,7 @@ export const vThreadDoc = v.object({
   streamFnHandle: v.string(),
   workpoolEnqueueAction: v.optional(v.string()),
   toolExecutionWorkpoolEnqueueAction: v.optional(v.string()),
+  retryState: v.optional(vRetryState),
 });
 
 export const vThreadDocWithStreamFnHandle = v.object({
@@ -104,8 +108,19 @@ export const vThreadDocWithStreamFnHandle = v.object({
   activeStream: v.optional(v.union(v.id("streams"), v.null())),
   continue: v.optional(v.boolean()),
   seq: v.number(),
+  retryState: v.optional(vRetryState),
 });
 
+async function cancelRetryFnIfPending(ctx: MutationCtx, thread: Doc<"threads">): Promise<void> {
+  if (!thread.retryState?.retryFnId) {
+    return;
+  }
+  const retryFn = await ctx.db.system.get(thread.retryState.retryFnId);
+  if (retryFn?.state.kind === "pending") {
+    await ctx.scheduler.cancel(thread.retryState.retryFnId);
+  }
+}
+
 export const create = mutation({
   args: {
     streamFnHandle: v.string(),
@@ -149,6 +164,74 @@ export const getWithStreamFnHandle = internalQuery({
   },
 });
 
+export const scheduleRetry = mutation({
+  args: {
+    threadId: v.id("threads"),
+    scope: v.literal("stream"),
+    attempt: v.number(),
+    maxAttempts: v.number(),
+    nextRetryAt: v.number(),
+    error: v.string(),
+    kind: v.optional(v.string()),
+    retryable: v.boolean(),
+    requiresExplicitHandling: v.boolean(),
+  },
+  returns: v.null(),
+  handler: async (ctx, args) => {
+    const thread = await ctx.db.get(args.threadId);
+    if (!thread) {
+      throw new Error(`Thread ${args.threadId} not found`);
+    }
+
+    await cancelRetryFnIfPending(ctx, thread);
+
+    if (thread.stopSignal) {
+      await ctx.db.patch(args.threadId, {
+        retryState: undefined,
+      });
+      return null;
+    }
+
+    const delayMs = Math.max(0, args.nextRetryAt - Date.now());
+    const retryFnId = await ctx.scheduler.runAfter(delayMs, api.agent.continueStream, {
+      threadId: args.threadId,
+    });
+
+    await ctx.db.patch(args.threadId, {
+      retryState: {
+        scope: args.scope,
+        attempt: args.attempt,
+        maxAttempts: args.maxAttempts,
+        nextRetryAt: args.nextRetryAt,
+        error: args.error,
+        kind: args.kind,
+        retryable: args.retryable,
+        requiresExplicitHandling: args.requiresExplicitHandling,
+        retryFnId,
+      },
+    });
+    return null;
+  },
+});
+
+export const clearRetryState = mutation({
+  args: {
+    threadId: v.id("threads"),
+  },
+  returns: v.null(),
+  handler: async (ctx, args) => {
+    const thread = await ctx.db.get(args.threadId);
+    if (!thread) {
+      throw new Error(`Thread ${args.threadId} not found`);
+    }
+    await cancelRetryFnIfPending(ctx, thread);
+    await ctx.db.patch(args.threadId, {
+      retryState: undefined,
+    });
+    return null;
+  },
+});
+
 export const resume = mutation({
   args: {
     threadId: v.id("threads"),
@@ -344,6 +427,11 @@ export const setStopSignal = mutation({
     if (!thread) {
       throw new Error(`Thread ${args.threadId} not found`);
     }
+    if (args.stopSignal) {
+      await cancelRetryFnIfPending(ctx, thread);
+      await ctx.db.patch(args.threadId, { stopSignal: args.stopSignal, retryState: undefined });
+      return null;
+    }
     await ctx.db.patch(args.threadId, { stopSignal: args.stopSignal });
     return null;
   },
@@ -383,6 +471,7 @@ export const remove = mutation({
     if (!thread) {
       throw new Error(`Thread ${args.threadId} not found`);
     }
+    await cancelRetryFnIfPending(ctx, thread);
     // Delete all messages for this thread
     const messages = await ctx.db
       .query("messages")