@standardagents/builder 0.25.4 → 0.25.6

package/dist/runtime.d.ts

@@ -1,5 +1,5 @@
-import { e as ThreadEnv, A as Agent, h as ThreadMetadata, i as ToolResult, M as Message, l as FileRecord, F as FlowState, n as FileStats, G as GrepResult, m as AttachmentRef, r as MessageContent, E as Env, g as ThreadInstance } from './index-BsS0cHzL.js';
-export { f as BuilderThreadEndpointHandler, B as Controller, a as ControllerContext, j as FlowResult, p as ImageContentPart, I as ImageMetadata, L as LLMResponse, q as MultimodalContent, R as RequestContext, S as StorageBackend, k as TelemetryEvent, o as TextContentPart, b as ThreadEndpointContext, T as ToolCall, c as createThreadEndpointHandler, d as defineController } from './index-BsS0cHzL.js';
+import { e as ThreadEnv, A as Agent, h as ThreadMetadata, i as ToolResult, M as Message, l as FileRecord, F as FlowState, n as FileStats, G as GrepResult, m as AttachmentRef, r as MessageContent, E as Env, g as ThreadInstance } from './index-CX9Ti98s.js';
+export { f as BuilderThreadEndpointHandler, B as Controller, a as ControllerContext, j as FlowResult, p as ImageContentPart, I as ImageMetadata, L as LLMResponse, q as MultimodalContent, R as RequestContext, S as StorageBackend, k as TelemetryEvent, o as TextContentPart, b as ThreadEndpointContext, T as ToolCall, c as createThreadEndpointHandler, d as defineController } from './index-CX9Ti98s.js';
 import { CodeExecutionOptions, CodeExecutionResult, SubagentRegistryEntry, InjectMessageInput, QueueMessageInput, ClientToolRequest, ClientToolResponse, SkillMetadata, SkillFileInput, ModelDefinition as ModelDefinition$1, ToolArgs, PromptTextPart, PromptEnvPart, VariableDefinition, SubpromptConfig as SubpromptConfig$1, PromptToolConfig as PromptToolConfig$1, SubagentToolConfig as SubagentToolConfig$1, ReasoningConfig, SideConfig as SideConfig$1, LLMProviderInterface, ContentPart, TextPart, ImagePart, FilePart, NamespaceContext, DefinitionLoader } from '@standardagents/spec';
 export { AgentType, DefinitionLoader, GlobalNamespaceContext, HookSignatures, ImageContent, ModelCapabilities, ModelProvider, NamespaceContext, PackageSignature, PackedExports, PackedMeta, PackedMetadata, PackedNamespaceContext, PromptInput, PromptTextPart, ProviderAssistantMessage, ProviderError, ProviderErrorCode, ProviderFactory, ProviderFactoryConfig, ProviderFinishReason, ProviderGeneratedImage, ProviderMessage, ProviderMessageContent, ModelCapabilities as ProviderModelCapabilities, ProviderReasoningDetail, ProviderRequest, ProviderResponse, ProviderStreamChunk, ProviderSystemMessage, ProviderTool, ProviderToolCallPart, ProviderToolMessage, ProviderToolResultContent, ProviderUsage, ProviderUserMessage, ReasoningConfig, StructuredPrompt, TextContent, Tool, ToolArgs, ToolArgsNode, ToolArgsRawShape, ToolContent, belongsToPackage, defineAgent, defineHook, defineModel, definePrompt, defineTool, isPacked, isVisibleInNamespace, mapReasoningLevel } from '@standardagents/spec';
 import { DurableObject, WorkerEntrypoint } from 'cloudflare:workers';
@@ -184,6 +184,11 @@ declare class DurableThread<Env extends ThreadEnv = ThreadEnv> extends DurableOb
     private alarmQueueRearmed;
     private currentAbortController;
     private isExecuting;
+    private lastProgressAt;
+    private watchdogWedgeStrikes;
+    private readonly watchdogIntervalMs;
+    private readonly watchdogResumeWindowMs;
+    private readonly watchdogMaxResumesPerWindow;
     private codeExecutionBridges;
     constructor(ctx: DurableObjectState, env: Env);
     /**
@@ -1080,6 +1085,29 @@ declare class DurableThread<Env extends ThreadEnv = ThreadEnv> extends DurableOb
      * the alarm chain. All errors are caught and logged.
      */
     alarm(): Promise<void>;
+    /** Heartbeat: called by LLMRequest on each attempt and streamed chunk. */
+    noteExecutionProgress(): void;
+    /** Arm the standing watchdog for an execution start. One pending row max. */
+    private armExecutionWatchdog;
+    /** Disarm on clean execution end. Completed check rows are noise, not history. */
+    private disarmExecutionWatchdog;
+    /**
+     * The standing check. Evaluates the thread's state AT FIRE TIME — it knows
+     * nothing about whichever request was in flight when it was armed.
+     */
+    private watchdogCheck;
+    /** Mark logs the dead executor abandoned, and tell connected clients. */
+    private finalizeLostExecutionLogs;
+    /**
+     * A subagent whose execution the watchdog gave up on leaves its parent
+     * waiting forever for a completion message. Deliver the failure the same way
+     * a sessionFail would, so the parent's turn wakes and can route around it.
+     * Explicit-communication children (e.g. the compaction agent) stay quiet by
+     * contract — only the registry status updates for those.
+     */
+    private notifyParentOfLostExecution;
+    /** At most N auto-resumes per rolling window, tracked durably. */
+    private consumeWatchdogResumeBudget;
     private normalizePlainRecord;
     private buildExecutionArguments;
     /**

package/dist/runtime.js

@@ -1057,6 +1057,19 @@ function resolveStreamTimeoutMs(env, key, fallback) {
   const parsed = typeof raw === "number" ? raw : typeof raw === "string" ? Number.parseInt(raw, 10) : NaN;
   return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
 }
+function resolveWatchdogStaleCeilingMs(env) {
+  const firstChunk = resolveStreamTimeoutMs(
+    env,
+    "STANDARD_AGENTS_STREAM_FIRST_CHUNK_TIMEOUT_MS",
+    STREAM_FIRST_CHUNK_TIMEOUT_MS
+  );
+  const stall = resolveStreamTimeoutMs(
+    env,
+    "STANDARD_AGENTS_STREAM_STALL_TIMEOUT_MS",
+    STREAM_STALL_TIMEOUT_MS
+  );
+  return Math.max(firstChunk, stall) + 6e4;
+}
 function modelSupportsImageInput(modelDef) {
   const supportsImages = modelDef.capabilities?.supportsImages;
   if (typeof supportsImages === "boolean") {
@@ -1569,6 +1582,7 @@ var init_LLMRequest = __esm({
           attempts++;
           try {
             logId = await this.logRequest(state, actualModelId, context, modelDef, previousLogId);
+            state.thread.instance.noteExecutionProgress?.();
             state.emitTelemetry?.({
               type: "llm_request",
               model: actualModelId,
@@ -1835,6 +1849,7 @@ var init_LLMRequest = __esm({
                 }
                 if (next.done) break;
                 const chunk = next.value;
+                state.thread.instance.noteExecutionProgress?.();
                 if (firstChunkAt === 0) {
                   firstChunkAt = Date.now();
                   emitLogPatch({ time_to_first_token_ms: firstChunkAt - requestSentAt });
@@ -2370,8 +2385,11 @@ var init_LLMRequest = __esm({
        */
       static async logError(state, error, errorType, modelId, startTime, existingLogId) {
         try {
-          const rawErrorMessage = error instanceof Error ? error.message : String(error);
+          let rawErrorMessage = error instanceof Error ? error.message : String(error);
           const errorStack = error instanceof Error ? error.stack : void 0;
+          if (error instanceof Error && error.name === "AbortError" && /operation was aborted/i.test(rawErrorMessage)) {
+            rawErrorMessage = state.abortController?.signal?.aborted ? "Aborted: the execution this request belonged to was cancelled (user stop, watchdog takeover, or a restart) \u2014 retries and fallbacks of a cancelled execution fail immediately." : "Aborted: the request's connection was cancelled mid-flight (provider disconnect or executor restart).";
+          }
           let errorMessage = rawErrorMessage;
           let parsedProviderBody = null;
           const trimmedRaw = rawErrorMessage.trim();
@@ -15023,6 +15041,38 @@ var AlarmQueue = class {
   async ensureAlarmScheduled() {
     await this.rescheduleAlarm();
   }
+  /**
+   * Recover queue items orphaned in 'processing' by a dead predecessor
+   * instance.
+   *
+   * processNext marks an item 'processing' before executing it. If the DO is
+   * evicted or a deploy restarts it mid-execution, that row used to stay
+   * 'processing' FOREVER — only 'pending' rows are ever picked up again — so
+   * queued subagent executions, queued messages, and scheduled effects that
+   * were mid-flight at a restart were silently lost (a subagent that "just
+   * died half way through" with nothing to revive it).
+   *
+   * MUST ONLY be called from the once-per-instance wake path (DurableThread's
+   * alarmQueueRearmed guard): at that moment this fresh instance is executing
+   * nothing, so ANY 'processing' row provably belongs to a dead predecessor.
+   * Calling it at any other time would re-queue items that are legitimately
+   * mid-execution on this very instance.
+   */
+  async recoverOrphanedProcessing(maxAttempts = 3) {
+    const now = Date.now() * 1e3;
+    await this.sql.exec(
+      `UPDATE alarm_queue
+       SET status = 'failed',
+           error = 'Lost mid-execution (the executor restarted) and retry attempts were exhausted.',
+           completed_at = ?
+       WHERE status = 'processing' AND attempts >= ?`,
+      now,
+      maxAttempts
+    );
+    await this.sql.exec(
+      `UPDATE alarm_queue SET status = 'pending' WHERE status = 'processing'`
+    );
+  }
   // ============================================================
   // Effect-specific Methods
   // ============================================================
@@ -15287,6 +15337,7 @@ var AlarmQueue = class {
 
 // src/durable-objects/DurableThread.ts
 init_StreamManager();
+init_LLMRequest();
 
 // src/packing/resolver.ts
 var NamespaceResolutionError = class extends Error {
@@ -15774,6 +15825,18 @@ var DurableThread = class _DurableThread extends DurableObject {
   alarmQueueRearmed = false;
   currentAbortController = null;
   isExecuting = false;
+  // ── Execution watchdog state (see watchdogCheck) ─────────────────────────
+  // In-memory heartbeat: bumped by noteExecutionProgress() on every streamed
+  // chunk and request attempt. Deliberately NOT persisted — after a DO restart
+  // the isExecuting flag alone proves the executor is gone, and while the DO
+  // is alive the in-memory value is authoritative. Zero hot-path storage cost.
+  lastProgressAt = 0;
+  // Consecutive watchdog firings that found a live-but-silent executor. Strike
+  // one aborts the in-flight request; strike two forces takeover.
+  watchdogWedgeStrikes = 0;
+  watchdogIntervalMs = 3e4;
+  watchdogResumeWindowMs = 15 * 60 * 1e3;
+  watchdogMaxResumesPerWindow = 2;
   codeExecutionBridges = /* @__PURE__ */ new Map();
   constructor(ctx, env) {
     super(ctx, env);
@@ -15809,6 +15872,9 @@ var DurableThread = class _DurableThread extends DurableObject {
           case "continueExecution":
             await this.doContinueExecution(args.threadId, args.side);
             break;
+          case "watchdogCheck":
+            await this.watchdogCheck(args.threadId);
+            break;
           default:
             throw new Error(`Unknown method: ${method}`);
         }
@@ -16929,6 +16995,9 @@ var DurableThread = class _DurableThread extends DurableObject {
     }
     this.migratedToVersion = LATEST_SCHEMA_VERSION;
     if (!this.alarmQueueRearmed && typeof this.alarmQueue.ensureAlarmScheduled === "function") {
+      if (typeof this.alarmQueue.recoverOrphanedProcessing === "function") {
+        await this.alarmQueue.recoverOrphanedProcessing();
+      }
       await this.alarmQueue.ensureAlarmScheduled();
       this.alarmQueueRearmed = true;
     }
@@ -19561,6 +19630,234 @@ ${resultContent}${attachmentSummary}`;
       console.error("[DurableThread] Alarm handler failed:", error);
     }
   }
+  // ══════════════════════════════════════════════════════════════════════════
+  // Execution watchdog
+  //
+  // The in-flight stream guards (LLMRequest's first-chunk/stall timeouts) live
+  // inside the execution's promise chain — they die with the DO. A deploy or
+  // eviction mid-request therefore used to leave a durable log row stuck at
+  // is_complete = 0 forever: the thread looked "working", queued messages
+  // never played, and nothing woke it (finalizeOrphanedLogs only runs at the
+  // START of the next execution). Persisted DO alarms are the one timer that
+  // survives restarts and fires without inbound traffic, so a standing
+  // per-thread watchdog alarm re-checks the thread's CURRENT state every 30s
+  // while work is active.
+  //
+  // Design invariants (do not weaken):
+  // - Progress, never duration: only silence ages. A response can stream for
+  //   ten minutes; every chunk bumps the in-memory heartbeat.
+  // - Per-thread, not per-request: the check carries no request identity. By
+  //   the time it fires the thread may be several requests further along —
+  //   it always evaluates fresh SQL + live flags, never a stale snapshot.
+  // - Strictly second line: the staleness ceiling is DERIVED from the stream
+  //   timeouts (resolveWatchdogStaleCeilingMs = max(first-chunk, stall) + 60s)
+  //   so the in-flight guards always get to act first. The watchdog only
+  //   moves when the guard that should have fired didn't.
+  // - Restart proof: isExecuting is volatile; an incomplete log with
+  //   isExecuting === false is PROOF the executor died (executions cannot
+  //   survive a restart). No heartbeat needed for that branch.
+  // - Nonce re-read: every await inside the check is a yield point; state is
+  //   re-read immediately before acting so a healthy thread racing the
+  //   watchdog always wins.
+  // ══════════════════════════════════════════════════════════════════════════
+  /** Heartbeat: called by LLMRequest on each attempt and streamed chunk. */
+  noteExecutionProgress() {
+    this.lastProgressAt = Date.now();
+  }
+  /** Arm the standing watchdog for an execution start. One pending row max. */
+  async armExecutionWatchdog(threadId) {
+    try {
+      this.lastProgressAt = Date.now();
+      this.watchdogWedgeStrikes = 0;
+      const pending = (await this.ctx.storage.sql.exec(
+        `SELECT id FROM alarm_queue WHERE method = 'watchdogCheck' AND status = 'pending' LIMIT 1`
+      )).toArray();
+      if (pending.length > 0) return;
+      await this.alarmQueue.enqueue("watchdogCheck", { threadId }, this.watchdogIntervalMs);
+    } catch (error) {
+      console.error("[DurableThread] Failed to arm execution watchdog (non-fatal):", error);
+    }
+  }
+  /** Disarm on clean execution end. Completed check rows are noise, not history. */
+  async disarmExecutionWatchdog() {
+    try {
+      await this.ctx.storage.sql.exec(
+        `DELETE FROM alarm_queue WHERE method = 'watchdogCheck' AND status IN ('pending', 'completed')`
+      );
+      await this.alarmQueue.ensureAlarmScheduled();
+    } catch (error) {
+      console.error("[DurableThread] Failed to disarm execution watchdog (non-fatal):", error);
+    }
+  }
+  /**
+   * The standing check. Evaluates the thread's state AT FIRE TIME — it knows
+   * nothing about whichever request was in flight when it was armed.
+   */
+  async watchdogCheck(threadId) {
+    const rearm = () => this.alarmQueue.enqueue("watchdogCheck", { threadId }, this.watchdogIntervalMs);
+    try {
+      await this.ctx.storage.sql.exec(
+        `DELETE FROM alarm_queue WHERE method = 'watchdogCheck' AND status = 'completed'`
+      );
+      const readIncomplete = async () => (await this.ctx.storage.sql.exec(
+        `SELECT id FROM logs WHERE is_complete = 0`
+      )).toArray();
+      let incomplete = await readIncomplete();
+      const ceiling = resolveWatchdogStaleCeilingMs(this.env);
+      const silenceMs = this.lastProgressAt === 0 ? Number.POSITIVE_INFINITY : Date.now() - this.lastProgressAt;
+      if (this.isExecuting) {
+        if (incomplete.length === 0) {
+          this.watchdogWedgeStrikes = 0;
+          await rearm();
+          return;
+        }
+        if (silenceMs <= ceiling) {
+          this.watchdogWedgeStrikes = 0;
+          await rearm();
+          return;
+        }
+        this.watchdogWedgeStrikes++;
+        if (this.watchdogWedgeStrikes < 2) {
+          console.warn(
+            `[DurableThread] Watchdog: execution silent for ${Math.round(silenceMs / 1e3)}s (ceiling ${Math.round(ceiling / 1e3)}s) \u2014 observing; takeover next cycle if still silent`
+          );
+          await rearm();
+          return;
+        }
+        console.error(
+          `[DurableThread] Watchdog: wedged execution (${Math.round(silenceMs / 1e3)}s silent across two cycles) \u2014 forcing takeover`
+        );
+        try {
+          this.currentAbortController?.abort();
+        } catch {
+        }
+        this.isExecuting = false;
+        this.currentAbortController = null;
+      } else if (incomplete.length > 0 && silenceMs <= ceiling) {
+        await rearm();
+        return;
+      }
+      incomplete = await readIncomplete();
+      if (this.isExecuting) {
+        await rearm();
+        return;
+      }
+      if (incomplete.length > 0) {
+        await this.finalizeLostExecutionLogs(incomplete.map((row) => row.id));
+      }
+      const stopped = await this.shouldStop();
+      const queued = (await this.ctx.storage.sql.exec(
+        `SELECT COUNT(*) AS c FROM message_queue`
+      )).toArray()[0]?.c ?? 0;
+      if (stopped || incomplete.length === 0 && queued === 0) {
+        await this.disarmExecutionWatchdog();
+        return;
+      }
+      if (await this.consumeWatchdogResumeBudget()) {
+        const side = await this.getLatestQueuedMessageSide() ?? "a";
+        await this.alarmQueue.enqueue("continueExecution", { threadId, side }, 0);
+        console.warn(
+          `[DurableThread] Watchdog: lost execution recovered \u2014 resuming thread ${threadId}`
+        );
+      } else {
+        console.error(
+          `[DurableThread] Watchdog: resume budget exhausted for thread ${threadId} \u2014 leaving thread stopped`
+        );
+        await this.notifyParentOfLostExecution(threadId);
+      }
+      await this.disarmExecutionWatchdog();
+    } catch (error) {
+      console.error("[DurableThread] Watchdog check failed (non-fatal):", error);
+      try {
+        await rearm();
+      } catch {
+      }
+    }
+  }
+  /** Mark logs the dead executor abandoned, and tell connected clients. */
+  async finalizeLostExecutionLogs(logIds) {
+    const error = "Request lost \u2014 the executor restarted (deploy or eviction) or hung before finishing. The thread watchdog finalized it.";
+    const errorType = "execution_lost";
+    await this.ctx.storage.sql.exec(
+      `UPDATE logs SET is_complete = 1, error = ?1, error_type = ?2 WHERE is_complete = 0`,
+      error,
+      errorType
+    );
+    for (const id of logIds) {
+      this.broadcastLog({
+        type: "log_data",
+        log_id: id,
+        data: { id, is_complete: true, error, error_type: errorType }
+      });
+    }
+  }
+  /**
+   * A subagent whose execution the watchdog gave up on leaves its parent
+   * waiting forever for a completion message. Deliver the failure the same way
+   * a sessionFail would, so the parent's turn wakes and can route around it.
+   * Explicit-communication children (e.g. the compaction agent) stay quiet by
+   * contract — only the registry status updates for those.
+   */
+  async notifyParentOfLostExecution(threadId) {
+    try {
+      const agentBuilderId = this.env.AGENT_BUILDER.idFromName("singleton");
+      const agentBuilder = this.env.AGENT_BUILDER.get(agentBuilderId);
+      const meta = await agentBuilder.getThread(threadId);
+      const parentThreadId = meta?.parent ?? null;
+      if (!parentThreadId) return;
+      const parentId = this.env.AGENT_BUILDER_THREAD.idFromName(parentThreadId);
+      const parentStub = this.env.AGENT_BUILDER_THREAD.get(parentId);
+      const parentCommunication = await this.getParentCommunicationModeForChild(
+        parentStub,
+        parentThreadId,
+        threadId
+      );
+      if (parentCommunication !== "explicit") {
+        await parentStub.queueMessage(parentThreadId, {
+          role: "user",
+          content: `Subagent (reference: ${threadId}) has reported a failure:
+
+Its execution was lost (the executor restarted or hung) and automatic recovery was exhausted. Treat this subagent's task as failed \u2014 do not wait for it; retry the work another way or report the failure.`,
+          attachments: [],
+          silent: true,
+          metadata: { subagent_id: threadId }
+        });
+      }
+      await parentStub.updateChildRegistryStatus(parentThreadId, threadId, "idle");
+    } catch (error) {
+      console.error(
+        "[DurableThread] Watchdog: failed to notify parent of lost execution (non-fatal):",
+        error
+      );
+    }
+  }
+  /** At most N auto-resumes per rolling window, tracked durably. */
+  async consumeWatchdogResumeBudget() {
+    const now = Date.now();
+    let count = 0;
+    let windowStart = now;
+    const row = (await this.ctx.storage.sql.exec(
+      `SELECT value FROM execution_state WHERE key = 'watchdog_resume_state'`
+    )).toArray()[0];
+    if (row) {
+      try {
+        const parsed = JSON.parse(row.value);
+        if (typeof parsed.count === "number" && typeof parsed.windowStart === "number" && now - parsed.windowStart < this.watchdogResumeWindowMs) {
+          count = parsed.count;
+          windowStart = parsed.windowStart;
+        }
+      } catch {
+      }
+    }
+    if (count >= this.watchdogMaxResumesPerWindow) return false;
+    const value = JSON.stringify({ count: count + 1, windowStart });
+    await this.ctx.storage.sql.exec(
+      `INSERT INTO execution_state (key, value) VALUES ('watchdog_resume_state', ?1)
+       ON CONFLICT(key) DO UPDATE SET value = ?1`,
+      value
+    );
+    return true;
+  }
   normalizePlainRecord(value) {
     if (!value || typeof value !== "object" || Array.isArray(value)) {
       return null;
@@ -19586,6 +19883,7 @@ ${resultContent}${attachmentSummary}`;
     await this.assertNotTerminated();
     this.isExecuting = true;
     this.currentAbortController = new AbortController();
+    await this.armExecutionWatchdog(threadId);
     try {
       const agentDef = await this.loadAgent(agentName);
       const packageId = agentDef.__package?.packageId;
@@ -19716,6 +20014,7 @@ ${resultContent}${attachmentSummary}`;
     } finally {
       this.isExecuting = false;
       this.currentAbortController = null;
+      await this.disarmExecutionWatchdog();
     }
   }
   /**
@@ -19762,6 +20061,7 @@ ${resultContent}${attachmentSummary}`;
     }
     this.isExecuting = true;
     this.currentAbortController = new AbortController();
+    await this.armExecutionWatchdog(threadId);
     try {
       const agentBuilderId = this.env.AGENT_BUILDER.idFromName("singleton");
       const agentBuilder = this.env.AGENT_BUILDER.get(agentBuilderId);
@@ -19900,6 +20200,7 @@ ${resultContent}${attachmentSummary}`;
     } finally {
       this.isExecuting = false;
       this.currentAbortController = null;
+      await this.disarmExecutionWatchdog();
     }
   }
   /**
@@ -19910,6 +20211,7 @@ ${resultContent}${attachmentSummary}`;
     await this.assertNotTerminated();
     this.isExecuting = true;
     this.currentAbortController = new AbortController();
+    await this.armExecutionWatchdog(threadId);
     try {
       const agentBuilderId = this.env.AGENT_BUILDER.idFromName("singleton");
       const agentBuilder = this.env.AGENT_BUILDER.get(agentBuilderId);
@@ -20018,6 +20320,7 @@ ${resultContent}${attachmentSummary}`;
     } finally {
       this.isExecuting = false;
       this.currentAbortController = null;
+      await this.disarmExecutionWatchdog();
       const latestQueuedSide = await this.getLatestQueuedMessageSide();
       if (latestQueuedSide) {
         await this.ensureQueuedMessageExecution(threadId, latestQueuedSide);