@standardagents/builder 0.25.3 → 0.25.5

package/dist/runtime.d.ts

@@ -1,5 +1,5 @@
-import { e as ThreadEnv, A as Agent, h as ThreadMetadata, i as ToolResult, M as Message, l as FileRecord, F as FlowState, n as FileStats, G as GrepResult, m as AttachmentRef, r as MessageContent, E as Env, g as ThreadInstance } from './index-BsS0cHzL.js';
-export { f as BuilderThreadEndpointHandler, B as Controller, a as ControllerContext, j as FlowResult, p as ImageContentPart, I as ImageMetadata, L as LLMResponse, q as MultimodalContent, R as RequestContext, S as StorageBackend, k as TelemetryEvent, o as TextContentPart, b as ThreadEndpointContext, T as ToolCall, c as createThreadEndpointHandler, d as defineController } from './index-BsS0cHzL.js';
+import { e as ThreadEnv, A as Agent, h as ThreadMetadata, i as ToolResult, M as Message, l as FileRecord, F as FlowState, n as FileStats, G as GrepResult, m as AttachmentRef, r as MessageContent, E as Env, g as ThreadInstance } from './index-CX9Ti98s.js';
+export { f as BuilderThreadEndpointHandler, B as Controller, a as ControllerContext, j as FlowResult, p as ImageContentPart, I as ImageMetadata, L as LLMResponse, q as MultimodalContent, R as RequestContext, S as StorageBackend, k as TelemetryEvent, o as TextContentPart, b as ThreadEndpointContext, T as ToolCall, c as createThreadEndpointHandler, d as defineController } from './index-CX9Ti98s.js';
 import { CodeExecutionOptions, CodeExecutionResult, SubagentRegistryEntry, InjectMessageInput, QueueMessageInput, ClientToolRequest, ClientToolResponse, SkillMetadata, SkillFileInput, ModelDefinition as ModelDefinition$1, ToolArgs, PromptTextPart, PromptEnvPart, VariableDefinition, SubpromptConfig as SubpromptConfig$1, PromptToolConfig as PromptToolConfig$1, SubagentToolConfig as SubagentToolConfig$1, ReasoningConfig, SideConfig as SideConfig$1, LLMProviderInterface, ContentPart, TextPart, ImagePart, FilePart, NamespaceContext, DefinitionLoader } from '@standardagents/spec';
 export { AgentType, DefinitionLoader, GlobalNamespaceContext, HookSignatures, ImageContent, ModelCapabilities, ModelProvider, NamespaceContext, PackageSignature, PackedExports, PackedMeta, PackedMetadata, PackedNamespaceContext, PromptInput, PromptTextPart, ProviderAssistantMessage, ProviderError, ProviderErrorCode, ProviderFactory, ProviderFactoryConfig, ProviderFinishReason, ProviderGeneratedImage, ProviderMessage, ProviderMessageContent, ModelCapabilities as ProviderModelCapabilities, ProviderReasoningDetail, ProviderRequest, ProviderResponse, ProviderStreamChunk, ProviderSystemMessage, ProviderTool, ProviderToolCallPart, ProviderToolMessage, ProviderToolResultContent, ProviderUsage, ProviderUserMessage, ReasoningConfig, StructuredPrompt, TextContent, Tool, ToolArgs, ToolArgsNode, ToolArgsRawShape, ToolContent, belongsToPackage, defineAgent, defineHook, defineModel, definePrompt, defineTool, isPacked, isVisibleInNamespace, mapReasoningLevel } from '@standardagents/spec';
 import { DurableObject, WorkerEntrypoint } from 'cloudflare:workers';
@@ -184,6 +184,11 @@ declare class DurableThread<Env extends ThreadEnv = ThreadEnv> extends DurableOb
     private alarmQueueRearmed;
     private currentAbortController;
     private isExecuting;
+    private lastProgressAt;
+    private watchdogWedgeStrikes;
+    private readonly watchdogIntervalMs;
+    private readonly watchdogResumeWindowMs;
+    private readonly watchdogMaxResumesPerWindow;
     private codeExecutionBridges;
     constructor(ctx: DurableObjectState, env: Env);
     /**
@@ -1080,6 +1085,21 @@ declare class DurableThread<Env extends ThreadEnv = ThreadEnv> extends DurableOb
      * the alarm chain. All errors are caught and logged.
      */
     alarm(): Promise<void>;
+    /** Heartbeat: called by LLMRequest on each attempt and streamed chunk. */
+    noteExecutionProgress(): void;
+    /** Arm the standing watchdog for an execution start. One pending row max. */
+    private armExecutionWatchdog;
+    /** Disarm on clean execution end. Completed check rows are noise, not history. */
+    private disarmExecutionWatchdog;
+    /**
+     * The standing check. Evaluates the thread's state AT FIRE TIME — it knows
+     * nothing about whichever request was in flight when it was armed.
+     */
+    private watchdogCheck;
+    /** Mark logs the dead executor abandoned, and tell connected clients. */
+    private finalizeLostExecutionLogs;
+    /** At most N auto-resumes per rolling window, tracked durably. */
+    private consumeWatchdogResumeBudget;
     private normalizePlainRecord;
     private buildExecutionArguments;
     /**

package/dist/runtime.js

@@ -1057,6 +1057,19 @@ function resolveStreamTimeoutMs(env, key, fallback) {
   const parsed = typeof raw === "number" ? raw : typeof raw === "string" ? Number.parseInt(raw, 10) : NaN;
   return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
 }
+function resolveWatchdogStaleCeilingMs(env) {
+  const firstChunk = resolveStreamTimeoutMs(
+    env,
+    "STANDARD_AGENTS_STREAM_FIRST_CHUNK_TIMEOUT_MS",
+    STREAM_FIRST_CHUNK_TIMEOUT_MS
+  );
+  const stall = resolveStreamTimeoutMs(
+    env,
+    "STANDARD_AGENTS_STREAM_STALL_TIMEOUT_MS",
+    STREAM_STALL_TIMEOUT_MS
+  );
+  return Math.max(firstChunk, stall) + 6e4;
+}
 function modelSupportsImageInput(modelDef) {
   const supportsImages = modelDef.capabilities?.supportsImages;
   if (typeof supportsImages === "boolean") {
@@ -1569,6 +1582,7 @@ var init_LLMRequest = __esm({
           attempts++;
           try {
             logId = await this.logRequest(state, actualModelId, context, modelDef, previousLogId);
+            state.thread.instance.noteExecutionProgress?.();
             state.emitTelemetry?.({
               type: "llm_request",
               model: actualModelId,
@@ -1835,6 +1849,7 @@ var init_LLMRequest = __esm({
                 }
                 if (next.done) break;
                 const chunk = next.value;
+                state.thread.instance.noteExecutionProgress?.();
                 if (firstChunkAt === 0) {
                   firstChunkAt = Date.now();
                   emitLogPatch({ time_to_first_token_ms: firstChunkAt - requestSentAt });
@@ -15287,6 +15302,7 @@ var AlarmQueue = class {
 
 // src/durable-objects/DurableThread.ts
 init_StreamManager();
+init_LLMRequest();
 
 // src/packing/resolver.ts
 var NamespaceResolutionError = class extends Error {
@@ -15774,6 +15790,18 @@ var DurableThread = class _DurableThread extends DurableObject {
   alarmQueueRearmed = false;
   currentAbortController = null;
   isExecuting = false;
+  // ── Execution watchdog state (see watchdogCheck) ─────────────────────────
+  // In-memory heartbeat: bumped by noteExecutionProgress() on every streamed
+  // chunk and request attempt. Deliberately NOT persisted — after a DO restart
+  // the isExecuting flag alone proves the executor is gone, and while the DO
+  // is alive the in-memory value is authoritative. Zero hot-path storage cost.
+  lastProgressAt = 0;
+  // Consecutive watchdog firings that found a live-but-silent executor. Strike
+  // one aborts the in-flight request; strike two forces takeover.
+  watchdogWedgeStrikes = 0;
+  watchdogIntervalMs = 3e4;
+  watchdogResumeWindowMs = 15 * 60 * 1e3;
+  watchdogMaxResumesPerWindow = 2;
   codeExecutionBridges = /* @__PURE__ */ new Map();
   constructor(ctx, env) {
     super(ctx, env);
@@ -15809,6 +15837,9 @@ var DurableThread = class _DurableThread extends DurableObject {
           case "continueExecution":
             await this.doContinueExecution(args.threadId, args.side);
             break;
+          case "watchdogCheck":
+            await this.watchdogCheck(args.threadId);
+            break;
           default:
             throw new Error(`Unknown method: ${method}`);
         }
@@ -19561,6 +19592,190 @@ ${resultContent}${attachmentSummary}`;
       console.error("[DurableThread] Alarm handler failed:", error);
     }
   }
+  // ══════════════════════════════════════════════════════════════════════════
+  // Execution watchdog
+  //
+  // The in-flight stream guards (LLMRequest's first-chunk/stall timeouts) live
+  // inside the execution's promise chain — they die with the DO. A deploy or
+  // eviction mid-request therefore used to leave a durable log row stuck at
+  // is_complete = 0 forever: the thread looked "working", queued messages
+  // never played, and nothing woke it (finalizeOrphanedLogs only runs at the
+  // START of the next execution). Persisted DO alarms are the one timer that
+  // survives restarts and fires without inbound traffic, so a standing
+  // per-thread watchdog alarm re-checks the thread's CURRENT state every 30s
+  // while work is active.
+  //
+  // Design invariants (do not weaken):
+  // - Progress, never duration: only silence ages. A response can stream for
+  //   ten minutes; every chunk bumps the in-memory heartbeat.
+  // - Per-thread, not per-request: the check carries no request identity. By
+  //   the time it fires the thread may be several requests further along —
+  //   it always evaluates fresh SQL + live flags, never a stale snapshot.
+  // - Strictly second line: the staleness ceiling is DERIVED from the stream
+  //   timeouts (resolveWatchdogStaleCeilingMs = max(first-chunk, stall) + 60s)
+  //   so the in-flight guards always get to act first. The watchdog only
+  //   moves when the guard that should have fired didn't.
+  // - Restart proof: isExecuting is volatile; an incomplete log with
+  //   isExecuting === false is PROOF the executor died (executions cannot
+  //   survive a restart). No heartbeat needed for that branch.
+  // - Nonce re-read: every await inside the check is a yield point; state is
+  //   re-read immediately before acting so a healthy thread racing the
+  //   watchdog always wins.
+  // ══════════════════════════════════════════════════════════════════════════
+  /** Heartbeat: called by LLMRequest on each attempt and streamed chunk. */
+  noteExecutionProgress() {
+    this.lastProgressAt = Date.now();
+  }
+  /** Arm the standing watchdog for an execution start. One pending row max. */
+  async armExecutionWatchdog(threadId) {
+    try {
+      this.lastProgressAt = Date.now();
+      this.watchdogWedgeStrikes = 0;
+      const pending = (await this.ctx.storage.sql.exec(
+        `SELECT id FROM alarm_queue WHERE method = 'watchdogCheck' AND status = 'pending' LIMIT 1`
+      )).toArray();
+      if (pending.length > 0) return;
+      await this.alarmQueue.enqueue("watchdogCheck", { threadId }, this.watchdogIntervalMs);
+    } catch (error) {
+      console.error("[DurableThread] Failed to arm execution watchdog (non-fatal):", error);
+    }
+  }
+  /** Disarm on clean execution end. Completed check rows are noise, not history. */
+  async disarmExecutionWatchdog() {
+    try {
+      await this.ctx.storage.sql.exec(
+        `DELETE FROM alarm_queue WHERE method = 'watchdogCheck' AND status IN ('pending', 'completed')`
+      );
+      await this.alarmQueue.ensureAlarmScheduled();
+    } catch (error) {
+      console.error("[DurableThread] Failed to disarm execution watchdog (non-fatal):", error);
+    }
+  }
+  /**
+   * The standing check. Evaluates the thread's state AT FIRE TIME — it knows
+   * nothing about whichever request was in flight when it was armed.
+   */
+  async watchdogCheck(threadId) {
+    const rearm = () => this.alarmQueue.enqueue("watchdogCheck", { threadId }, this.watchdogIntervalMs);
+    try {
+      await this.ctx.storage.sql.exec(
+        `DELETE FROM alarm_queue WHERE method = 'watchdogCheck' AND status = 'completed'`
+      );
+      const readIncomplete = async () => (await this.ctx.storage.sql.exec(
+        `SELECT id FROM logs WHERE is_complete = 0`
+      )).toArray();
+      let incomplete = await readIncomplete();
+      if (this.isExecuting) {
+        if (incomplete.length === 0) {
+          this.watchdogWedgeStrikes = 0;
+          await rearm();
+          return;
+        }
+        const ceiling = resolveWatchdogStaleCeilingMs(this.env);
+        const silenceMs = Date.now() - this.lastProgressAt;
+        if (silenceMs <= ceiling) {
+          this.watchdogWedgeStrikes = 0;
+          await rearm();
+          return;
+        }
+        this.watchdogWedgeStrikes++;
+        if (this.watchdogWedgeStrikes < 2) {
+          console.warn(
+            `[DurableThread] Watchdog: execution silent for ${Math.round(silenceMs / 1e3)}s (ceiling ${Math.round(ceiling / 1e3)}s) \u2014 aborting in-flight request`
+          );
+          try {
+            this.currentAbortController?.abort();
+          } catch {
+          }
+          await rearm();
+          return;
+        }
+        console.error(
+          "[DurableThread] Watchdog: wedged execution did not unwind after abort \u2014 forcing takeover"
+        );
+        this.isExecuting = false;
+        this.currentAbortController = null;
+      }
+      incomplete = await readIncomplete();
+      if (this.isExecuting) {
+        await rearm();
+        return;
+      }
+      if (incomplete.length > 0) {
+        await this.finalizeLostExecutionLogs(incomplete.map((row) => row.id));
+      }
+      const stopped = await this.shouldStop();
+      const queued = (await this.ctx.storage.sql.exec(
+        `SELECT COUNT(*) AS c FROM message_queue`
+      )).toArray()[0]?.c ?? 0;
+      if (stopped || incomplete.length === 0 && queued === 0) {
+        await this.disarmExecutionWatchdog();
+        return;
+      }
+      if (await this.consumeWatchdogResumeBudget()) {
+        const side = await this.getLatestQueuedMessageSide() ?? "a";
+        await this.alarmQueue.enqueue("continueExecution", { threadId, side }, 0);
+        console.warn(
+          `[DurableThread] Watchdog: lost execution recovered \u2014 resuming thread ${threadId}`
+        );
+      } else {
+        console.error(
+          `[DurableThread] Watchdog: resume budget exhausted for thread ${threadId} \u2014 leaving thread stopped`
+        );
+      }
+      await this.disarmExecutionWatchdog();
+    } catch (error) {
+      console.error("[DurableThread] Watchdog check failed (non-fatal):", error);
+      try {
+        await rearm();
+      } catch {
+      }
+    }
+  }
+  /** Mark logs the dead executor abandoned, and tell connected clients. */
+  async finalizeLostExecutionLogs(logIds) {
+    const error = "Request lost \u2014 the executor restarted (deploy or eviction) or hung before finishing. The thread watchdog finalized it.";
+    const errorType = "execution_lost";
+    await this.ctx.storage.sql.exec(
+      `UPDATE logs SET is_complete = 1, error = ?1, error_type = ?2 WHERE is_complete = 0`,
+      error,
+      errorType
+    );
+    for (const id of logIds) {
+      this.broadcastLog({
+        type: "log_data",
+        log_id: id,
+        data: { id, is_complete: true, error, error_type: errorType }
+      });
+    }
+  }
+  /** At most N auto-resumes per rolling window, tracked durably. */
+  async consumeWatchdogResumeBudget() {
+    const now = Date.now();
+    let count = 0;
+    let windowStart = now;
+    const row = (await this.ctx.storage.sql.exec(
+      `SELECT value FROM execution_state WHERE key = 'watchdog_resume_state'`
+    )).toArray()[0];
+    if (row) {
+      try {
+        const parsed = JSON.parse(row.value);
+        if (typeof parsed.count === "number" && typeof parsed.windowStart === "number" && now - parsed.windowStart < this.watchdogResumeWindowMs) {
+          count = parsed.count;
+          windowStart = parsed.windowStart;
+        }
+      } catch {
+      }
+    }
+    if (count >= this.watchdogMaxResumesPerWindow) return false;
+    const value = JSON.stringify({ count: count + 1, windowStart });
+    await this.ctx.storage.sql.exec(
+      `INSERT INTO execution_state (key, value) VALUES ('watchdog_resume_state', ?1)
+       ON CONFLICT(key) DO UPDATE SET value = ?1`,
+      value
+    );
+    return true;
+  }
   normalizePlainRecord(value) {
     if (!value || typeof value !== "object" || Array.isArray(value)) {
       return null;
@@ -19586,6 +19801,7 @@ ${resultContent}${attachmentSummary}`;
     await this.assertNotTerminated();
     this.isExecuting = true;
     this.currentAbortController = new AbortController();
+    await this.armExecutionWatchdog(threadId);
     try {
       const agentDef = await this.loadAgent(agentName);
       const packageId = agentDef.__package?.packageId;
@@ -19716,6 +19932,7 @@ ${resultContent}${attachmentSummary}`;
     } finally {
       this.isExecuting = false;
       this.currentAbortController = null;
+      await this.disarmExecutionWatchdog();
     }
   }
   /**
@@ -19762,6 +19979,7 @@ ${resultContent}${attachmentSummary}`;
     }
     this.isExecuting = true;
     this.currentAbortController = new AbortController();
+    await this.armExecutionWatchdog(threadId);
     try {
       const agentBuilderId = this.env.AGENT_BUILDER.idFromName("singleton");
       const agentBuilder = this.env.AGENT_BUILDER.get(agentBuilderId);
@@ -19900,6 +20118,7 @@ ${resultContent}${attachmentSummary}`;
     } finally {
       this.isExecuting = false;
       this.currentAbortController = null;
+      await this.disarmExecutionWatchdog();
     }
   }
   /**
@@ -19910,6 +20129,7 @@ ${resultContent}${attachmentSummary}`;
     await this.assertNotTerminated();
     this.isExecuting = true;
     this.currentAbortController = new AbortController();
+    await this.armExecutionWatchdog(threadId);
     try {
       const agentBuilderId = this.env.AGENT_BUILDER.idFromName("singleton");
       const agentBuilder = this.env.AGENT_BUILDER.get(agentBuilderId);
@@ -20018,6 +20238,7 @@ ${resultContent}${attachmentSummary}`;
     } finally {
       this.isExecuting = false;
       this.currentAbortController = null;
+      await this.disarmExecutionWatchdog();
       const latestQueuedSide = await this.getLatestQueuedMessageSide();
       if (latestQueuedSide) {
         await this.ensureQueuedMessageExecution(threadId, latestQueuedSide);