@standardagents/builder 0.25.5 → 0.25.6

package/dist/index.js

@@ -2394,8 +2394,11 @@ var init_LLMRequest = __esm({
        */
       static async logError(state, error, errorType, modelId, startTime, existingLogId) {
         try {
-          const rawErrorMessage = error instanceof Error ? error.message : String(error);
+          let rawErrorMessage = error instanceof Error ? error.message : String(error);
           const errorStack = error instanceof Error ? error.stack : void 0;
+          if (error instanceof Error && error.name === "AbortError" && /operation was aborted/i.test(rawErrorMessage)) {
+            rawErrorMessage = state.abortController?.signal?.aborted ? "Aborted: the execution this request belonged to was cancelled (user stop, watchdog takeover, or a restart) \u2014 retries and fallbacks of a cancelled execution fail immediately." : "Aborted: the request's connection was cancelled mid-flight (provider disconnect or executor restart).";
+          }
           let errorMessage = rawErrorMessage;
           let parsedProviderBody = null;
           const trimmedRaw = rawErrorMessage.trim();
@@ -23329,6 +23332,38 @@ var AlarmQueue = class {
   async ensureAlarmScheduled() {
     await this.rescheduleAlarm();
   }
+  /**
+   * Recover queue items orphaned in 'processing' by a dead predecessor
+   * instance.
+   *
+   * processNext marks an item 'processing' before executing it. If the DO is
+   * evicted or a deploy restarts it mid-execution, that row used to stay
+   * 'processing' FOREVER — only 'pending' rows are ever picked up again — so
+   * queued subagent executions, queued messages, and scheduled effects that
+   * were mid-flight at a restart were silently lost (a subagent that "just
+   * died half way through" with nothing to revive it).
+   *
+   * MUST ONLY be called from the once-per-instance wake path (DurableThread's
+   * alarmQueueRearmed guard): at that moment this fresh instance is executing
+   * nothing, so ANY 'processing' row provably belongs to a dead predecessor.
+   * Calling it at any other time would re-queue items that are legitimately
+   * mid-execution on this very instance.
+   */
+  async recoverOrphanedProcessing(maxAttempts = 3) {
+    const now = Date.now() * 1e3;
+    await this.sql.exec(
+      `UPDATE alarm_queue
+       SET status = 'failed',
+           error = 'Lost mid-execution (the executor restarted) and retry attempts were exhausted.',
+           completed_at = ?
+       WHERE status = 'processing' AND attempts >= ?`,
+      now,
+      maxAttempts
+    );
+    await this.sql.exec(
+      `UPDATE alarm_queue SET status = 'pending' WHERE status = 'processing'`
+    );
+  }
   // ============================================================
   // Effect-specific Methods
   // ============================================================
@@ -25251,6 +25286,9 @@ var DurableThread = class _DurableThread extends DurableObject {
     }
     this.migratedToVersion = LATEST_SCHEMA_VERSION;
     if (!this.alarmQueueRearmed && typeof this.alarmQueue.ensureAlarmScheduled === "function") {
+      if (typeof this.alarmQueue.recoverOrphanedProcessing === "function") {
+        await this.alarmQueue.recoverOrphanedProcessing();
+      }
       await this.alarmQueue.ensureAlarmScheduled();
       this.alarmQueueRearmed = true;
     }
@@ -27956,14 +27994,14 @@ ${resultContent}${attachmentSummary}`;
         `SELECT id FROM logs WHERE is_complete = 0`
       )).toArray();
       let incomplete = await readIncomplete();
+      const ceiling = resolveWatchdogStaleCeilingMs(this.env);
+      const silenceMs = this.lastProgressAt === 0 ? Number.POSITIVE_INFINITY : Date.now() - this.lastProgressAt;
       if (this.isExecuting) {
         if (incomplete.length === 0) {
           this.watchdogWedgeStrikes = 0;
           await rearm();
           return;
         }
-        const ceiling = resolveWatchdogStaleCeilingMs(this.env);
-        const silenceMs = Date.now() - this.lastProgressAt;
         if (silenceMs <= ceiling) {
           this.watchdogWedgeStrikes = 0;
           await rearm();
@@ -27972,20 +28010,23 @@ ${resultContent}${attachmentSummary}`;
         this.watchdogWedgeStrikes++;
         if (this.watchdogWedgeStrikes < 2) {
           console.warn(
-            `[DurableThread] Watchdog: execution silent for ${Math.round(silenceMs / 1e3)}s (ceiling ${Math.round(ceiling / 1e3)}s) \u2014 aborting in-flight request`
+            `[DurableThread] Watchdog: execution silent for ${Math.round(silenceMs / 1e3)}s (ceiling ${Math.round(ceiling / 1e3)}s) \u2014 observing; takeover next cycle if still silent`
           );
-          try {
-            this.currentAbortController?.abort();
-          } catch {
-          }
           await rearm();
           return;
         }
         console.error(
-          "[DurableThread] Watchdog: wedged execution did not unwind after abort \u2014 forcing takeover"
+          `[DurableThread] Watchdog: wedged execution (${Math.round(silenceMs / 1e3)}s silent across two cycles) \u2014 forcing takeover`
         );
+        try {
+          this.currentAbortController?.abort();
+        } catch {
+        }
         this.isExecuting = false;
         this.currentAbortController = null;
+      } else if (incomplete.length > 0 && silenceMs <= ceiling) {
+        await rearm();
+        return;
       }
       incomplete = await readIncomplete();
       if (this.isExecuting) {
@@ -28013,6 +28054,7 @@ ${resultContent}${attachmentSummary}`;
         console.error(
           `[DurableThread] Watchdog: resume budget exhausted for thread ${threadId} \u2014 leaving thread stopped`
         );
+        await this.notifyParentOfLostExecution(threadId);
       }
       await this.disarmExecutionWatchdog();
     } catch (error) {
@@ -28040,6 +28082,46 @@ ${resultContent}${attachmentSummary}`;
       });
     }
   }
+  /**
+   * A subagent whose execution the watchdog gave up on leaves its parent
+   * waiting forever for a completion message. Deliver the failure the same way
+   * a sessionFail would, so the parent's turn wakes and can route around it.
+   * Explicit-communication children (e.g. the compaction agent) stay quiet by
+   * contract — only the registry status updates for those.
+   */
+  async notifyParentOfLostExecution(threadId) {
+    try {
+      const agentBuilderId = this.env.AGENT_BUILDER.idFromName("singleton");
+      const agentBuilder = this.env.AGENT_BUILDER.get(agentBuilderId);
+      const meta = await agentBuilder.getThread(threadId);
+      const parentThreadId = meta?.parent ?? null;
+      if (!parentThreadId) return;
+      const parentId = this.env.AGENT_BUILDER_THREAD.idFromName(parentThreadId);
+      const parentStub = this.env.AGENT_BUILDER_THREAD.get(parentId);
+      const parentCommunication = await this.getParentCommunicationModeForChild(
+        parentStub,
+        parentThreadId,
+        threadId
+      );
+      if (parentCommunication !== "explicit") {
+        await parentStub.queueMessage(parentThreadId, {
+          role: "user",
+          content: `Subagent (reference: ${threadId}) has reported a failure:
+
+Its execution was lost (the executor restarted or hung) and automatic recovery was exhausted. Treat this subagent's task as failed \u2014 do not wait for it; retry the work another way or report the failure.`,
+          attachments: [],
+          silent: true,
+          metadata: { subagent_id: threadId }
+        });
+      }
+      await parentStub.updateChildRegistryStatus(parentThreadId, threadId, "idle");
+    } catch (error) {
+      console.error(
+        "[DurableThread] Watchdog: failed to notify parent of lost execution (non-fatal):",
+        error
+      );
+    }
+  }
   /** At most N auto-resumes per rolling window, tracked durably. */
   async consumeWatchdogResumeBudget() {
     const now = Date.now();