@agent-native/core 0.131.4 → 0.131.6

package/dist/agent/run-manager.js

@@ -64,15 +64,6 @@ export const BACKGROUND_SOFT_TIMEOUT_CEILING_MS = 13 * 60_000; // 780_000
  * off to a chained background continuation.
  */
 export const DEFAULT_BACKGROUND_RUN_SOFT_TIMEOUT_MS = BACKGROUND_SOFT_TIMEOUT_CEILING_MS;
-/**
- * Default no-progress window for a run executing inside a proven durable
- * background function. Keep this below the 13-minute soft timeout so a truly
- * wedged background turn can still checkpoint, persist, and continue before
- * the function budget expires, but far above the foreground 150s window so
- * large Design/Plan/Assets generations are not chopped up while the model is
- * legitimately planning a big tool payload.
- */
-export const DEFAULT_BACKGROUND_NO_PROGRESS_TIMEOUT_MS = BACKGROUND_SOFT_TIMEOUT_CEILING_MS - 60_000;
 /**
  * AUTHORITATIVE no-progress backstop for a run, enforced by the run manager
  * itself (timer-driven, independent of any layer below).
@@ -101,6 +92,15 @@ export const DEFAULT_BACKGROUND_NO_PROGRESS_TIMEOUT_MS = BACKGROUND_SOFT_TIMEOUT
  * runs); local dev stays unbounded.
  */
 export const RUN_NO_PROGRESS_HARD_TIMEOUT_MS = 150_000;
+/**
+ * Default no-progress window for a run executing inside a proven durable
+ * background function. A background worker that is heartbeating but has no
+ * real progress and no tool/A2A work in flight is still wedged; letting that
+ * state outlive the client's follow budget only turns a recoverable stall into
+ * a terminal timeout. In-flight work suspends this backstop, so use the same
+ * server-owned 150s bound as the long foreground regime.
+ */
+export const DEFAULT_BACKGROUND_NO_PROGRESS_TIMEOUT_MS = RUN_NO_PROGRESS_HARD_TIMEOUT_MS;
 /**
  * Fraction of the soft timeout a foreground no-progress backstop may consume.
  * The hosted foreground path rides a synchronous serverless function whose
@@ -196,12 +196,23 @@ export const SQL_SUBSCRIPTION_IDLE_POLL_MS = 500;
 export const SQL_SUBSCRIPTION_ACTIVE_GRACE_MS = 2_000;
 /** Keep terminal/status probes at the historical cadence to bound DB work. */
 export const SQL_SUBSCRIPTION_STATUS_POLL_MS = 500;
+/** Initial retry delay after a transient cross-isolate SQL polling failure. */
+export const SQL_SUBSCRIPTION_RETRY_BASE_MS = 250;
+/** Bound consecutive SQL polling failures so a dead subscription fails loud. */
+export const SQL_SUBSCRIPTION_MAX_CONSECUTIVE_FAILURES = 4;
+/** Cap retry delay even if the failure bound changes independently. */
+export const SQL_SUBSCRIPTION_RETRY_MAX_MS = 2_000;
 export function resolveSqlSubscriptionPollMs(now, activePollUntil) {
     return now < activePollUntil
         ? SQL_SUBSCRIPTION_ACTIVE_POLL_MS
         : SQL_SUBSCRIPTION_IDLE_POLL_MS;
 }
+export function resolveSqlSubscriptionRetryMs(consecutiveFailures) {
+    const retryIndex = Math.max(0, Math.floor(consecutiveFailures) - 1);
+    return Math.min(SQL_SUBSCRIPTION_RETRY_MAX_MS, SQL_SUBSCRIPTION_RETRY_BASE_MS * 2 ** retryIndex);
+}
 const PROVIDER_RATE_LIMITED_ERROR_CODE = "provider_rate_limited";
+const PROVIDER_NETWORK_ERROR_CODE = "provider_network_error";
 function isPreparingActionActivityEvent(event) {
     if (event.type !== "activity")
         return false;
@@ -218,11 +229,21 @@ function getRunErrorMessage(err) {
     }
     return "Unknown error";
 }
-function getEngineRunErrorCode(err) {
-    if (err.errorCode)
-        return err.errorCode;
-    if (err.statusCode === 429)
-        return PROVIDER_RATE_LIMITED_ERROR_CODE;
+function isProviderConnectionErrorMessage(message) {
+    const normalized = message.trim().toLowerCase();
+    return (normalized === "connection error." ||
+        normalized.includes("cannot connect to api:"));
+}
+function getRunErrorCode(err) {
+    if (err instanceof EngineError) {
+        if (err.errorCode)
+            return err.errorCode;
+        if (err.statusCode === 429)
+            return PROVIDER_RATE_LIMITED_ERROR_CODE;
+    }
+    if (err instanceof Error && isProviderConnectionErrorMessage(err.message)) {
+        return PROVIDER_NETWORK_ERROR_CODE;
+    }
     return undefined;
 }
 function getEngineRunErrorDetails(err) {
@@ -231,22 +252,25 @@ function getEngineRunErrorDetails(err) {
     return undefined;
 }
 function shouldCaptureRunError(err) {
-    if (!(err instanceof EngineError))
-        return true;
-    const errorCode = getEngineRunErrorCode(err);
+    const errorCode = getRunErrorCode(err);
     if (isLlmCredentialError(err, errorCode))
         return false;
-    if (err.statusCode === 401 || err.statusCode === 403)
+    if (err instanceof EngineError &&
+        (err.statusCode === 401 || err.statusCode === 403)) {
         return false;
+    }
+    if (!(err instanceof Error))
+        return true;
     if (/^40[13] status code\b/i.test(err.message))
         return false;
-    if (err.message.trim().toLowerCase() === "connection error.")
+    if (isProviderConnectionErrorMessage(err.message))
         return false;
     if (!errorCode)
         return true;
     const normalizedCode = errorCode.toLowerCase();
     return (!normalizedCode.startsWith("credits-limit") &&
         normalizedCode !== "builder_gateway_network_error" &&
+        normalizedCode !== PROVIDER_NETWORK_ERROR_CODE &&
         normalizedCode !== "provider_rate_limited" &&
         normalizedCode !== "rate_limit_exceeded");
 }
@@ -792,7 +816,7 @@ export function startRun(runId, threadId, runFn, onComplete, options) {
         : null;
     let pendingTerminalEvent = null;
     const captureRunError = (error, phase) => {
-        const errorCode = error instanceof EngineError ? getEngineRunErrorCode(error) : undefined;
+        const errorCode = getRunErrorCode(error);
         captureError(error, {
             route: "/_agent-native/agent-chat",
             tags: {
@@ -899,7 +923,7 @@ export function startRun(runId, threadId, runFn, onComplete, options) {
             captureRunError(err, "run");
         }
         const errorMessage = getRunErrorMessage(err);
-        const errorCode = err instanceof EngineError ? getEngineRunErrorCode(err) : undefined;
+        const errorCode = getRunErrorCode(err);
         const details = err instanceof EngineError ? getEngineRunErrorDetails(err) : undefined;
         send({
             type: "error",
@@ -1266,6 +1290,7 @@ function subscribeFromSQL(runId, fromSeq) {
             let lastSeq = fromSeq;
             let activePollUntil = 0;
             let lastStatusCheckAt = 0;
+            let consecutivePollFailures = 0;
             const ping = () => {
                 try {
                     controller.enqueue(encoder.encode(`: ping ${Date.now()}\n\n`));
@@ -1278,6 +1303,45 @@ function subscribeFromSQL(runId, fromSeq) {
             };
             ping();
             pingTimer = setInterval(ping, 10_000);
+            const closeStream = () => {
+                cancelled = true;
+                if (pollTimer)
+                    clearTimeout(pollTimer);
+                if (pingTimer)
+                    clearInterval(pingTimer);
+                try {
+                    controller.close();
+                }
+                catch { }
+            };
+            const failSubscription = (error) => {
+                const event = {
+                    type: "error",
+                    error: "The live agent connection could not load persisted run progress.",
+                    errorCode: "run_subscription_poll_failed",
+                    details: "The agent may still be running. Reconnect to resume from the last persisted event.",
+                    recoverable: true,
+                };
+                captureError(error, {
+                    route: "/_agent-native/agent-chat/runs/:id/events",
+                    tags: {
+                        source: "agent-run-manager",
+                        phase: "sql-subscription-poll",
+                        consecutiveFailures: String(consecutivePollFailures),
+                    },
+                    extra: {
+                        runId,
+                        fromSeq,
+                        lastSeq,
+                        error: getRunErrorMessage(error),
+                    },
+                });
+                try {
+                    controller.enqueue(encoder.encode(`data: ${JSON.stringify({ ...event, seq: lastSeq })}\n\n`));
+                }
+                catch { }
+                closeStream();
+            };
             const poll = async () => {
                 if (cancelled)
                     return;
@@ -1320,6 +1384,7 @@ function subscribeFromSQL(runId, fromSeq) {
                         const now = Date.now();
                         if (now - lastStatusCheckAt < SQL_SUBSCRIPTION_STATUS_POLL_MS) {
                             if (!cancelled) {
+                                consecutivePollFailures = 0;
                                 const pollMs = resolveSqlSubscriptionPollMs(now, activePollUntil);
                                 pollTimer = setTimeout(poll, pollMs);
                             }
@@ -1443,34 +1508,20 @@ function subscribeFromSQL(runId, fromSeq) {
                     }
                     // Schedule next poll
                     if (!cancelled) {
+                        consecutivePollFailures = 0;
                         const pollMs = resolveSqlSubscriptionPollMs(Date.now(), activePollUntil);
                         pollTimer = setTimeout(poll, pollMs);
                     }
                 }
-                catch {
-                    // SQL error — close stream
-                    try {
-                        if (pingTimer)
-                            clearInterval(pingTimer);
-                        controller.close();
+                catch (error) {
+                    consecutivePollFailures += 1;
+                    if (consecutivePollFailures >= SQL_SUBSCRIPTION_MAX_CONSECUTIVE_FAILURES) {
+                        failSubscription(error);
+                        return;
                     }
-                    catch { }
+                    pollTimer = setTimeout(poll, resolveSqlSubscriptionRetryMs(consecutivePollFailures));
                 }
             };
-            // Verify run exists before starting poll
-            try {
-                const run = await getRunById(runId);
-                if (!run) {
-                    if (pingTimer)
-                        clearInterval(pingTimer);
-                    controller.close();
-                    return;
-                }
-            }
-            catch {
-                controller.close();
-                return;
-            }
             await poll();
         },
         cancel() {