@poncho-ai/cli 0.30.1 → 0.30.2

package/.turbo/turbo-build.log

@@ -1,5 +1,5 @@
 
-> @poncho-ai/cli@0.30.1 build /home/runner/work/poncho-ai/poncho-ai/packages/cli
+> @poncho-ai/cli@0.30.2 build /home/runner/work/poncho-ai/poncho-ai/packages/cli
 > tsup src/index.ts src/cli.ts --format esm --dts
 
 CLI Building entry: src/cli.ts, src/index.ts
@@ -8,11 +8,11 @@
 CLI Target: es2022
 ESM Build start
 ESM dist/cli.js                          94.00 B
+ESM dist/run-interactive-ink-FUMHN6DS.js 56.86 KB
 ESM dist/index.js                        857.00 B
-ESM dist/run-interactive-ink-ZSIGWFLZ.js 56.86 KB
-ESM dist/chunk-UYZOJWGL.js               481.08 KB
-ESM ⚡️ Build success in 64ms
+ESM dist/chunk-FA546WPW.js               490.84 KB
+ESM ⚡️ Build success in 71ms
 DTS Build start
-DTS ⚡️ Build success in 3869ms
+DTS ⚡️ Build success in 4822ms
 DTS dist/cli.d.ts   20.00 B
 DTS dist/index.d.ts 4.16 KB

package/CHANGELOG.md

@@ -1,5 +1,20 @@
 # @poncho-ai/cli
 
+## 0.30.2
+
+### Patch Changes
+
+- [`98df42f`](https://github.com/cesr/poncho-ai/commit/98df42f79e0a376d0a864598557758bfa644039d) Thanks [@cesr](https://github.com/cesr)! - Fix serverless subagent and continuation reliability
+  - Use stable internal secret across serverless instances for callback auth
+  - Wrap continuation self-fetches in waitUntil to survive function shutdown
+  - Set runStatus during callback re-runs so clients detect active processing
+  - Add post-streaming soft deadline check to catch long model responses
+  - Client auto-recovers from abrupt stream termination and orphaned continuations
+  - Fix callback continuation losing \_continuationMessages when no pending results
+
+- Updated dependencies [[`98df42f`](https://github.com/cesr/poncho-ai/commit/98df42f79e0a376d0a864598557758bfa644039d)]:
+  - @poncho-ai/harness@0.28.2
+
 ## 0.30.1
 
 ### Patch Changes

package/dist/{chunk-UYZOJWGL.js → chunk-FA546WPW.js}

@@ -3173,6 +3173,101 @@ var getWebUiClientScript = (markedSource2) => `
           });
         } else if (willStream) {
           setStreaming(true);
+        } else if (payload.needsContinuation && !payload.conversation.parentConversationId) {
+          console.log("[poncho] Detected orphaned continuation for", conversationId, "\u2014 auto-resuming");
+          (async () => {
+            try {
+              setStreaming(true);
+              var localMsgs = state.activeMessages || [];
+              var contAssistant = {
+                role: "assistant",
+                content: "",
+                _sections: [],
+                _currentText: "",
+                _currentTools: [],
+                _toolImages: [],
+                _activeActivities: [],
+                _pendingApprovals: [],
+                metadata: { toolActivity: [] }
+              };
+              localMsgs.push(contAssistant);
+              state.activeMessages = localMsgs;
+              state._activeStreamMessages = localMsgs;
+              renderMessages(localMsgs, true);
+              var contResp = await fetch(
+                "/api/conversations/" + encodeURIComponent(conversationId) + "/messages",
+                {
+                  method: "POST",
+                  credentials: "include",
+                  headers: { "Content-Type": "application/json", "x-csrf-token": state.csrfToken },
+                  body: JSON.stringify({ continuation: true }),
+                },
+              );
+              if (!contResp.ok || !contResp.body) {
+                contAssistant._error = "Failed to resume \u2014 reload to retry";
+                setStreaming(false);
+                renderMessages(localMsgs, false);
+                return;
+              }
+              state.activeStreamConversationId = conversationId;
+              var contReader = contResp.body.getReader();
+              var contDecoder = new TextDecoder();
+              var contBuffer = "";
+              while (true) {
+                var chunk = await contReader.read();
+                if (chunk.done) break;
+                contBuffer += contDecoder.decode(chunk.value, { stream: true });
+                contBuffer = parseSseChunk(contBuffer, function(evtName, evtPayload) {
+                  if (evtName === "model:chunk" && evtPayload.content) {
+                    contAssistant.content = (contAssistant.content || "") + evtPayload.content;
+                    contAssistant._currentText += evtPayload.content;
+                  }
+                  if (evtName === "tool:started") {
+                    if (contAssistant._currentText) {
+                      contAssistant._sections.push({ type: "text", content: contAssistant._currentText });
+                      contAssistant._currentText = "";
+                    }
+                    contAssistant._currentTools.push("- start \`" + evtPayload.tool + "\`");
+                  }
+                  if (evtName === "tool:completed") {
+                    contAssistant._currentTools.push("- done \`" + evtPayload.tool + "\` (" + evtPayload.duration + "ms)");
+                  }
+                  if (evtName === "tool:error") {
+                    contAssistant._currentTools.push("- error \`" + evtPayload.tool + "\`: " + evtPayload.error);
+                  }
+                  if (evtName === "run:completed" || evtName === "run:error" || evtName === "run:cancelled") {
+                    if (contAssistant._currentTools.length > 0) {
+                      contAssistant._sections.push({ type: "tools", content: contAssistant._currentTools });
+                      contAssistant._currentTools = [];
+                    }
+                    if (contAssistant._currentText) {
+                      contAssistant._sections.push({ type: "text", content: contAssistant._currentText });
+                      contAssistant._currentText = "";
+                    }
+                    contAssistant._activeActivities = [];
+                    if (evtName === "run:error") {
+                      contAssistant._error = evtPayload.error?.message || "Something went wrong";
+                    }
+                    if (evtName === "run:completed" && evtPayload.result?.continuation === true) {
+                      // Another continuation needed \u2014 reload to pick it up
+                      loadConversation(conversationId).catch(function() {});
+                    }
+                  }
+                  renderMessages(localMsgs, true);
+                });
+              }
+              setStreaming(false);
+              renderMessages(localMsgs, false);
+              await loadConversations();
+            } catch (contErr) {
+              console.error("[poncho] Auto-continuation failed:", contErr);
+              setStreaming(false);
+              await loadConversation(conversationId).catch(function() {});
+            } finally {
+              state.activeStreamConversationId = null;
+              state._activeStreamMessages = null;
+            }
+          })();
         }
       };
 
@@ -4256,6 +4351,7 @@ var getWebUiClientScript = (markedSource2) => `
           let _totalSteps = 0;
           let _maxSteps = 0;
           let _isContinuation = false;
+          let _receivedTerminalEvent = false;
           while (true) {
           let _shouldContinue = false;
           let fetchOpts;
@@ -4585,6 +4681,7 @@ var getWebUiClientScript = (markedSource2) => `
                   }
                 }
                 if (eventName === "run:completed") {
+                  _receivedTerminalEvent = true;
                   _totalSteps += typeof payload.result?.steps === "number" ? payload.result.steps : 0;
                   if (typeof payload.result?.maxSteps === "number") _maxSteps = payload.result.maxSteps;
                   if (payload.result?.continuation === true && (_maxSteps <= 0 || _totalSteps < _maxSteps)) {
@@ -4601,10 +4698,12 @@ var getWebUiClientScript = (markedSource2) => `
                   }
                 }
                 if (eventName === "run:cancelled") {
+                  _receivedTerminalEvent = true;
                   finalizeAssistantMessage();
                   renderIfActiveConversation(false);
                 }
                 if (eventName === "run:error") {
+                  _receivedTerminalEvent = true;
                   finalizeAssistantMessage();
                   const errMsg = payload.error?.message || "Something went wrong";
                   assistantMessage._error = errMsg;
@@ -4615,7 +4714,19 @@ var getWebUiClientScript = (markedSource2) => `
               }
             });
           }
+          if (!_shouldContinue && !_receivedTerminalEvent) {
+            try {
+              const recoveryPayload = await api("/api/conversations/" + encodeURIComponent(conversationId));
+              if (recoveryPayload.needsContinuation) {
+                _shouldContinue = true;
+                console.log("[poncho] Stream ended without terminal event, server has continuation \u2014 resuming");
+              }
+            } catch (_recoverErr) {
+              console.warn("[poncho] Recovery check failed after abrupt stream end");
+            }
+          }
           if (!_shouldContinue) break;
+          _receivedTerminalEvent = false;
           _isContinuation = true;
           }
           // Update active state only if user is still on this conversation.
@@ -7664,12 +7775,16 @@ Set environment variables on your deployment platform:
 ANTHROPIC_API_KEY=sk-ant-...   # Required
 PONCHO_AUTH_TOKEN=your-secret  # Optional: protect your endpoint
 PONCHO_MAX_DURATION=55         # Optional: serverless timeout in seconds (enables auto-continuation)
+PONCHO_INTERNAL_SECRET=...     # Recommended on serverless: shared secret for internal callback auth
 \`\`\`
 
 When \`PONCHO_MAX_DURATION\` is set, the agent automatically checkpoints and resumes across
 request cycles when it approaches the platform timeout. The web UI and client SDK handle
 this transparently.
 
+For serverless deployments with subagents or background callbacks, use a shared state backend
+(\`upstash\`, \`redis\`, or \`dynamodb\`) instead of \`state.provider: 'local'\` / \`'memory'\`.
+
 ## Troubleshooting
 
 ### Vercel deploy issues
@@ -7677,6 +7792,7 @@ this transparently.
 - After upgrading \`@poncho-ai/cli\`, re-run \`poncho build vercel --force\` to refresh generated deploy files.
 - If Vercel fails during \`pnpm install\` due to a lockfile mismatch, run \`pnpm install --no-frozen-lockfile\` locally and commit \`pnpm-lock.yaml\`.
 - Deploy from the project root: \`vercel deploy --prod\`.
+- For subagents/background callbacks, set \`PONCHO_INTERNAL_SECRET\` and use non-local state storage.
 
 For full reference:
 https://github.com/cesr/poncho-ai
@@ -8392,7 +8508,8 @@ data: ${JSON.stringify(statusPayload)}
   await harness.initialize();
   const telemetry = new TelemetryEmitter(config?.telemetry);
   const identity = await ensureAgentIdentity2(workingDir);
-  const conversationStore = createConversationStore(resolveStateConfig(config), {
+  const stateConfig = resolveStateConfig(config);
+  const conversationStore = createConversationStore(stateConfig, {
     workingDir,
     agentId: identity.id
   });
@@ -8731,9 +8848,10 @@ data: ${JSON.stringify(statusPayload)}
           } catch {
           }
           if (isServerless) {
-            selfFetchWithRetry(`/api/internal/subagent/${encodeURIComponent(childConversationId)}/run`, { continuation: true }).catch(
+            const work = selfFetchWithRetry(`/api/internal/subagent/${encodeURIComponent(childConversationId)}/run`, { continuation: true }).catch(
               (err) => console.error(`[poncho][subagent] Continuation self-fetch failed:`, err instanceof Error ? err.message : err)
             );
+            doWaitUntil(work);
           } else {
             runSubagent(childConversationId, parentConversationId, task, ownerId, true).catch(
               (err) => console.error(`[poncho][subagent] Continuation failed:`, err instanceof Error ? err.message : err)
@@ -8837,7 +8955,8 @@ data: ${JSON.stringify(statusPayload)}
     const conversation = await conversationStore.get(conversationId);
     if (!conversation) return;
     const pendingResults = conversation.pendingSubagentResults ?? [];
-    if (pendingResults.length === 0) return;
+    const hasOrphanedContinuation = pendingResults.length === 0 && Array.isArray(conversation._continuationMessages) && conversation._continuationMessages.length > 0 && !activeConversationRuns.has(conversationId);
+    if (pendingResults.length === 0 && !hasOrphanedContinuation) return;
     if (!skipLockCheck && conversation.runningCallbackSince) {
       const elapsed = Date.now() - conversation.runningCallbackSince;
       if (elapsed < CALLBACK_LOCK_STALE_MS) {
@@ -8847,6 +8966,7 @@ data: ${JSON.stringify(statusPayload)}
     }
     conversation.pendingSubagentResults = [];
     conversation.runningCallbackSince = Date.now();
+    conversation.runStatus = "running";
     const callbackCount = (conversation.subagentCallbackCount ?? 0) + 1;
     conversation.subagentCallbackCount = callbackCount;
     for (const pr of pendingResults) {
@@ -8866,10 +8986,12 @@ ${resultBody}`,
     if (callbackCount > MAX_SUBAGENT_CALLBACK_COUNT) {
       console.warn(`[poncho][subagent-callback] Circuit breaker: ${callbackCount} callbacks for ${conversationId}, skipping re-run`);
       conversation.runningCallbackSince = void 0;
+      conversation.runStatus = "idle";
       await conversationStore.update(conversation);
       return;
     }
-    console.log(`[poncho][subagent-callback] Processing ${pendingResults.length} result(s) for ${conversationId} (callback #${callbackCount})`);
+    const isContinuationResume = hasOrphanedContinuation && pendingResults.length === 0;
+    console.log(`[poncho][subagent-callback] Processing ${pendingResults.length} result(s) for ${conversationId} (callback #${callbackCount})${isContinuationResume ? " (continuation resume)" : ""}`);
     const abortController = new AbortController();
     activeConversationRuns.set(conversationId, {
       ownerId: conversation.ownerId,
@@ -8887,7 +9009,7 @@ ${resultBody}`,
         finished: false
       });
     }
-    const historyMessages = [...conversation.messages];
+    const historyMessages = isContinuationResume && conversation._continuationMessages?.length ? [...conversation._continuationMessages] : [...conversation.messages];
     let assistantResponse = "";
     let latestRunId = "";
     let runContinuation = false;
@@ -8976,6 +9098,7 @@ ${resultBody}`,
           }
           freshConv.runtimeRunId = latestRunId || freshConv.runtimeRunId;
           freshConv.runningCallbackSince = void 0;
+          freshConv.runStatus = "idle";
           if (runContextTokens > 0) freshConv.contextTokens = runContextTokens;
           if (runContextWindow > 0) freshConv.contextWindow = runContextWindow;
           freshConv.updatedAt = Date.now();
@@ -9000,9 +9123,14 @@ ${resultBody}`,
       }
       if (runContinuation) {
         if (isServerless) {
-          selfFetchWithRetry(`/api/internal/conversations/${encodeURIComponent(conversationId)}/subagent-callback`).catch(
+          const work = selfFetchWithRetry(`/api/internal/conversations/${encodeURIComponent(conversationId)}/subagent-callback`).catch(
             (err) => console.error(`[poncho][subagent-callback] Continuation self-fetch failed:`, err instanceof Error ? err.message : err)
           );
+          doWaitUntil(work);
+        } else {
+          processSubagentCallback(conversationId, true).catch(
+            (err) => console.error(`[poncho][subagent-callback] Continuation failed:`, err instanceof Error ? err.message : err)
+          );
         }
       }
     } catch (err) {
@@ -9010,6 +9138,7 @@ ${resultBody}`,
       const errConv = await conversationStore.get(conversationId);
       if (errConv) {
         errConv.runningCallbackSince = void 0;
+        errConv.runStatus = "idle";
         await conversationStore.update(errConv);
       }
     } finally {
@@ -9661,34 +9790,89 @@ ${resultBody}`,
     }
   }
   const isServerless = !!waitUntilHook;
-  const internalSecret = globalThis.crypto?.randomUUID?.() ?? `${Date.now()}-${Math.random()}`;
+  const configuredInternalSecret = process.env.PONCHO_INTERNAL_SECRET?.trim();
+  const vercelDeploymentSecret = process.env.VERCEL_DEPLOYMENT_ID?.trim();
+  const fallbackInternalSecret = globalThis.crypto?.randomUUID?.() ?? `${Date.now()}-${Math.random()}`;
+  const internalSecret = configuredInternalSecret || vercelDeploymentSecret || fallbackInternalSecret;
+  const isUsingEphemeralInternalSecret = !configuredInternalSecret && !vercelDeploymentSecret;
   let selfBaseUrl = process.env.VERCEL_URL ? `https://${process.env.VERCEL_URL}` : null;
+  if (!selfBaseUrl && process.env.VERCEL_PROJECT_PRODUCTION_URL) {
+    selfBaseUrl = `https://${process.env.VERCEL_PROJECT_PRODUCTION_URL}`;
+  }
+  if (!selfBaseUrl && process.env.PONCHO_SELF_BASE_URL) {
+    selfBaseUrl = process.env.PONCHO_SELF_BASE_URL.replace(/\/+$/, "");
+  }
+  if (isServerless && isUsingEphemeralInternalSecret) {
+    console.warn(
+      "[poncho][serverless] No stable internal secret found. Set PONCHO_INTERNAL_SECRET to avoid intermittent internal callback failures across instances."
+    );
+  }
+  if (isServerless && !selfBaseUrl) {
+    console.warn(
+      "[poncho][serverless] No self base URL available. Set PONCHO_SELF_BASE_URL if internal background callbacks fail."
+    );
+  }
+  const stateProvider = stateConfig?.provider ?? "local";
+  if (isServerless && (stateProvider === "local" || stateProvider === "memory")) {
+    console.warn(
+      `[poncho][serverless] state.provider="${stateProvider}" may lose cross-invocation state. Prefer "upstash", "redis", or "dynamodb" for subagents/reliability.`
+    );
+  }
   const doWaitUntil = (promise) => {
     if (waitUntilHook) waitUntilHook(promise);
   };
-  const selfFetch = (path, body) => {
-    if (!selfBaseUrl) return Promise.resolve();
-    return fetch(`${selfBaseUrl}${path}`, {
-      method: "POST",
-      headers: {
-        "Content-Type": "application/json",
-        "x-poncho-internal": internalSecret
-      },
-      body: body ? JSON.stringify(body) : void 0
-    }).catch((err) => {
-      console.error(`[poncho][self-fetch] Failed ${path}:`, err instanceof Error ? err.message : err);
-    });
-  };
   const selfFetchWithRetry = async (path, body, retries = 3) => {
+    if (!selfBaseUrl) {
+      console.error(`[poncho][self-fetch] Missing self base URL for ${path}`);
+      return;
+    }
+    let lastError;
     for (let attempt = 0; attempt < retries; attempt++) {
       try {
-        const result = await selfFetch(path, body);
-        return result;
+        const result = await fetch(`${selfBaseUrl}${path}`, {
+          method: "POST",
+          headers: {
+            "Content-Type": "application/json",
+            "x-poncho-internal": internalSecret
+          },
+          body: body ? JSON.stringify(body) : void 0
+        });
+        if (result.ok) {
+          return result;
+        }
+        const responseText = await result.text().catch(() => "");
+        lastError = new Error(
+          `HTTP ${result.status}${responseText ? `: ${responseText.slice(0, 200)}` : ""}`
+        );
       } catch (err) {
-        if (attempt === retries - 1) throw err;
-        await new Promise((r) => setTimeout(r, 1e3 * (attempt + 1)));
+        lastError = err;
+      }
+      if (attempt === retries - 1) {
+        break;
       }
+      await new Promise((resolveSleep) => setTimeout(resolveSleep, 1e3 * (attempt + 1)));
     }
+    if (lastError) {
+      console.error(
+        `[poncho][self-fetch] Failed ${path} after ${retries} attempt(s):`,
+        lastError instanceof Error ? lastError.message : String(lastError)
+      );
+      if (lastError instanceof Error && (lastError.message.includes("HTTP 403") || lastError.message.includes("HTTP 401"))) {
+        console.error(
+          "[poncho][self-fetch] Internal auth failed. Ensure all serverless instances share PONCHO_INTERNAL_SECRET."
+        );
+      }
+    } else {
+      console.error(`[poncho][self-fetch] Failed ${path} after ${retries} attempt(s).`);
+    }
+  };
+  const getInternalRequestHeader = (headers) => {
+    const value = headers["x-poncho-internal"];
+    return Array.isArray(value) ? value[0] : value;
+  };
+  const isValidInternalRequest = (headers) => {
+    const headerValue = getInternalRequestHeader(headers);
+    return typeof headerValue === "string" && headerValue === internalSecret;
   };
   const messagingAdapters = /* @__PURE__ */ new Map();
   const messagingBridges = [];
@@ -10005,7 +10189,7 @@ ${resultBody}`,
       }
     }
     if (pathname?.startsWith("/api/internal/") && request.method === "POST") {
-      if (request.headers["x-poncho-internal"] !== internalSecret) {
+      if (!isValidInternalRequest(request.headers)) {
         writeJson(response, 403, { code: "FORBIDDEN", message: "Internal endpoint" });
         return;
       }
@@ -10641,14 +10825,18 @@ data: ${JSON.stringify(frame)}
             }
           }
         }
+        const hasPendingCallbackResults = Array.isArray(conversation.pendingSubagentResults) && conversation.pendingSubagentResults.length > 0;
+        const needsContinuation = !hasActiveRun && Array.isArray(conversation._continuationMessages) && conversation._continuationMessages.length > 0;
         writeJson(response, 200, {
           conversation: {
             ...conversation,
-            pendingApprovals: storedPending
+            pendingApprovals: storedPending,
+            _continuationMessages: void 0
           },
           subagentPendingApprovals: subagentPending,
-          hasActiveRun,
-          hasRunningSubagents
+          hasActiveRun: hasActiveRun || hasPendingCallbackResults,
+          hasRunningSubagents,
+          needsContinuation
         });
         return;
       }
@@ -11983,7 +12171,7 @@ var runInteractive = async (workingDir, params) => {
   await harness.initialize();
   const identity = await ensureAgentIdentity2(workingDir);
   try {
-    const { runInteractiveInk } = await import("./run-interactive-ink-ZSIGWFLZ.js");
+    const { runInteractiveInk } = await import("./run-interactive-ink-FUMHN6DS.js");
     await runInteractiveInk({
       harness,
       params,

package/dist/cli.js

@@ -1,7 +1,7 @@
 #!/usr/bin/env node
 import {
   main
-} from "./chunk-UYZOJWGL.js";
+} from "./chunk-FA546WPW.js";
 
 // src/cli.ts
 void main();

package/dist/index.js

@@ -23,7 +23,7 @@ import {
   runTests,
   startDevServer,
   updateAgentGuidance
-} from "./chunk-UYZOJWGL.js";
+} from "./chunk-FA546WPW.js";
 export {
   addSkill,
   buildCli,

package/dist/{run-interactive-ink-ZSIGWFLZ.js → run-interactive-ink-FUMHN6DS.js}

@@ -2,7 +2,7 @@ import {
   consumeFirstRunIntro,
   inferConversationTitle,
   resolveHarnessEnvironment
-} from "./chunk-UYZOJWGL.js";
+} from "./chunk-FA546WPW.js";
 
 // src/run-interactive-ink.ts
 import * as readline from "readline";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@poncho-ai/cli",
-  "version": "0.30.1",
+  "version": "0.30.2",
   "description": "CLI for building and deploying AI agents",
   "repository": {
     "type": "git",
@@ -27,9 +27,9 @@
     "react": "^19.2.4",
     "react-devtools-core": "^6.1.5",
     "yaml": "^2.8.1",
-    "@poncho-ai/harness": "0.28.1",
-    "@poncho-ai/messaging": "0.7.2",
-    "@poncho-ai/sdk": "1.6.1"
+    "@poncho-ai/harness": "0.28.2",
+    "@poncho-ai/sdk": "1.6.1",
+    "@poncho-ai/messaging": "0.7.2"
   },
   "devDependencies": {
     "@types/busboy": "^1.5.4",

package/src/index.ts

@@ -721,12 +721,16 @@ Set environment variables on your deployment platform:
 ANTHROPIC_API_KEY=sk-ant-...   # Required
 PONCHO_AUTH_TOKEN=your-secret  # Optional: protect your endpoint
 PONCHO_MAX_DURATION=55         # Optional: serverless timeout in seconds (enables auto-continuation)
+PONCHO_INTERNAL_SECRET=...     # Recommended on serverless: shared secret for internal callback auth
 \`\`\`
 
 When \`PONCHO_MAX_DURATION\` is set, the agent automatically checkpoints and resumes across
 request cycles when it approaches the platform timeout. The web UI and client SDK handle
 this transparently.
 
+For serverless deployments with subagents or background callbacks, use a shared state backend
+(\`upstash\`, \`redis\`, or \`dynamodb\`) instead of \`state.provider: 'local'\` / \`'memory'\`.
+
 ## Troubleshooting
 
 ### Vercel deploy issues
@@ -734,6 +738,7 @@ this transparently.
 - After upgrading \`@poncho-ai/cli\`, re-run \`poncho build vercel --force\` to refresh generated deploy files.
 - If Vercel fails during \`pnpm install\` due to a lockfile mismatch, run \`pnpm install --no-frozen-lockfile\` locally and commit \`pnpm-lock.yaml\`.
 - Deploy from the project root: \`vercel deploy --prod\`.
+- For subagents/background callbacks, set \`PONCHO_INTERNAL_SECRET\` and use non-local state storage.
 
 For full reference:
 https://github.com/cesr/poncho-ai
@@ -1554,7 +1559,8 @@ export const createRequestHandler = async (options?: {
   await harness.initialize();
   const telemetry = new TelemetryEmitter(config?.telemetry);
   const identity = await ensureAgentIdentity(workingDir);
-  const conversationStore = createConversationStore(resolveStateConfig(config), {
+  const stateConfig = resolveStateConfig(config);
+  const conversationStore = createConversationStore(stateConfig, {
     workingDir,
     agentId: identity.id,
   });
@@ -1956,9 +1962,10 @@ export const createRequestHandler = async (options?: {
           try { await childHarness.shutdown(); } catch {}
 
           if (isServerless) {
-            selfFetchWithRetry(`/api/internal/subagent/${encodeURIComponent(childConversationId)}/run`, { continuation: true }).catch(err =>
+            const work = selfFetchWithRetry(`/api/internal/subagent/${encodeURIComponent(childConversationId)}/run`, { continuation: true }).catch(err =>
               console.error(`[poncho][subagent] Continuation self-fetch failed:`, err instanceof Error ? err.message : err),
             );
+            doWaitUntil(work);
           } else {
             runSubagent(childConversationId, parentConversationId, task, ownerId, true).catch(err =>
               console.error(`[poncho][subagent] Continuation failed:`, err instanceof Error ? err.message : err),
@@ -2079,7 +2086,11 @@ export const createRequestHandler = async (options?: {
     if (!conversation) return;
 
     const pendingResults = conversation.pendingSubagentResults ?? [];
-    if (pendingResults.length === 0) return;
+    const hasOrphanedContinuation = pendingResults.length === 0
+      && Array.isArray(conversation._continuationMessages)
+      && conversation._continuationMessages.length > 0
+      && !activeConversationRuns.has(conversationId);
+    if (pendingResults.length === 0 && !hasOrphanedContinuation) return;
 
     // Store-based lock for serverless: skip if another invocation is processing.
     // When re-triggered from a previous callback's finally block, skipLockCheck
@@ -2095,6 +2106,7 @@ export const createRequestHandler = async (options?: {
     // Acquire lock and clear pending
     conversation.pendingSubagentResults = [];
     conversation.runningCallbackSince = Date.now();
+    conversation.runStatus = "running";
     const callbackCount = (conversation.subagentCallbackCount ?? 0) + 1;
     conversation.subagentCallbackCount = callbackCount;
 
@@ -2116,11 +2128,13 @@ export const createRequestHandler = async (options?: {
     if (callbackCount > MAX_SUBAGENT_CALLBACK_COUNT) {
       console.warn(`[poncho][subagent-callback] Circuit breaker: ${callbackCount} callbacks for ${conversationId}, skipping re-run`);
       conversation.runningCallbackSince = undefined;
+      conversation.runStatus = "idle";
       await conversationStore.update(conversation);
       return;
     }
 
-    console.log(`[poncho][subagent-callback] Processing ${pendingResults.length} result(s) for ${conversationId} (callback #${callbackCount})`);
+    const isContinuationResume = hasOrphanedContinuation && pendingResults.length === 0;
+    console.log(`[poncho][subagent-callback] Processing ${pendingResults.length} result(s) for ${conversationId} (callback #${callbackCount})${isContinuationResume ? " (continuation resume)" : ""}`);
 
     const abortController = new AbortController();
     activeConversationRuns.set(conversationId, {
@@ -2142,7 +2156,9 @@ export const createRequestHandler = async (options?: {
       });
     }
 
-    const historyMessages = [...conversation.messages];
+    const historyMessages = isContinuationResume && conversation._continuationMessages?.length
+      ? [...conversation._continuationMessages]
+      : [...conversation.messages];
     let assistantResponse = "";
     let latestRunId = "";
     let runContinuation = false;
@@ -2236,6 +2252,7 @@ export const createRequestHandler = async (options?: {
           }
           freshConv.runtimeRunId = latestRunId || freshConv.runtimeRunId;
           freshConv.runningCallbackSince = undefined;
+          freshConv.runStatus = "idle";
           if (runContextTokens > 0) freshConv.contextTokens = runContextTokens;
           if (runContextWindow > 0) freshConv.contextWindow = runContextWindow;
           freshConv.updatedAt = Date.now();
@@ -2264,9 +2281,14 @@ export const createRequestHandler = async (options?: {
       // Handle continuation for the callback run itself
       if (runContinuation) {
         if (isServerless) {
-          selfFetchWithRetry(`/api/internal/conversations/${encodeURIComponent(conversationId)}/subagent-callback`).catch(err =>
+          const work = selfFetchWithRetry(`/api/internal/conversations/${encodeURIComponent(conversationId)}/subagent-callback`).catch(err =>
             console.error(`[poncho][subagent-callback] Continuation self-fetch failed:`, err instanceof Error ? err.message : err),
           );
+          doWaitUntil(work);
+        } else {
+          processSubagentCallback(conversationId, true).catch(err =>
+            console.error(`[poncho][subagent-callback] Continuation failed:`, err instanceof Error ? err.message : err),
+          );
         }
       }
     } catch (err) {
@@ -2274,6 +2296,7 @@ export const createRequestHandler = async (options?: {
       const errConv = await conversationStore.get(conversationId);
       if (errConv) {
         errConv.runningCallbackSince = undefined;
+        errConv.runStatus = "idle";
         await conversationStore.update(errConv);
       }
     } finally {
@@ -3014,41 +3037,102 @@ export const createRequestHandler = async (options?: {
   }
 
   const isServerless = !!waitUntilHook;
-  const internalSecret = globalThis.crypto?.randomUUID?.() ?? `${Date.now()}-${Math.random()}`;
+  const configuredInternalSecret = process.env.PONCHO_INTERNAL_SECRET?.trim();
+  const vercelDeploymentSecret = process.env.VERCEL_DEPLOYMENT_ID?.trim();
+  const fallbackInternalSecret = globalThis.crypto?.randomUUID?.() ?? `${Date.now()}-${Math.random()}`;
+  const internalSecret = configuredInternalSecret || vercelDeploymentSecret || fallbackInternalSecret;
+  const isUsingEphemeralInternalSecret = !configuredInternalSecret && !vercelDeploymentSecret;
   let selfBaseUrl: string | null = process.env.VERCEL_URL
     ? `https://${process.env.VERCEL_URL}`
     : null;
 
+  if (!selfBaseUrl && process.env.VERCEL_PROJECT_PRODUCTION_URL) {
+    selfBaseUrl = `https://${process.env.VERCEL_PROJECT_PRODUCTION_URL}`;
+  }
+  if (!selfBaseUrl && process.env.PONCHO_SELF_BASE_URL) {
+    selfBaseUrl = process.env.PONCHO_SELF_BASE_URL.replace(/\/+$/, "");
+  }
+
+  if (isServerless && isUsingEphemeralInternalSecret) {
+    console.warn(
+      "[poncho][serverless] No stable internal secret found. Set PONCHO_INTERNAL_SECRET to avoid intermittent internal callback failures across instances.",
+    );
+  }
+  if (isServerless && !selfBaseUrl) {
+    console.warn(
+      "[poncho][serverless] No self base URL available. Set PONCHO_SELF_BASE_URL if internal background callbacks fail.",
+    );
+  }
+  const stateProvider = stateConfig?.provider ?? "local";
+  if (isServerless && (stateProvider === "local" || stateProvider === "memory")) {
+    console.warn(
+      `[poncho][serverless] state.provider="${stateProvider}" may lose cross-invocation state. Prefer "upstash", "redis", or "dynamodb" for subagents/reliability.`,
+    );
+  }
+
   const doWaitUntil = (promise: Promise<unknown>): void => {
     if (waitUntilHook) waitUntilHook(promise);
   };
 
-  const selfFetch = (path: string, body?: Record<string, unknown>): Promise<Response | void> => {
-    if (!selfBaseUrl) return Promise.resolve();
-    return fetch(`${selfBaseUrl}${path}`, {
-      method: "POST",
-      headers: {
-        "Content-Type": "application/json",
-        "x-poncho-internal": internalSecret,
-      },
-      body: body ? JSON.stringify(body) : undefined,
-    }).catch(err => {
-      console.error(`[poncho][self-fetch] Failed ${path}:`, err instanceof Error ? err.message : err);
-    }) as Promise<Response | void>;
-  };
-
   const selfFetchWithRetry = async (path: string, body?: Record<string, unknown>, retries = 3): Promise<Response | void> => {
+    if (!selfBaseUrl) {
+      console.error(`[poncho][self-fetch] Missing self base URL for ${path}`);
+      return;
+    }
+    let lastError: unknown;
     for (let attempt = 0; attempt < retries; attempt++) {
       try {
-        const result = await selfFetch(path, body);
-        return result;
+        const result = await fetch(`${selfBaseUrl}${path}`, {
+          method: "POST",
+          headers: {
+            "Content-Type": "application/json",
+            "x-poncho-internal": internalSecret,
+          },
+          body: body ? JSON.stringify(body) : undefined,
+        });
+        if (result.ok) {
+          return result;
+        }
+        const responseText = await result.text().catch(() => "");
+        lastError = new Error(
+          `HTTP ${result.status}${responseText ? `: ${responseText.slice(0, 200)}` : ""}`,
+        );
       } catch (err) {
-        if (attempt === retries - 1) throw err;
-        await new Promise(r => setTimeout(r, 1000 * (attempt + 1)));
+        lastError = err;
+      }
+      if (attempt === retries - 1) {
+        break;
+      }
+      await new Promise((resolveSleep) => setTimeout(resolveSleep, 1000 * (attempt + 1)));
+    }
+    if (lastError) {
+      console.error(
+        `[poncho][self-fetch] Failed ${path} after ${retries} attempt(s):`,
+        lastError instanceof Error ? lastError.message : String(lastError),
+      );
+      if (
+        lastError instanceof Error
+        && (lastError.message.includes("HTTP 403") || lastError.message.includes("HTTP 401"))
+      ) {
+        console.error(
+          "[poncho][self-fetch] Internal auth failed. Ensure all serverless instances share PONCHO_INTERNAL_SECRET.",
+        );
       }
+    } else {
+      console.error(`[poncho][self-fetch] Failed ${path} after ${retries} attempt(s).`);
     }
   };
 
+  const getInternalRequestHeader = (headers: IncomingMessage["headers"]): string | undefined => {
+    const value = headers["x-poncho-internal"];
+    return Array.isArray(value) ? value[0] : value;
+  };
+
+  const isValidInternalRequest = (headers: IncomingMessage["headers"]): boolean => {
+    const headerValue = getInternalRequestHeader(headers);
+    return typeof headerValue === "string" && headerValue === internalSecret;
+  };
+
   const messagingAdapters = new Map<string, MessagingAdapter>();
   const messagingBridges: AgentBridge[] = [];
   if (config?.messaging && config.messaging.length > 0) {
@@ -3417,7 +3501,7 @@ export const createRequestHandler = async (options?: {
 
     // ── Internal endpoints (self-fetch only, secured by startup secret) ──
     if (pathname?.startsWith("/api/internal/") && request.method === "POST") {
-      if (request.headers["x-poncho-internal"] !== internalSecret) {
+      if (!isValidInternalRequest(request.headers)) {
         writeJson(response, 403, { code: "FORBIDDEN", message: "Internal endpoint" });
         return;
       }
@@ -4171,14 +4255,21 @@ export const createRequestHandler = async (options?: {
             }
           }
         }
+        const hasPendingCallbackResults = Array.isArray(conversation.pendingSubagentResults)
+          && conversation.pendingSubagentResults.length > 0;
+        const needsContinuation = !hasActiveRun
+          && Array.isArray(conversation._continuationMessages)
+          && conversation._continuationMessages.length > 0;
         writeJson(response, 200, {
           conversation: {
             ...conversation,
             pendingApprovals: storedPending,
+            _continuationMessages: undefined,
           },
           subagentPendingApprovals: subagentPending,
-          hasActiveRun,
+          hasActiveRun: hasActiveRun || hasPendingCallbackResults,
           hasRunningSubagents,
+          needsContinuation,
         });
         return;
       }

package/src/web-ui-client.ts

@@ -1344,6 +1344,101 @@ export const getWebUiClientScript = (markedSource: string): string => `
           });
         } else if (willStream) {
           setStreaming(true);
+        } else if (payload.needsContinuation && !payload.conversation.parentConversationId) {
+          console.log("[poncho] Detected orphaned continuation for", conversationId, "— auto-resuming");
+          (async () => {
+            try {
+              setStreaming(true);
+              var localMsgs = state.activeMessages || [];
+              var contAssistant = {
+                role: "assistant",
+                content: "",
+                _sections: [],
+                _currentText: "",
+                _currentTools: [],
+                _toolImages: [],
+                _activeActivities: [],
+                _pendingApprovals: [],
+                metadata: { toolActivity: [] }
+              };
+              localMsgs.push(contAssistant);
+              state.activeMessages = localMsgs;
+              state._activeStreamMessages = localMsgs;
+              renderMessages(localMsgs, true);
+              var contResp = await fetch(
+                "/api/conversations/" + encodeURIComponent(conversationId) + "/messages",
+                {
+                  method: "POST",
+                  credentials: "include",
+                  headers: { "Content-Type": "application/json", "x-csrf-token": state.csrfToken },
+                  body: JSON.stringify({ continuation: true }),
+                },
+              );
+              if (!contResp.ok || !contResp.body) {
+                contAssistant._error = "Failed to resume — reload to retry";
+                setStreaming(false);
+                renderMessages(localMsgs, false);
+                return;
+              }
+              state.activeStreamConversationId = conversationId;
+              var contReader = contResp.body.getReader();
+              var contDecoder = new TextDecoder();
+              var contBuffer = "";
+              while (true) {
+                var chunk = await contReader.read();
+                if (chunk.done) break;
+                contBuffer += contDecoder.decode(chunk.value, { stream: true });
+                contBuffer = parseSseChunk(contBuffer, function(evtName, evtPayload) {
+                  if (evtName === "model:chunk" && evtPayload.content) {
+                    contAssistant.content = (contAssistant.content || "") + evtPayload.content;
+                    contAssistant._currentText += evtPayload.content;
+                  }
+                  if (evtName === "tool:started") {
+                    if (contAssistant._currentText) {
+                      contAssistant._sections.push({ type: "text", content: contAssistant._currentText });
+                      contAssistant._currentText = "";
+                    }
+                    contAssistant._currentTools.push("- start \`" + evtPayload.tool + "\`");
+                  }
+                  if (evtName === "tool:completed") {
+                    contAssistant._currentTools.push("- done \`" + evtPayload.tool + "\` (" + evtPayload.duration + "ms)");
+                  }
+                  if (evtName === "tool:error") {
+                    contAssistant._currentTools.push("- error \`" + evtPayload.tool + "\`: " + evtPayload.error);
+                  }
+                  if (evtName === "run:completed" || evtName === "run:error" || evtName === "run:cancelled") {
+                    if (contAssistant._currentTools.length > 0) {
+                      contAssistant._sections.push({ type: "tools", content: contAssistant._currentTools });
+                      contAssistant._currentTools = [];
+                    }
+                    if (contAssistant._currentText) {
+                      contAssistant._sections.push({ type: "text", content: contAssistant._currentText });
+                      contAssistant._currentText = "";
+                    }
+                    contAssistant._activeActivities = [];
+                    if (evtName === "run:error") {
+                      contAssistant._error = evtPayload.error?.message || "Something went wrong";
+                    }
+                    if (evtName === "run:completed" && evtPayload.result?.continuation === true) {
+                      // Another continuation needed — reload to pick it up
+                      loadConversation(conversationId).catch(function() {});
+                    }
+                  }
+                  renderMessages(localMsgs, true);
+                });
+              }
+              setStreaming(false);
+              renderMessages(localMsgs, false);
+              await loadConversations();
+            } catch (contErr) {
+              console.error("[poncho] Auto-continuation failed:", contErr);
+              setStreaming(false);
+              await loadConversation(conversationId).catch(function() {});
+            } finally {
+              state.activeStreamConversationId = null;
+              state._activeStreamMessages = null;
+            }
+          })();
         }
       };
 
@@ -2427,6 +2522,7 @@ export const getWebUiClientScript = (markedSource: string): string => `
           let _totalSteps = 0;
           let _maxSteps = 0;
           let _isContinuation = false;
+          let _receivedTerminalEvent = false;
           while (true) {
           let _shouldContinue = false;
           let fetchOpts;
@@ -2756,6 +2852,7 @@ export const getWebUiClientScript = (markedSource: string): string => `
                   }
                 }
                 if (eventName === "run:completed") {
+                  _receivedTerminalEvent = true;
                   _totalSteps += typeof payload.result?.steps === "number" ? payload.result.steps : 0;
                   if (typeof payload.result?.maxSteps === "number") _maxSteps = payload.result.maxSteps;
                   if (payload.result?.continuation === true && (_maxSteps <= 0 || _totalSteps < _maxSteps)) {
@@ -2772,10 +2869,12 @@ export const getWebUiClientScript = (markedSource: string): string => `
                   }
                 }
                 if (eventName === "run:cancelled") {
+                  _receivedTerminalEvent = true;
                   finalizeAssistantMessage();
                   renderIfActiveConversation(false);
                 }
                 if (eventName === "run:error") {
+                  _receivedTerminalEvent = true;
                   finalizeAssistantMessage();
                   const errMsg = payload.error?.message || "Something went wrong";
                   assistantMessage._error = errMsg;
@@ -2786,7 +2885,19 @@ export const getWebUiClientScript = (markedSource: string): string => `
               }
             });
           }
+          if (!_shouldContinue && !_receivedTerminalEvent) {
+            try {
+              const recoveryPayload = await api("/api/conversations/" + encodeURIComponent(conversationId));
+              if (recoveryPayload.needsContinuation) {
+                _shouldContinue = true;
+                console.log("[poncho] Stream ended without terminal event, server has continuation — resuming");
+              }
+            } catch (_recoverErr) {
+              console.warn("[poncho] Recovery check failed after abrupt stream end");
+            }
+          }
           if (!_shouldContinue) break;
+          _receivedTerminalEvent = false;
           _isContinuation = true;
           }
           // Update active state only if user is still on this conversation.