@skydiveai/pi-server 0.1.0-beta.1310 → 0.1.0-beta.1738

package/dist/index.mjs

@@ -373,6 +373,57 @@ async function runConversation({ session, prompt, images, log, postPrompt }) {
 	});
 }
 /**
+* Track the agent session's built-in model-call auto-retry so it leaves a
+* trace.
+*
+* `AgentSession` already restarts a failed assistant turn in place (via
+* `agent.continue()`, so no prompt is replayed and no tool re-executes) for the
+* transient provider/transport failures pi classifies as retryable -- dropped
+* streams, `terminated`, 5xx, overloaded, rate limits. It is on by default,
+* with its own budget and backoff, and it emits `auto_retry_start` /
+* `auto_retry_end` around each attempt.
+*
+* Nothing consumed those events, so a retry left no trace anywhere: a call that
+* succeeded first try and one that burned the whole budget before failing
+* produced the same terminal error, and the fleet-wide retry rate was
+* unmeasurable. That gap is why a 2026-08-16 investigation into three runs lost
+* to `provider_error: terminated` could not tell whether the budget had run out
+* (ANY-7101).
+*
+* Exposed as a handler rather than its own `session.subscribe` call so each
+* protocol feeds it from the single subscription it already owns -- one
+* subscriber, explicit ordering.
+*
+* `attempts()` reports what has been spent so far, so a terminal error can
+* carry the count to the worker, where it lands in a log group we can query
+* fleet-wide (the sandbox's own logs are not).
+*/
+function createAutoRetryObserver(log) {
+	let attempts = 0;
+	return {
+		observe(event) {
+			if (event.type === "auto_retry_start") {
+				attempts = typeof event.attempt === "number" ? event.attempt : attempts + 1;
+				log.warn({
+					event: "model_call_auto_retry",
+					attempt: event.attempt,
+					max_attempts: event.maxAttempts,
+					delay_ms: event.delayMs,
+					error_message: event.errorMessage
+				}, "retrying failed model call in place");
+				return;
+			}
+			if (event.type === "auto_retry_end") log.warn({
+				event: "model_call_auto_retry_end",
+				attempt: event.attempt,
+				success: event.success,
+				final_error: event.finalError
+			}, event.success ? "model call recovered after retry" : "model call retries exhausted");
+		},
+		attempts: () => attempts
+	};
+}
+/**
 * Hard-stop the in-flight turn for a session. Shared by every protocol's
 * `/:id/abort` route: a cancel signals the stop explicitly instead of relying
 * on a dropped connection. `session.abort()` interrupts the turn and resolves
@@ -1159,12 +1210,14 @@ function runStream$2({ session, sm, prompt, images, id, sessionId, modelName, lo
 			textBlockOpen = false;
 		};
 		let capturedModelError = null;
+		const autoRetry = createAutoRetryObserver(log);
 		const emitProviderError = (providerError) => {
 			log.warn({
 				event: "model_provider_error",
 				code: providerError.code,
 				upstream_status: providerError.upstreamStatus,
-				provider: providerError.provider
+				provider: providerError.provider,
+				retry_attempts: autoRetry.attempts()
 			}, "forwarding model-provider error to client");
 			sseEvent(writer, "error", {
 				type: "error",
@@ -1174,13 +1227,15 @@ function runStream$2({ session, sm, prompt, images, id, sessionId, modelName, lo
 					x_model_provider_error: {
 						code: providerError.code,
 						provider: providerError.provider,
-						upstream_status: providerError.upstreamStatus
+						upstream_status: providerError.upstreamStatus,
+						retry_attempts: autoRetry.attempts()
 					}
 				}
 			});
 		};
 		session.subscribe((event) => {
 			const ev = event;
+			autoRetry.observe(ev);
 			const endedMessage = ev.message ?? (Array.isArray(ev.messages) ? ev.messages[ev.messages.length - 1] : null);
 			if (endedMessage?.stopReason === "error" && typeof endedMessage.errorMessage === "string" && endedMessage.errorMessage.length > 0) capturedModelError = endedMessage.errorMessage;
 			if (ev.type !== "message_update") return;
@@ -1856,6 +1911,7 @@ function runStream$1({ session, sm, prompt, images, sessionId, created, reqStart
 		let usage = null;
 		let lastFollowUpCount = 0;
 		let capturedModelError = null;
+		const autoRetry = createAutoRetryObserver(log);
 		const sendChunk = (delta, finishReason) => {
 			if (firstChunkAt === null) {
 				firstChunkAt = performance.now();
@@ -1886,6 +1942,7 @@ function runStream$1({ session, sm, prompt, images, sessionId, created, reqStart
 		let _evCount = 0;
 		session.subscribe((event) => {
 			const ev = event;
+			autoRetry.observe(ev);
 			const endedMessage = ev.message ?? (Array.isArray(ev.messages) ? ev.messages[ev.messages.length - 1] : null);
 			if (endedMessage?.stopReason === "error" && typeof endedMessage.errorMessage === "string" && endedMessage.errorMessage.length > 0) capturedModelError = endedMessage.errorMessage;
 			if (ev.type === "queue_update") {
@@ -2013,13 +2070,15 @@ function runStream$1({ session, sm, prompt, images, sessionId, created, reqStart
 					code: providerError.code,
 					upstream_status: providerError.upstreamStatus,
 					provider: providerError.provider,
+					retry_attempts: autoRetry.attempts(),
 					source: "stop_reason_error"
 				}, "forwarding model-provider error to client");
 				emitModelProviderError({
 					writer,
 					sessionId,
 					created,
-					providerError
+					providerError,
+					retryAttempts: autoRetry.attempts()
 				});
 				writer.write("data: [DONE]\n\n");
 			} else {
@@ -2058,13 +2117,15 @@ function runStream$1({ session, sm, prompt, images, sessionId, created, reqStart
 					chatcmpl_id: sessionId,
 					code: providerError.code,
 					upstream_status: providerError.upstreamStatus,
-					provider: providerError.provider
+					provider: providerError.provider,
+					retry_attempts: autoRetry.attempts()
 				}, "forwarding model-provider error to client");
 				emitModelProviderError({
 					writer,
 					sessionId,
 					created,
-					providerError
+					providerError,
+					retryAttempts: autoRetry.attempts()
 				});
 			} else sendChunk({ content: `\n[error: ${err?.message ?? err}]` }, "stop");
 			writer.write("data: [DONE]\n\n");
@@ -2105,7 +2166,7 @@ function emitSessionMessagesTrailer({ writer, sm, baselineMessageCount, sessionI
 * cleanly closes the stream for any OpenAI-shaped reader that ignores the
 * extension field.
 */
-function emitModelProviderError({ writer, sessionId, created, providerError }) {
+function emitModelProviderError({ writer, sessionId, created, providerError, retryAttempts }) {
 	const chunk = {
 		id: sessionId,
 		object: "chat.completion.chunk",
@@ -2120,7 +2181,8 @@ function emitModelProviderError({ writer, sessionId, created, providerError }) {
 			message: providerError.message,
 			provider: providerError.provider,
 			type: providerError.type,
-			upstream_status: providerError.upstreamStatus
+			upstream_status: providerError.upstreamStatus,
+			retry_attempts: retryAttempts
 		}
 	};
 	writer.write(`data: ${JSON.stringify(chunk)}\n\n`);
@@ -2500,6 +2562,7 @@ function runStream({ session, prompt, images, responseId, sessionId, modelName,
 		const tcByContentIdx = /* @__PURE__ */ new Map();
 		let nextOutputIndex = 0;
 		let capturedModelError = null;
+		const autoRetry = createAutoRetryObserver(log);
 		const send = (event) => {
 			writer.write(`data: ${JSON.stringify(event)}\n\n`);
 		};
@@ -2508,7 +2571,8 @@ function runStream({ session, prompt, images, responseId, sessionId, modelName,
 				event: "model_provider_error",
 				code: providerError.code,
 				upstream_status: providerError.upstreamStatus,
-				provider: providerError.provider
+				provider: providerError.provider,
+				retry_attempts: autoRetry.attempts()
 			}, "forwarding model-provider error to client");
 			send({
 				type: "response.failed",
@@ -2522,7 +2586,8 @@ function runStream({ session, prompt, images, responseId, sessionId, modelName,
 						x_model_provider_error: {
 							provider: providerError.provider,
 							type: providerError.type,
-							upstream_status: providerError.upstreamStatus
+							upstream_status: providerError.upstreamStatus,
+							retry_attempts: autoRetry.attempts()
 						}
 					}
 				}
@@ -2558,6 +2623,7 @@ function runStream({ session, prompt, images, responseId, sessionId, modelName,
 		};
 		session.subscribe((event) => {
 			const ev = event;
+			autoRetry.observe(ev);
 			const endedMessage = ev.message ?? (Array.isArray(ev.messages) ? ev.messages[ev.messages.length - 1] : null);
 			if (endedMessage?.stopReason === "error" && typeof endedMessage.errorMessage === "string" && endedMessage.errorMessage.length > 0) capturedModelError = endedMessage.errorMessage;
 			if (ev.type !== "message_update") return;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@skydiveai/pi-server",
-  "version": "0.1.0-beta.1310",
+  "version": "0.1.0-beta.1738",
   "homepage": "https://skydive.com",
   "license": "MIT",
   "author": "Create, Inc.",
@@ -29,8 +29,7 @@
     "build": "tsdown",
     "typecheck": "tsgo --noEmit",
     "test:unit": "vitest run --passWithNoTests",
-    "test:ci": "vitest run --coverage --coverage.reporter=lcovonly --reporter=default --reporter=github-actions --minWorkers=1 --maxWorkers=2 --passWithNoTests",
-    "publish:system-artifacts": "doppler run --preserve-env -- node ../../scripts/anyone/publish-system-artifact.mjs"
+    "test:ci": "vitest run --coverage --coverage.reporter=lcovonly --reporter=default --reporter=github-actions --minWorkers=1 --maxWorkers=2 --passWithNoTests"
   },
   "dependencies": {
     "@a2a-js/sdk": "^0.3.13",