@sayknow-cli/ai 0.3.0 → 0.3.2

package/src/utils/overflow.ts

@@ -52,14 +52,26 @@ const OVERFLOW_PATTERNS = [
 	/\b413\b.*\b(request|payload|entity)\b.*\btoo large\b/i, // "413 Request Entity Too Large" variants
 	/model_context_window_exceeded/i, // z.ai non-standard finish_reason surfaced as error text
 ];
+/**
+ * Threshold below which a "successful" (stopReason "stop") response with empty
+ * content is considered anomalous. Some proxies (notably LiteLLM) return an
+ * empty `choices[0].message.content` with a near-zero `usage` (e.g. input: 1,
+ * output: 1) when the upstream model context window is exceeded, instead of
+ * surfacing a proper error. The total token count for such a response is well
+ * below any realistic turn, so we treat it as a proxy-level overflow signal.
+ */
+const EMPTY_RESPONSE_USAGE_THRESHOLD = 5;
 /**
  * Check if an assistant message represents a context overflow error.
  *
- * This handles two cases:
+ * This handles three cases:
  * 1. Error-based overflow: Most providers return stopReason "error" with a
  *    specific error message pattern.
  * 2. Silent overflow: Some providers accept overflow requests and return
  *    successfully. For these, we check if usage.input exceeds the context window.
+ * 3. Proxy-level overflow: Some proxies (e.g. LiteLLM) return a "successful"
+ *    response with empty content and a fabricated near-zero usage when the
+ *    upstream model's context window is exceeded.
  *
  * ## Reliability by Provider
  *
@@ -83,6 +95,13 @@ const OVERFLOW_PATTERNS = [
  * - z.ai: Sometimes accepts overflow silently (detectable via usage.input > contextWindow),
  *   sometimes returns rate limit errors. Pass contextWindow param to detect silent overflow.
  * - Ollama: Silently truncates input without error. Cannot be detected via this function.
+ * - LiteLLM proxy: Returns a "successful" response with empty content and a
+ *   fabricated near-zero usage (e.g. input: 1, output: 1) when the upstream
+ *   model's context window is exceeded. Detected via Case 3 (empty content +
+ *   anomalously low usage). Note: the LiteLLM proxy's context limit may differ
+ *   from the underlying model's advertised contextWindow (e.g. configured via
+ *   `model_info.max_tokens` in LiteLLM's config.yaml), so Case 2 (which compares
+ *   usage.input against contextWindow) may not catch it.
  *   The response will have usage.input < expected, but we don't know the expected value.
  *
  * ## Custom Providers
@@ -126,6 +145,21 @@ export function isContextOverflow(message: AssistantMessage, contextWindow?: num
 		}
 	}
 
+	// Case 3: Empty response with anomalously low usage (proxy-level overflow)
+	// Some proxies (e.g. LiteLLM) return a "successful" response (stopReason "stop")
+	// with empty content and a near-zero token count when the upstream model's
+	// context window is exceeded. This is distinct from silent overflow (Case 2),
+	// where the provider reports the real input token count. Here the proxy
+	// fabricates a bogus usage (input: 1, output: 1) that is far below any
+	// realistic turn, so we detect it heuristically.
+	if (
+		message.stopReason === "stop" &&
+		message.content.length === 0 &&
+		message.usage.input + message.usage.output <= EMPTY_RESPONSE_USAGE_THRESHOLD
+	) {
+		return true;
+	}
+
 	return false;
 }
 

package/src/utils.ts

@@ -91,6 +91,92 @@ export function sanitizeOpenAIResponsesHistoryItemsForReplay(items: Array<Record
 		return sanitized ? [sanitized] : [];
 	});
 }
+function stringifyResponsesStringParamForReplay(value: unknown): string {
+	if (typeof value === "string") return value.toWellFormed();
+	try {
+		const encoded = JSON.stringify(value);
+		if (typeof encoded === "string") return encoded.toWellFormed();
+	} catch {
+		// Fall through to String().
+	}
+	return String(value ?? "").toWellFormed();
+}
+
+function normalizeResponsesMessageTextForReplay(value: unknown): string {
+	if (typeof value === "string") return value.toWellFormed();
+	if (value && typeof value === "object") {
+		const nestedText = (value as { text?: unknown }).text;
+		if (typeof nestedText === "string") return nestedText.toWellFormed();
+	}
+	return stringifyResponsesStringParamForReplay(value);
+}
+
+type ResponsesImageDetail = "auto" | "low" | "high";
+
+interface NormalizedResponsesImageUrl {
+	readonly imageUrl: string;
+	readonly detail?: ResponsesImageDetail;
+}
+
+function isResponsesImageDetail(value: unknown): value is ResponsesImageDetail {
+	return value === "auto" || value === "low" || value === "high";
+}
+
+function normalizeResponsesImageUrlForReplay(value: unknown): NormalizedResponsesImageUrl {
+	if (typeof value === "string") return { imageUrl: value.toWellFormed() };
+	if (value && typeof value === "object" && "url" in value && typeof value.url === "string") {
+		const detail = "detail" in value && isResponsesImageDetail(value.detail) ? value.detail : undefined;
+		return {
+			imageUrl: value.url.toWellFormed(),
+			...(detail ? { detail } : {}),
+		};
+	}
+	return { imageUrl: stringifyResponsesStringParamForReplay(value) };
+}
+
+function sanitizeResponsesMessageContentForReplay(content: unknown): unknown {
+	if (typeof content === "string") return content.toWellFormed();
+	if (!Array.isArray(content)) return content;
+	return content.map(part => {
+		if (!part || typeof part !== "object") return part;
+		const sanitizedPart = { ...(part as Record<string, unknown>) };
+		if ("text" in sanitizedPart) {
+			sanitizedPart.text = normalizeResponsesMessageTextForReplay(sanitizedPart.text);
+		}
+		if ("image_url" in sanitizedPart) {
+			const normalizedImageUrl = normalizeResponsesImageUrlForReplay(sanitizedPart.image_url);
+			sanitizedPart.image_url = normalizedImageUrl.imageUrl;
+			if (sanitizedPart.type === "image_url") {
+				sanitizedPart.type = "input_image";
+			}
+			if (normalizedImageUrl.detail) {
+				sanitizedPart.detail = normalizedImageUrl.detail;
+			} else if ("detail" in sanitizedPart && !isResponsesImageDetail(sanitizedPart.detail)) {
+				delete sanitizedPart.detail;
+			}
+		}
+		return sanitizedPart;
+	});
+}
+
+function sanitizeResponsesStringFieldsForReplay(item: Record<string, unknown>): void {
+	if (item.type === "message") {
+		item.content = sanitizeResponsesMessageContentForReplay(item.content);
+	}
+	if (item.type === "function_call" && "arguments" in item && typeof item.arguments !== "string") {
+		item.arguments = stringifyResponsesStringParamForReplay(item.arguments);
+	}
+	if (item.type === "custom_tool_call" && "input" in item && typeof item.input !== "string") {
+		item.input = stringifyResponsesStringParamForReplay(item.input);
+	}
+	if (
+		(item.type === "function_call_output" || item.type === "custom_tool_call_output") &&
+		"output" in item &&
+		typeof item.output !== "string"
+	) {
+		item.output = stringifyResponsesStringParamForReplay(item.output);
+	}
+}
 
 function sanitizeOpenAIResponsesHistoryItemForReplay(
 	item: Record<string, unknown>,
@@ -105,6 +191,7 @@ function sanitizeOpenAIResponsesHistoryItemForReplay(
 	if (typeof item.call_id === "string") {
 		sanitizedItem.call_id = normalizeReplayedResponsesHistoryCallId(item.call_id, normalizedCallIds);
 	}
+	sanitizeResponsesStringFieldsForReplay(sanitizedItem);
 
 	return sanitizedItem as unknown as OpenAIResponsesReplayItem;
 }