@oh-my-pi/pi-ai 16.2.13 → 16.3.2

package/src/providers/anthropic.ts

@@ -4,7 +4,7 @@ import { scheduler } from "node:timers/promises";
 import * as tls from "node:tls";
 import { isOfficialAnthropicApiUrl } from "@oh-my-pi/pi-catalog/compat/anthropic";
 import { mapEffortToAnthropicAdaptiveEffort } from "@oh-my-pi/pi-catalog/model-thinking";
-import { calculateCost } from "@oh-my-pi/pi-catalog/models";
+import { calculateCost, getBundledModel } from "@oh-my-pi/pi-catalog/models";
 import { isAnthropicOAuthToken } from "@oh-my-pi/pi-catalog/utils";
 import { parseGitHubCopilotApiKey } from "@oh-my-pi/pi-catalog/wire/github-copilot";
 import {
@@ -20,6 +20,7 @@ import { renderDemotedThinking } from "../dialect/demotion";
 import * as AIError from "../error";
 import { getEnvApiKey, OUTPUT_FALLBACK_BUFFER } from "../stream";
 import type {
+	AnthropicFallbackContent,
 	Api,
 	AssistantMessage,
 	CacheRetention,
@@ -73,7 +74,9 @@ import {
 import type {
 	ToolInputSchema as AnthropicToolInputSchema,
 	Tool as AnthropicWireTool,
+	Usage as AnthropicWireUsage,
 	ContentBlockParam,
+	FallbackParam,
 	MessageCreateParamsStreaming,
 	MessageParam,
 	RawMessageStreamEvent,
@@ -150,6 +153,7 @@ const redactThinkingBeta = "redact-thinking-2026-02-12";
 const fastModeBeta = "fast-mode-2026-02-01";
 const taskBudgetBeta = "task-budgets-2026-03-13";
 const effortBeta = "effort-2025-11-24";
+const serverSideFallbackBeta = "server-side-fallback-2026-06-01";
 
 function buildClaudeCodeBetas(
 	agentRequest: boolean,
@@ -1060,6 +1064,15 @@ export interface AnthropicOptions extends StreamOptions {
 	 * including SDK clients such as `AnthropicVertex`.
 	 */
 	client?: AnthropicMessagesClientLike;
+	/**
+	 * Server-side fallback beta chain (`server-side-fallback-2026-06-01`).
+	 * When set, `fallbacks` is forwarded on the request body and the beta
+	 * header is auto-attached; the response parser then honors mid-stream
+	 * `fallback` content blocks and `usage.iterations` for served-model
+	 * promotion and per-attempt pricing. Opt-in ONLY — leaving this
+	 * undefined preserves the pre-fallback behavior on every code path.
+	 */
+	fallbacks?: FallbackParam[];
 }
 
 export type AnthropicClientOptionsArgs = {
@@ -1529,6 +1542,108 @@ export function applyAnthropicUsageExtras(usage: Usage, source: AnthropicUsageLi
 	}
 }
 
+function parseAnthropicFallbackWireBlock(value: unknown): AnthropicFallbackContent | undefined {
+	if (!isRecord(value) || value.type !== "fallback") return undefined;
+	const from = isRecord(value.from) && typeof value.from.model === "string" ? value.from.model : undefined;
+	const to = isRecord(value.to) && typeof value.to.model === "string" ? value.to.model : undefined;
+	if (!from?.trim() || !to?.trim()) return undefined;
+	return { type: "fallback", from: { model: from }, to: { model: to } };
+}
+
+/**
+ * The definitive "served by fallback" signal per Anthropic's fallback
+ * billing cookbook (§4): a `fallback_message` iteration in `usage.iterations`.
+ * Any other iteration type is per-attempt bookkeeping for the requested model
+ * (including its dated snapshot alias) and MUST NOT retag the assistant turn.
+ */
+function fallbackServedModelFromUsage(source: AnthropicWireUsage): string | undefined {
+	const iterations = source.iterations ?? [];
+	for (let index = iterations.length - 1; index >= 0; index -= 1) {
+		const iteration = iterations[index];
+		if (iteration?.type === "fallback_message" && iteration.model?.trim()) return iteration.model;
+	}
+	return undefined;
+}
+
+/**
+ * Price a fallback turn per the fallback billing cookbook §4:
+ *   • A pre-served attempt with zero output/cache-creation is not billed
+ *     (waived classifier block); its iteration is skipped.
+ *   • Mid-stream refusals bill their attempting model's input+output at
+ *     that model's normal rates.
+ *   • The `fallback_message` attempt's input tokens are rebilled at the
+ *     served model's cache-read rate (fallback credit — 10% of base input).
+ *
+ * Top-level `usage.input/output/cacheRead/cacheWrite` stay Anthropic's raw
+ * served-attempt counts; `usage.cost` reflects the per-iteration attributed
+ * total. Non-fallback turns skip this path entirely and use the requested
+ * model at the normal `calculateCost` call.
+ */
+/**
+ * Resolve a served/iteration model id to its bundled catalog entry when
+ * possible so the per-iteration cost uses the served model's pricing
+ * (e.g. Opus 4.8 rates for a Fable→Opus fallback). Falls back to
+ * `requestModel` when the id is empty, matches the request, or the
+ * catalog has no entry under it — the caller keeps the requested-model
+ * pricing as the safe default and logs at the source.
+ */
+function resolveIterationModel(
+	requestModel: Model<"anthropic-messages">,
+	iterationModelId: string | null | undefined,
+): Model<Api> {
+	const id = iterationModelId?.trim();
+	if (!id || id === requestModel.id) return requestModel;
+	// Bundled catalog lookup: only Anthropic provider entries are safe to
+	// reference (dated snapshots resolve to their alias entry when present).
+	if (requestModel.provider === "anthropic") {
+		const bundled = getBundledModel("anthropic", id);
+		if (bundled?.api === "anthropic-messages") return bundled;
+	}
+	return requestModel;
+}
+
+function calculateFallbackTurnCost(
+	requestModel: Model<"anthropic-messages">,
+	usage: Usage,
+	source: AnthropicWireUsage,
+): boolean {
+	const iterations = source.iterations ?? [];
+	if (iterations.length === 0) return false;
+	const cost = { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 };
+	const hasFallbackMessage = iterations.some(iter => iter.type === "fallback_message");
+	let applied = false;
+	for (const iteration of iterations) {
+		const inputTokens = iteration.input_tokens ?? 0;
+		const outputTokens = iteration.output_tokens ?? 0;
+		const cacheReadTokens = iteration.cache_read_input_tokens ?? 0;
+		const cacheWriteTokens = iteration.cache_creation_input_tokens ?? 0;
+		const isFallback = iteration.type === "fallback_message";
+		if (hasFallbackMessage && !isFallback && outputTokens === 0 && cacheWriteTokens === 0) continue;
+		const iterationUsage = createEmptyUsage();
+		if (isFallback) {
+			iterationUsage.input = 0;
+			iterationUsage.cacheRead = cacheReadTokens + inputTokens;
+		} else {
+			iterationUsage.input = inputTokens;
+			iterationUsage.cacheRead = cacheReadTokens;
+		}
+		iterationUsage.output = outputTokens;
+		iterationUsage.cacheWrite = cacheWriteTokens;
+		iterationUsage.totalTokens =
+			iterationUsage.input + iterationUsage.output + iterationUsage.cacheRead + iterationUsage.cacheWrite;
+		calculateCost(resolveIterationModel(requestModel, iteration.model), iterationUsage);
+		cost.input += iterationUsage.cost.input;
+		cost.output += iterationUsage.cost.output;
+		cost.cacheRead += iterationUsage.cost.cacheRead;
+		cost.cacheWrite += iterationUsage.cost.cacheWrite;
+		cost.total += iterationUsage.cost.total;
+		applied = true;
+	}
+	if (!applied) return false;
+	usage.cost = cost;
+	return true;
+}
+
 const streamAnthropicOnce = (
 	model: Model<"anthropic-messages">,
 	context: Context,
@@ -1644,6 +1759,27 @@ const streamAnthropicOnce = (
 				) {
 					extraBetas.push(contextManagementBeta);
 				}
+				// Server-side fallback beta chain: opt-in via `options.fallbacks`.
+				// Nested overrides (`speed`, `output_config.effort`,
+				// `output_config.task_budget`) reuse the same top-level betas
+				// Anthropic requires for the primary request, so scan the chain
+				// and add every companion beta the fallback entries touch.
+				if (options?.fallbacks?.length) {
+					if (!extraBetas.includes(serverSideFallbackBeta)) {
+						extraBetas.push(serverSideFallbackBeta);
+					}
+					for (const entry of options.fallbacks) {
+						if (entry.speed === "fast" && !extraBetas.includes(fastModeBeta)) {
+							extraBetas.push(fastModeBeta);
+						}
+						if (entry.output_config?.effort && !extraBetas.includes(effortBeta)) {
+							extraBetas.push(effortBeta);
+						}
+						if (entry.output_config?.task_budget && !extraBetas.includes(taskBudgetBeta)) {
+							extraBetas.push(taskBudgetBeta);
+						}
+					}
+				}
 
 				const created = createClient(model, {
 					model,
@@ -1696,10 +1832,16 @@ const streamAnthropicOnce = (
 			};
 			let params = await prepareParams();
 
+			// Opt-in flag: the response parser only honors `fallback` content
+			// blocks and `usage.iterations` when the current request opted into
+			// the server-side-fallback beta chain. Leaving `options.fallbacks`
+			// unset preserves the pre-fallback stream shape on every event.
+			const serverSideFallback = !!options?.fallbacks?.length;
 			type Block = (
 				| ThinkingContent
 				| RedactedThinkingContent
 				| TextContent
+				| AnthropicFallbackContent
 				| (ToolCall & { [kStreamingPartialJson]: string; [kStreamingLastParseLen]?: number })
 			) & { [kStreamingBlockIndex]: number };
 			const idleTimeoutMs = options?.streamIdleTimeoutMs ?? getStreamIdleTimeoutMs();
@@ -1815,7 +1957,10 @@ const streamAnthropicOnce = (
 					const closedBlockIndexes = new Set<number>();
 					const openBlocks = new Map<
 						number,
-						{ contentIndex: number; kind: "text" | "thinking" | "redactedThinking" | "toolCall" | "ignored" }
+						{
+							contentIndex: number;
+							kind: "text" | "thinking" | "redactedThinking" | "fallback" | "toolCall" | "ignored";
+						}
 					>();
 
 					// Pings keep the idle deadline alive once content is flowing, but a
@@ -1866,7 +2011,15 @@ const streamAnthropicOnce = (
 								output.usage.cacheWrite = startUsage.cache_creation_input_tokens || 0;
 								output.usage.totalTokens =
 									output.usage.input + output.usage.output + output.usage.cacheRead + output.usage.cacheWrite;
-								calculateCost(model, output.usage);
+								if (serverSideFallback) {
+									const served = fallbackServedModelFromUsage(startUsage);
+									if (served) output.model = served;
+									if (!calculateFallbackTurnCost(model, output.usage, startUsage)) {
+										calculateCost(model, output.usage);
+									}
+								} else {
+									calculateCost(model, output.usage);
+								}
 							} else {
 								reportAnthropicEnvelopeAnomaly("message_start missing usage");
 							}
@@ -1904,6 +2057,33 @@ const streamAnthropicOnce = (
 								continue;
 							}
 							if (!firstTokenTime) firstTokenTime = performance.now();
+							if (event.content_block.type === "fallback") {
+								// Fallback boundary is only meaningful when the request
+								// opted into the beta chain — silently drop otherwise so
+								// unopted-in sessions never see the block persisted or
+								// influence downstream converters.
+								const fallback = parseAnthropicFallbackWireBlock(event.content_block);
+								if (!serverSideFallback || !fallback) {
+									if (!fallback) {
+										reportAnthropicEnvelopeAnomaly("fallback content_block missing model refs");
+									}
+									openBlocks.set(event.index, { contentIndex: -1, kind: "ignored" });
+									continue;
+								}
+								const block: Block = { ...fallback, [kStreamingBlockIndex]: event.index };
+								output.content.push(block);
+								openBlocks.set(event.index, {
+									contentIndex: output.content.length - 1,
+									kind: "fallback",
+								});
+								// A fallback content block is the mid-stream signal that a
+								// classifier block on the primary was retried on the
+								// fallback model. Adopt the served id immediately so
+								// pricing decisions downstream (final usage.iterations may
+								// arrive before/after) see the right model.
+								output.model = fallback.to.model;
+								continue;
+							}
 							if (event.content_block.type === "text") {
 								streamedReplayUnsafeContent = true;
 								const block: Block = {
@@ -2119,7 +2299,15 @@ const streamAnthropicOnce = (
 								applyAnthropicUsageExtras(output.usage, deltaUsage);
 								output.usage.totalTokens =
 									output.usage.input + output.usage.output + output.usage.cacheRead + output.usage.cacheWrite;
-								calculateCost(model, output.usage);
+								if (serverSideFallback) {
+									const served = fallbackServedModelFromUsage(deltaUsage);
+									if (served) output.model = served;
+									if (!calculateFallbackTurnCost(model, output.usage, deltaUsage)) {
+										calculateCost(model, output.usage);
+									}
+								} else {
+									calculateCost(model, output.usage);
+								}
 							}
 						} else if (event.type === "message_stop") {
 							sawTerminalEnvelope = true;
@@ -2180,6 +2368,7 @@ const streamAnthropicOnce = (
 						params = await prepareParams();
 						providerRetryAttempt = 0;
 						output.content.length = 0;
+						output.model = model.id;
 						output.responseId = undefined;
 						output.errorMessage = undefined;
 						output.providerPayload = undefined;
@@ -2206,6 +2395,7 @@ const streamAnthropicOnce = (
 						params = await prepareParams();
 						providerRetryAttempt = 0;
 						output.content.length = 0;
+						output.model = model.id;
 						output.responseId = undefined;
 						output.errorMessage = undefined;
 						output.providerPayload = undefined;
@@ -2248,6 +2438,7 @@ const streamAnthropicOnce = (
 						await scheduler.wait(delayMs, { signal: options?.signal });
 					}
 					output.content.length = 0;
+					output.model = model.id;
 					output.responseId = undefined;
 					output.errorMessage = undefined;
 					output.stopDetails = undefined;
@@ -2960,7 +3151,9 @@ function buildParams(
 	// metadata → max_tokens → thinking → context_management → output_config → stream.
 	const params: MessageCreateParamsStreaming = {
 		model: options?.requestModelId ?? model.requestModelId ?? model.id,
-		messages: convertAnthropicMessages(context.messages, model, isOAuthToken),
+		messages: convertAnthropicMessages(context.messages, model, isOAuthToken, {
+			serverSideFallbackEnabled: !!options?.fallbacks?.length,
+		}),
 		...(systemBlocks && { system: systemBlocks }),
 		...(tools !== undefined && { tools }),
 		...(metadata && { metadata }),
@@ -2968,6 +3161,7 @@ function buildParams(
 		...(thinking && { thinking }),
 		...(contextManagement && { context_management: contextManagement }),
 		...(outputConfig && { output_config: outputConfig }),
+		...(options?.fallbacks?.length ? { fallbacks: options.fallbacks } : {}),
 		stream: true,
 	};
 
@@ -3121,10 +3315,20 @@ function toWellFormedDeep(value: unknown): unknown {
 	return value;
 }
 
+/**
+ * Serialize omp {@link Message}s to Anthropic wire messages.
+ *
+ * `opts.serverSideFallbackEnabled` — when the CURRENT request itself
+ * opts into the server-side-fallback beta chain. Only then may a persisted
+ * `fallback` content block from a prior turn be replayed on the wire;
+ * otherwise the block is dropped to avoid a 400 on non-fallback requests
+ * that don't send the beta.
+ */
 export function convertAnthropicMessages(
 	messages: Message[],
 	model: Model<"anthropic-messages">,
 	isOAuthToken: boolean,
+	opts?: { serverSideFallbackEnabled?: boolean },
 ): AnthropicMessageParam[] {
 	// Indices of params emitted from `developer` messages. After the main pass,
 	// the ones whose placement satisfies Anthropic's mid-conversation rules are
@@ -3214,6 +3418,19 @@ export function convertAnthropicMessages(
 						type: "redacted_thinking",
 						data: block.data,
 					});
+				} else if (block.type === "fallback") {
+					// Replay ONLY when both sides are aligned: the current
+					// request opted into the beta chain, and the target is
+					// official Anthropic (the only endpoint that accepts the
+					// block on the wire). `transformMessages` already drops
+					// the block for cross-provider / non-official replays, so
+					// this is defense-in-depth for direct convert calls.
+					if (!opts?.serverSideFallbackEnabled || !model.compat.officialEndpoint) continue;
+					blocks.push({
+						type: "fallback",
+						from: block.from,
+						to: block.to,
+					});
 				} else if (block.type === "toolCall") {
 					blocks.push({
 						type: "tool_use",

package/src/providers/azure-openai-responses.ts

@@ -340,6 +340,7 @@ function buildParams(
 		systemRole,
 		includeThinkingSignatures: true,
 		developerStringContent: true,
+		preserveAssistantMessageIds: true,
 	});
 
 	const params: AzureOpenAIResponsesSamplingParams = {
@@ -375,7 +376,7 @@ function buildParams(
 		}
 	}
 
-	applyResponsesReasoningParams(params, model, options, messages);
+	applyResponsesReasoningParams(params, model, options);
 
 	return params;
 }

package/src/providers/openai-responses.ts

@@ -166,8 +166,8 @@ interface OpenAIResponsesProviderSessionState
 
 interface OpenAIResponsesChainState {
 	/**
-	 * Wire params of the last successful turn, with per-turn trailing
-	 * scaffolding stripped from `input` (never carries previous_response_id).
+	 * Wire params of the last successful turn; never carries
+	 * `previous_response_id`.
 	 */
 	lastParams?: OpenAIResponsesSamplingParams;
 	lastResponseId?: string;
@@ -256,30 +256,19 @@ interface OpenAIResponsesChainedParams {
  * (same options, strict history prefix), chain via `previous_response_id` +
  * delta-only `input`; otherwise break the chain and replay the full transcript.
  *
- * The prefix check runs on the wire form of the conversation arguments alone:
- * per-turn trailing scaffolding is excluded from both sides and re-appended to
- * the delta, so a decoration that trails every request can never masquerade as
- * a history mutation.
+ * The prefix check runs on the wire form of the conversation arguments, so
+ * history mutations or option changes force a full replay.
  */
 function buildOpenAIResponsesChainedParams(
 	params: OpenAIResponsesSamplingParams,
-	trailingScaffoldingItems: number,
 	chain: OpenAIResponsesChainState,
 ): OpenAIResponsesChainedParams {
-	const historyParams =
-		trailingScaffoldingItems > 0 && Array.isArray(params.input)
-			? { ...params, input: params.input.slice(0, params.input.length - trailingScaffoldingItems) }
-			: params;
 	const deltaInput = chain.canAppend
-		? buildResponsesDeltaInput(chain.lastParams, chain.lastResponseItems, historyParams)
+		? buildResponsesDeltaInput(chain.lastParams, chain.lastResponseItems, params)
 		: null;
 	if (deltaInput && deltaInput.length > 0 && chain.lastResponseId) {
-		const scaffolding =
-			historyParams !== params && Array.isArray(params.input)
-				? params.input.slice(params.input.length - trailingScaffoldingItems)
-				: [];
 		return {
-			params: { ...params, previous_response_id: chain.lastResponseId, input: [...deltaInput, ...scaffolding] },
+			params: { ...params, previous_response_id: chain.lastResponseId, input: deltaInput },
 			previousResponseId: chain.lastResponseId,
 		};
 	}
@@ -416,9 +405,7 @@ const streamOpenAIResponsesOnce = (
 			const strictToolsScope = getOpenAIStrictToolsScope(model, baseUrl);
 			const builtParams = buildParams(model, context, options, providerSessionState, strictToolsScope);
 			const params = builtParams.params;
-			const { trailingScaffoldingItems } = builtParams;
 			let activeParams = params;
-			let activeTrailingScaffoldingItems = trailingScaffoldingItems;
 			const resolvedBaseUrl = (baseUrl ?? "https://api.openai.com/v1").replace(/\/+$/, "");
 			const requestReasoningEffortFallbacks = new Map<string, OpenAIReasoningEffortFallback>();
 			const attemptedReasoningEffortFallbacks = new Set<string>();
@@ -448,9 +435,7 @@ const streamOpenAIResponsesOnce = (
 			}
 			applyReasoningEffortFallbackForRequest(params);
 			let chained: OpenAIResponsesChainedParams =
-				chainState && !chainState.disabled
-					? buildOpenAIResponsesChainedParams(params, trailingScaffoldingItems, chainState)
-					: { params };
+				chainState && !chainState.disabled ? buildOpenAIResponsesChainedParams(params, chainState) : { params };
 			sentPreviousResponseId = chained.previousResponseId;
 			const idleTimeoutMs =
 				options?.streamIdleTimeoutMs ?? getOpenAIStreamIdleTimeoutMs(model.compat.streamIdleTimeoutMs);
@@ -589,11 +574,7 @@ const streamOpenAIResponsesOnce = (
 						if (chainState && !chainState.disabled) fallbackParams.store = true;
 						let fallbackChained: OpenAIResponsesChainedParams =
 							chainState && !chainState.disabled
-								? buildOpenAIResponsesChainedParams(
-										fallbackParams,
-										fallbackBuilt.trailingScaffoldingItems,
-										chainState,
-									)
+								? buildOpenAIResponsesChainedParams(fallbackParams, chainState)
 								: { params: fallbackParams };
 						sentPreviousResponseId = fallbackChained.previousResponseId;
 						fallbackChained = {
@@ -603,7 +584,6 @@ const streamOpenAIResponsesOnce = (
 						chained = fallbackChained;
 						rawRequestDump.body = chained.params;
 						activeParams = fallbackParams;
-						activeTrailingScaffoldingItems = fallbackBuilt.trailingScaffoldingItems;
 						activeStrictToolsApplied = fallbackBuilt.strictToolsApplied;
 						continue;
 					}
@@ -646,7 +626,6 @@ const streamOpenAIResponsesOnce = (
 					chained = { params: retryParams };
 					rawRequestDump.body = retryParams;
 					activeParams = currentParams;
-					activeTrailingScaffoldingItems = currentBuilt.trailingScaffoldingItems;
 					activeStrictToolsApplied = currentBuilt.strictToolsApplied;
 				}
 			}
@@ -710,14 +689,7 @@ const streamOpenAIResponsesOnce = (
 			output.providerPayload = createOpenAIResponsesHistoryPayload(model.provider, nativeOutputItems);
 			if (providerSessionState) providerSessionState.nativeHistoryReplayWarmed = true;
 			if (chainState) {
-				chainState.lastParams = structuredCloneJSON(
-					activeTrailingScaffoldingItems > 0 && Array.isArray(activeParams.input)
-						? {
-								...activeParams,
-								input: activeParams.input.slice(0, activeParams.input.length - activeTrailingScaffoldingItems),
-							}
-						: activeParams,
-				);
+				chainState.lastParams = structuredCloneJSON(activeParams);
 				if (output.responseId) {
 					chainState.lastResponseId = output.responseId;
 					chainState.lastResponseItems = sanitizeOpenAIResponsesHistoryItemsForReplay(
@@ -790,7 +762,7 @@ export function buildParams(
 	providerSessionState: OpenAIResponsesProviderSessionState | undefined,
 	strictToolsScope?: OpenAIStrictToolsScope,
 	disableStrictToolsOverride = false,
-): { params: OpenAIResponsesSamplingParams; trailingScaffoldingItems: number; strictToolsApplied: boolean } {
+): { params: OpenAIResponsesSamplingParams; strictToolsApplied: boolean } {
 	const policy = resolveOpenAICompatPolicy(model, {
 		endpoint: "responses",
 		reasoning: options?.reasoning,
@@ -914,7 +886,7 @@ export function buildParams(
 		filterReasoningHistory: options?.filterReasoningHistory,
 		omitReasoningEffort: options?.omitReasoningEffort,
 	});
-	const trailingScaffoldingItems = applyResponsesCompatPolicy(params, messages, reasoningPolicy, {
+	applyResponsesCompatPolicy(params, reasoningPolicy, {
 		reasoningSummary: options?.reasoningSummary,
 		mapEffort: effort =>
 			model.compat.reasoningEffortMap?.[effort as NonNullable<OpenAIResponsesOptions["reasoning"]>] ??
@@ -926,7 +898,7 @@ export function buildParams(
 
 	applyOpenAIExtraBody(params, options?.extraBody);
 
-	return { params, trailingScaffoldingItems, strictToolsApplied };
+	return { params, strictToolsApplied };
 }
 
 /**