@providerkit/core 0.17.2 → 0.20.0

package/src/providers/responses.ts

@@ -20,6 +20,7 @@ import type {
   ImagePart,
   Provider,
   ProviderChunk,
+  ServiceTier,
   StreamOptions,
   ToolDefinition,
 } from "../types.ts";
@@ -56,6 +57,12 @@ export interface ResponsesConfig extends ProviderFallbackConfig {
   /** What this backend accepts as an image. Each image it would refuse is
    *  replaced by a note naming the reason, instead of failing the request. */
   imageLimits?: ImageLimits;
+  /** Ask for the encrypted reasoning items and replay them on the next turn,
+   *  so the model keeps its chain of thought across a stateless turn. Only for
+   *  backends that accept `include: ["reasoning.encrypted_content"]`. */
+  replayReasoning?: boolean;
+  /** Default `service_tier` for every call; `StreamOptions.serviceTier` wins. */
+  serviceTier?: ServiceTier;
 }
 
 const DEFAULT_BASE_URL = "https://api.openai.com";
@@ -91,7 +98,33 @@ type ResponsesContentPart =
   | { type: "output_text"; text: string }
   | { type: "input_image"; image_url: string };
 
+/** A reasoning item the backend made, sent back as it came. */
+type ReplayedReasoning = { type: "reasoning" } & Record<string, unknown>;
+
+function isReplayedReasoning(item: unknown): item is ReplayedReasoning {
+  return typeof item === "object" && item !== null && "type" in item && item.type === "reasoning";
+}
+
+/** The most encrypted reasoning one request carries back. Newest turns win. */
+const MAX_REPLAY_BYTES = 8 * 1024 * 1024;
+
+/** Which assistant messages get their reasoning replayed: this provider's own,
+ *  newest first, until the cap. An older turn's reasoning is the cheapest to lose. */
+function replayable(messages: readonly ChatMessage[], provider: string): Set<ChatMessage> {
+  const keep = new Set<ChatMessage>();
+  let bytes = 0;
+  for (let i = messages.length - 1; i >= 0; i--) {
+    const message = messages[i]!;
+    if (message.role !== "assistant" || message.reasoningItems?.provider !== provider) continue;
+    bytes += JSON.stringify(message.reasoningItems.items).length;
+    if (bytes > MAX_REPLAY_BYTES) break;
+    keep.add(message);
+  }
+  return keep;
+}
+
 type ResponsesInputItem =
+  | ReplayedReasoning
   | { type: "message"; role: "user" | "assistant"; content: ResponsesContentPart[] }
   | { type: "function_call"; call_id: string; name: string; arguments: string }
   | { type: "function_call_output"; call_id: string; output: string | ResponsesContentPart[] };
@@ -141,6 +174,7 @@ export function toResponsesInput(
     .map((message) => message.content)
     .join("\n\n");
 
+  const replayed = replayable(messages, provider);
   const input: ResponsesInputItem[] = [];
   for (const message of messages) {
     switch (message.role) {
@@ -167,12 +201,15 @@ export function toResponsesInput(
         break;
 
       case "assistant": {
-        // Reasoning is deliberately NOT replayed. This shape wants the ORIGINAL
-        // reasoning item back — its `rs_…` id, and under `store: false` its
-        // `encrypted_content` blob — and the seam carries neither, only the
-        // plain summary text a caller renders. A synthesized reasoning item is
-        // rejected; omitting it costs only the model re-deriving its own chain
-        // of thought, which is what every stateless caller already lives with.
+        // `reasoning` (the summary a caller renders) is never replayed: this
+        // shape wants the ORIGINAL item back, `rs_…` id and `encrypted_content`
+        // blob included, and a synthesized one is rejected. `reasoningItems`
+        // are those originals, so they go back first, but only to the provider
+        // that made them. Without them the model re-derives its own chain of
+        // thought, which is what every stateless caller already lives with.
+        if (replayed.has(message)) {
+          input.push(...message.reasoningItems!.items.filter(isReplayedReasoning));
+        }
         if (message.content) {
           input.push({
             type: "message",
@@ -213,6 +250,7 @@ interface ResponsesItem {
   call_id?: string;
   name?: string;
   arguments?: string;
+  encrypted_content?: string;
 }
 
 interface ResponsesEvent {
@@ -350,6 +388,7 @@ export function createResponsesProvider(config: ResponsesConfig): Provider {
         store: false,
       };
       if (instructions) request.instructions = instructions;
+      if (config.replayReasoning) request.include = ["reasoning.encrypted_content"];
       const maxTokens = opts.maxTokens ?? config.maxTokens;
       if (maxTokens !== undefined) request.max_output_tokens = maxTokens;
       if (opts.temperature !== undefined) request.temperature = opts.temperature;
@@ -386,6 +425,8 @@ export function createResponsesProvider(config: ResponsesConfig): Provider {
       // The cache affinity key. Without one the backend spreads a
       // conversation's turns across cache shards, and a re-sent prefix misses.
       if (opts.sessionId) request.prompt_cache_key = opts.sessionId;
+      const serviceTier = opts.serviceTier ?? config.serviceTier;
+      if (serviceTier) request.service_tier = serviceTier;
       // Muse Contributor accepts only `auto`. `none` has an exact wire-level
       // equivalent: send no tools. Required/named choices stay explicit and let
       // the endpoint refuse a promise it cannot keep.
@@ -421,6 +462,11 @@ export function createResponsesProvider(config: ResponsesConfig): Provider {
       let sawToolCall = false;
       let wall: ProviderError | undefined;
       let produced = false;
+      const reasoningItems: unknown[] = [];
+      const reasoningChunk = (): ProviderChunk[] =>
+        reasoningItems.length
+          ? [{ type: "delta", reasoningItems: { provider: id, items: reasoningItems } }]
+          : [];
 
       for await (const data of streamSse({
         url: apiUrl(baseUrl, config.path ?? DEFAULT_PATH),
@@ -522,6 +568,12 @@ export function createResponsesProvider(config: ResponsesConfig): Provider {
 
           case "response.output_item.done": {
             const item = event.item;
+            // Kept whole and emitted once, at the end of the turn: it is the
+            // backend's own record, and it only means something complete.
+            if (config.replayReasoning && item?.type === "reasoning") {
+              reasoningItems.push(item);
+              break;
+            }
             if (item?.type !== "function_call") break;
             const known = pendingFor(item.id, item.call_id);
             if (item.id) pending.delete(item.id);
@@ -583,6 +635,7 @@ export function createResponsesProvider(config: ResponsesConfig): Provider {
             if (response?.status === "failed" || response?.error) {
               throw streamError(id, response.error ?? { message: "response failed" });
             }
+            yield* reasoningChunk();
             if (response?.usage) yield usageChunk(response.usage);
             if (response?.status === "incomplete") {
               yield {
@@ -602,6 +655,7 @@ export function createResponsesProvider(config: ResponsesConfig): Provider {
           case "response.incomplete": {
             // A turn that ran out of output tokens still billed for its input,
             // and this event carries usage in the same shape as `completed`.
+            yield* reasoningChunk();
             const usage = event.response?.usage;
             if (usage) yield usageChunk(usage);
             yield {

package/src/types.ts

@@ -108,6 +108,14 @@ export type ChatMessage =
        * continuity across a tool round. Absent on every other dialect.
        */
       reasoningDetails?: unknown[];
+      /**
+       * The ChatGPT backend's own reasoning items, encrypted, riding back
+       * UNMODIFIED on the next turn so the model keeps its chain of thought.
+       * Tagged with the provider that made them: only that provider replays
+       * them, because another vendor (after a fallback) would reject them.
+       * Opaque, like `reasoningDetails`.
+       */
+      reasoningItems?: ReasoningItems;
       toolCalls?: ToolCall[];
     }
   | {
@@ -119,6 +127,12 @@ export type ChatMessage =
       images?: ImagePart[];
     };
 
+/** Provider-owned reasoning items and the id of the provider that made them. */
+export interface ReasoningItems {
+  provider: string;
+  items: unknown[];
+}
+
 /** JSON Schema for an object — what every provider's tool contract wants. */
 export interface JsonObjectSchema {
   type: "object";
@@ -192,6 +206,8 @@ export interface ProviderChunk {
   /** OpenRouter's normalized reasoning payload — hand it back on the next
    *  turn's assistant message verbatim. See ChatMessage.reasoningDetails. */
   reasoningDetails?: unknown[];
+  /** Once per turn, where a provider replays its reasoning. See ChatMessage.reasoningItems. */
+  reasoningItems?: ReasoningItems;
   toolCalls?: ToolCallDelta[];
   usage?: TokenUsage;
   finishReason?: FinishReason;
@@ -387,8 +403,19 @@ export interface StreamOptions {
    * so it refuses `false` rather than silently ignore it.
    */
   parallelToolCalls?: boolean;
+  /**
+   * Which service tier to buy this call on: `priority` is faster and costs
+   * more, `flex` is slower and costs less. Sent as `service_tier` on the
+   * OpenAI and Responses shapes. Anthropic and Gemini refuse it with an
+   * `invalid` error: Anthropic's own `service_tier` means something else, and
+   * Gemini has no such switch. Overrides the provider's own setting.
+   */
+  serviceTier?: ServiceTier;
 }
 
+// ponytail: Codex's `ultrafast` tier (gpt-6-astra only) is the next value to add.
+export type ServiceTier = "priority" | "flex";
+
 export interface Provider {
   readonly id: string;
   readonly model: string;
@@ -406,6 +433,8 @@ export interface Completion {
   reasoning: string;
   /** Present only where the provider sent one. See ChatMessage.reasoningDetails. */
   reasoningDetails?: unknown[];
+  /** Present only where the provider sent them. See ChatMessage.reasoningItems. */
+  reasoningItems?: ReasoningItems;
   usage: TokenUsage;
   finishReason: FinishReason | null;
   model: string;
@@ -424,6 +453,7 @@ export async function drainStream(
   // a drained turn replays only half its own reasoning on the next round —
   // which is the failure `reasoningDetails` exists to prevent.
   let reasoningDetails: unknown[] | undefined;
+  let reasoningItems: ReasoningItems | undefined;
   let usage: TokenUsage = EMPTY_USAGE;
   let finishReason: FinishReason | null = null;
   let source: ProviderChunk["source"];
@@ -433,6 +463,7 @@ export async function drainStream(
       if (chunk.content) text += chunk.content;
       if (chunk.reasoning) reasoning += chunk.reasoning;
       if (chunk.reasoningDetails?.length) reasoningDetails = chunk.reasoningDetails;
+      if (chunk.reasoningItems?.items.length) reasoningItems = chunk.reasoningItems;
     } else if (chunk.type === "usage" && chunk.usage) {
       usage = chunk.usage;
     } else if (chunk.type === "finish" && chunk.finishReason) {
@@ -443,6 +474,7 @@ export async function drainStream(
     text,
     reasoning,
     ...(reasoningDetails ? { reasoningDetails } : {}),
+    ...(reasoningItems ? { reasoningItems } : {}),
     usage,
     finishReason,
     model: source?.model ?? model,
@@ -465,11 +497,23 @@ export async function drainStream(
 export function stripReasoning(messages: readonly ChatMessage[]): ChatMessage[] {
   return messages.map((message) => {
     if (message.role !== "assistant") return message;
-    if (message.reasoning === undefined && message.reasoningDetails === undefined) return message;
-    // Both halves go. `reasoningDetails` is the same chain of thought in the
-    // provider's own words, so leaving it behind carries into a thinking-off
-    // turn exactly what stripping `reasoning` was meant to keep out.
-    const { reasoning: _text, reasoningDetails: _payload, ...rest } = message;
+    if (
+      message.reasoning === undefined &&
+      message.reasoningDetails === undefined &&
+      message.reasoningItems === undefined
+    ) {
+      return message;
+    }
+    // Every half goes. `reasoningDetails` and `reasoningItems` are the same
+    // chain of thought in the provider's own words, so leaving them behind
+    // carries into a thinking-off turn exactly what stripping `reasoning` was
+    // meant to keep out.
+    const {
+      reasoning: _text,
+      reasoningDetails: _payload,
+      reasoningItems: _items,
+      ...rest
+    } = message;
     return rest;
   });
 }