@tanstack/openai-base 0.10.15 → 0.11.1

package/src/adapters/responses-text.ts

@@ -1,4 +1,10 @@
-import { EventType, normalizeSystemPrompts } from '@tanstack/ai'
+import {
+  EventType,
+  fileReferenceFor,
+  isFileSource,
+  normalizeSystemPrompts,
+  unsupportedFileSourceError,
+} from '@tanstack/ai'
 import { BaseTextAdapter } from '@tanstack/ai/adapters'
 import {
   toRunErrorPayload,
@@ -11,6 +17,13 @@ import { createToolInputNormalizer } from '../utils/tool-input-normalizer'
 import type { StructuredOutputCompatibility } from '../utils/schema-converter'
 import { buildResponsesUsage } from '../usage'
 import { convertToolsToResponsesFormat } from './responses-tool-converter'
+import {
+  hostedShellCallIds,
+  readUserExecutedCall,
+  userToolRequestItem,
+  userToolResultItem,
+} from './responses-user-tools'
+import type { OpenAIUserToolName } from './responses-user-tools'
 import type OpenAI from 'openai'
 import type {
   StructuredOutputOptions,
@@ -98,7 +111,11 @@ function readReasoningItem(
  * item ID here so stateless follow-up requests can replay both values.
  */
 export interface OpenAIResponsesToolCallMetadata {
-  itemId: string
+  itemId?: string
+  /** Set for shell, local_shell, and apply_patch calls the app must run. */
+  openaiUserTool?: OpenAIUserToolName
+  /** Shell `action.max_output_length`, echoed on `shell_call_output`. */
+  maxOutputLength?: number | null
 }
 
 interface StreamedFunctionCallMetadata {
@@ -979,6 +996,56 @@ export abstract class OpenAIBaseResponsesTextAdapter<
       accumulatedReasoning = ''
     }
 
+    const userToolChunks = (
+      item: unknown,
+      outputIndex: number,
+      bareShell: boolean,
+    ): Array<AdapterYieldChunk> => {
+      const call = readUserExecutedCall(item, { bareShell })
+      if (!call || toolCallMetadata.get(call.callId)?.ended) return []
+      const tracked = toolCallMetadata.get(call.callId) ?? {
+        callId: call.callId,
+        index: outputIndex,
+        name: call.name,
+        started: false,
+      }
+      toolCallMetadata.set(call.callId, tracked)
+      tracked.started = true
+      tracked.ended = true
+      tracked.name = call.name
+      const timestamp = Date.now()
+      const modelName = model || options.model
+      const metadata: OpenAIResponsesToolCallMetadata = {
+        openaiUserTool: call.name,
+        ...(call.itemId ? { itemId: call.itemId } : {}),
+        ...(call.maxOutputLength !== undefined
+          ? { maxOutputLength: call.maxOutputLength }
+          : {}),
+      }
+      return [
+        {
+          type: EventType.TOOL_CALL_START,
+          toolCallId: call.callId,
+          toolCallName: call.name,
+          toolName: call.name,
+          parentMessageId: aguiState.messageId,
+          model: modelName,
+          timestamp,
+          index: outputIndex,
+          metadata,
+        },
+        {
+          type: EventType.TOOL_CALL_END,
+          toolCallId: call.callId,
+          toolCallName: call.name,
+          toolName: call.name,
+          model: modelName,
+          timestamp,
+          input: call.input,
+        },
+      ]
+    }
+
     const emitReasoningDelta = function* (
       text: string,
     ): Generator<AdapterYieldChunk> {
@@ -1341,6 +1408,7 @@ export abstract class OpenAIBaseResponsesTextAdapter<
               metadata.started = true
             }
           }
+          yield* userToolChunks(item, chunk.output_index, false)
         }
 
         // Handle function call arguments delta (streaming). Drop the
@@ -1545,6 +1613,7 @@ export abstract class OpenAIBaseResponsesTextAdapter<
               metadata.pendingArguments = undefined
             }
           }
+          yield* userToolChunks(item, chunk.output_index, false)
         }
 
         if (chunk.type === 'response.completed') {
@@ -1621,11 +1690,11 @@ export abstract class OpenAIBaseResponsesTextAdapter<
           // be silently dropped from the AG-UI stream while `hasFunctionCalls`
           // below still routes the run's finishReason to 'tool_calls' —
           // leaving consumers waiting for tool results they never saw start.
-          for (const item of chunk.response.output) {
+          for (const [outputIndex, item] of chunk.response.output.entries()) {
             if (item.type !== 'function_call' || !item.id) continue
             const metadata = toolCallMetadata.get(item.id) ?? {
               callId: item.call_id || item.id,
-              index: 0,
+              index: outputIndex,
               name: item.name || '',
               started: false,
             }
@@ -1697,6 +1766,19 @@ export abstract class OpenAIBaseResponsesTextAdapter<
             }
           }
 
+          const shellOutputs = hostedShellCallIds(chunk.response.output)
+          for (const [outputIndex, item] of chunk.response.output.entries()) {
+            if (
+              isRecord(item) &&
+              item.type === 'shell_call' &&
+              typeof item.call_id === 'string' &&
+              shellOutputs.has(item.call_id)
+            ) {
+              continue
+            }
+            yield* userToolChunks(item, outputIndex, true)
+          }
+
           yield* closeReasoning()
           // Emit TEXT_MESSAGE_END if we had text content
           if (hasEmittedTextMessageStart) {
@@ -1716,10 +1798,18 @@ export abstract class OpenAIBaseResponsesTextAdapter<
           // The Responses API's incomplete_details.reason ('max_output_tokens'
           // | 'content_filter') maps to the AG-UI finishReason vocabulary:
           // max_output_tokens → 'length', content_filter → 'content_filter'.
-          const hasFunctionCalls = chunk.response.output.some(
-            (item: unknown) =>
-              (item as { type: string }).type === 'function_call',
-          )
+          const hasFunctionCalls = chunk.response.output.some((item) => {
+            if (!isRecord(item)) return false
+            if (item.type === 'function_call') return true
+            if (
+              item.type === 'shell_call' &&
+              typeof item.call_id === 'string' &&
+              shellOutputs.has(item.call_id)
+            ) {
+              return false
+            }
+            return readUserExecutedCall(item, { bareShell: true }) !== null
+          })
           const incompleteReason = chunk.response.incomplete_details?.reason
           const finishReason:
             | 'tool_calls'
@@ -1923,10 +2013,33 @@ export abstract class OpenAIBaseResponsesTextAdapter<
     messages: Array<ModelMessage>,
   ): ResponseInput {
     const result: ResponseInput = []
+    // A reasoning item id may only appear once in `input`; replaying the same
+    // id twice fails with "Duplicate item found with id rs_...". Providers have
+    // been observed minting one id for two separate reasoning items, and a
+    // stored transcript can end up carrying it on two assistant messages.
+    const seenReasoningIds = new Set<string>()
+    const userToolCalls = new Map<
+      string,
+      {
+        id: string
+        function: { name: string; arguments: string }
+        metadata?: unknown
+      }
+    >()
 
     for (const message of messages) {
       // Handle tool messages - convert to FunctionToolCallOutput
       if (message.role === 'tool') {
+        const owner = message.toolCallId
+          ? userToolCalls.get(message.toolCallId)
+          : undefined
+        const userOutput = owner
+          ? userToolResultItem(owner, message.content)
+          : null
+        if (userOutput) {
+          result.push(userOutput)
+          continue
+        }
         const toolContent = message.content
         const output: string | Array<ResponseFunctionCallOutputItem> =
           Array.isArray(toolContent)
@@ -1944,11 +2057,21 @@ export abstract class OpenAIBaseResponsesTextAdapter<
 
       // Handle assistant messages
       if (message.role === 'assistant') {
+        // Every reasoning item this message could contribute, and how many of
+        // them actually made it into `input` after de-duplication. Both counts
+        // are needed below to decide whether the function calls can still be
+        // paired with their reasoning.
+        let reasoningCandidates = 0
+        let emittedReasoning = 0
         if (message.thinking) {
           for (const thinking of message.thinking) {
             if (!thinking.signature) continue
             const packed = unpackResponsesReasoningSignature(thinking.signature)
             if (!packed?.id) continue
+            reasoningCandidates++
+            if (seenReasoningIds.has(packed.id)) continue
+            seenReasoningIds.add(packed.id)
+            emittedReasoning++
             result.push({
               type: 'reasoning',
               id: packed.id,
@@ -1962,15 +2085,50 @@ export abstract class OpenAIBaseResponsesTextAdapter<
           }
         }
 
+        // The Responses API requires a persisted `function_call` item to sit
+        // directly after the reasoning item that produced it. A ModelMessage
+        // stores reasoning and tool calls as two flat arrays, so their original
+        // interleaving is gone: a message holding two or more reasoning items
+        // replays as reasoning A, reasoning B, call A, call B, and the request
+        // is rejected with "Item 'fc_...' of type 'function_call' was provided
+        // without its required 'reasoning' item: 'rs_...'". The same happens
+        // when this message's reasoning was dropped above as a duplicate.
+        //
+        // Sending the calls without their item id makes them fresh items that
+        // the API does not try to pair, which replays correctly. `call_id` is
+        // untouched, so the matching `function_call_output` still resolves.
+        // One reasoning item (or none at all) always lands adjacent to its
+        // calls, so those keep their ids and their prompt-cache hits. If any
+        // reasoning was dropped as a duplicate, a call may belong to it, so
+        // the ids go too.
+        const canPairReasoning =
+          reasoningCandidates === 0 ||
+          (reasoningCandidates === 1 && emittedReasoning === 1)
+
         // If the assistant message has tool calls, add them as FunctionToolCall objects
         // Responses API expects arguments as a string (JSON string)
         if (message.toolCalls && message.toolCalls.length > 0) {
           for (const toolCall of message.toolCalls) {
-            // Keep arguments as string for Responses API
             const argumentsString =
               typeof toolCall.function.arguments === 'string'
                 ? toolCall.function.arguments
                 : JSON.stringify(toolCall.function.arguments)
+            const replayCall = {
+              id: toolCall.id,
+              function: {
+                name: toolCall.function.name,
+                arguments: argumentsString,
+              },
+              ...(toolCall.metadata !== undefined
+                ? { metadata: toolCall.metadata }
+                : {}),
+            }
+            const userItem = userToolRequestItem(replayCall)
+            if (userItem) {
+              userToolCalls.set(toolCall.id, replayCall)
+              result.push(userItem)
+              continue
+            }
             const itemId = (
               toolCall.metadata as OpenAIResponsesToolCallMetadata | undefined
             )?.itemId
@@ -1978,7 +2136,7 @@ export abstract class OpenAIBaseResponsesTextAdapter<
             result.push({
               type: 'function_call',
               call_id: toolCall.id,
-              ...(itemId && { id: itemId }),
+              ...(itemId && canPairReasoning && { id: itemId }),
               name: toolCall.function.name,
               arguments: argumentsString,
             })
@@ -2046,6 +2204,16 @@ export abstract class OpenAIBaseResponsesTextAdapter<
         const imageMetadata = part.metadata as
           | { detail?: 'auto' | 'low' | 'high' }
           | undefined
+        if (isFileSource(part.source)) {
+          if (this.supportsFileSources !== true) {
+            throw unsupportedFileSourceError(this.name)
+          }
+          return {
+            type: 'input_image',
+            file_id: fileReferenceFor(part.source, this.name),
+            detail: imageMetadata?.detail || 'auto',
+          }
+        }
         if (part.source.type === 'url') {
           return {
             type: 'input_image',
@@ -2069,6 +2237,15 @@ export abstract class OpenAIBaseResponsesTextAdapter<
         }
       }
       case 'audio': {
+        if (isFileSource(part.source)) {
+          if (this.supportsFileSources !== true) {
+            throw unsupportedFileSourceError(this.name)
+          }
+          return {
+            type: 'input_file',
+            file_id: fileReferenceFor(part.source, this.name),
+          }
+        }
         if (part.source.type === 'url') {
           return {
             type: 'input_file',
@@ -2102,6 +2279,16 @@ export abstract class OpenAIBaseResponsesTextAdapter<
           documentMetadata?.detail !== undefined
             ? { detail: documentMetadata.detail as 'low' | 'high' }
             : {}
+        if (isFileSource(part.source)) {
+          if (this.supportsFileSources !== true) {
+            throw unsupportedFileSourceError(this.name)
+          }
+          return {
+            type: 'input_file',
+            file_id: fileReferenceFor(part.source, this.name),
+            ...documentDetail,
+          }
+        }
         if (part.source.type === 'url') {
           // The Responses API fetches the PDF itself; filename and MIME
           // type are inferred from the response.
@@ -2164,7 +2351,6 @@ export abstract class OpenAIBaseResponsesTextAdapter<
           ...documentDetail,
         }
       }
-
       case 'video':
       default:
         // OpenAI Responses API doesn't accept native video parts on this

package/src/adapters/responses-user-tools.ts

@@ -0,0 +1,355 @@
+import type { ResponseInputItem } from 'openai/resources/responses/responses'
+
+/**
+ * OpenAI Responses tools the app must run.
+ * A container `shell` runs on the provider. `apply_patch`, `local_shell`,
+ * and `shell` with `environment.type: "local"` (or no environment) do not.
+ */
+export type OpenAIUserToolName = 'shell' | 'apply_patch' | 'local_shell'
+
+export interface OpenAIUserExecutedCall {
+  name: OpenAIUserToolName
+  callId: string
+  itemId?: string
+  input: Record<string, unknown>
+  maxOutputLength?: number | null
+}
+
+interface ToolCallLike {
+  id: string
+  function: { name: string; arguments: string }
+  metadata?: unknown
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === 'object' && value !== null && !Array.isArray(value)
+}
+
+function stringList(value: unknown): Array<string> | null {
+  if (
+    !Array.isArray(value) ||
+    !value.every((entry) => typeof entry === 'string')
+  ) {
+    return null
+  }
+  return value
+}
+
+function stringEnv(value: unknown): Record<string, string> {
+  if (!isRecord(value)) return {}
+  const env: Record<string, string> = {}
+  for (const [key, entry] of Object.entries(value)) {
+    if (typeof entry === 'string') env[key] = entry
+  }
+  return env
+}
+
+function nullableNumber(value: unknown): number | null {
+  return typeof value === 'number' || value === null ? value : null
+}
+
+/**
+ * Hosted shell calls already include a `shell_call_output` in the same
+ * response. Those call ids must not pause the app for another run.
+ */
+export function hostedShellCallIds(
+  output: ReadonlyArray<unknown>,
+): Set<string> {
+  const ids = new Set<string>()
+  for (const item of output) {
+    if (
+      isRecord(item) &&
+      item.type === 'shell_call_output' &&
+      typeof item.call_id === 'string'
+    ) {
+      ids.add(item.call_id)
+    }
+  }
+  return ids
+}
+
+export function readUserToolName(metadata: unknown): OpenAIUserToolName | null {
+  if (!isRecord(metadata)) return null
+  const name = metadata.openaiUserTool
+  if (name === 'shell' || name === 'apply_patch' || name === 'local_shell') {
+    return name
+  }
+  return null
+}
+
+function readItemId(metadata: unknown): string | undefined {
+  if (!isRecord(metadata)) return undefined
+  return typeof metadata.itemId === 'string' && metadata.itemId.length > 0
+    ? metadata.itemId
+    : undefined
+}
+
+function readMaxOutputLength(metadata: unknown): number | null | undefined {
+  if (!isRecord(metadata)) return undefined
+  const value = metadata.maxOutputLength
+  return typeof value === 'number' || value === null ? value : undefined
+}
+
+function applyPatchOperation(
+  value: unknown,
+):
+  | { type: 'create_file'; path: string; diff: string }
+  | { type: 'update_file'; path: string; diff: string }
+  | { type: 'delete_file'; path: string }
+  | null {
+  if (!isRecord(value) || typeof value.path !== 'string') return null
+  if (value.type === 'delete_file') {
+    return { type: 'delete_file', path: value.path }
+  }
+  if (
+    (value.type === 'create_file' || value.type === 'update_file') &&
+    typeof value.diff === 'string'
+  ) {
+    return { type: value.type, path: value.path, diff: value.diff }
+  }
+  return null
+}
+
+/**
+ * Read a user-run Responses output item.
+ * `bareShell` is true only once the full response is known. A shell call
+ * with no environment waits for that pass, so a hosted call that later
+ * carries `shell_call_output` is not asked of the app.
+ */
+export function readUserExecutedCall(
+  item: unknown,
+  options: { bareShell: boolean },
+): OpenAIUserExecutedCall | null {
+  if (!isRecord(item) || typeof item.type !== 'string') return null
+  const callId = typeof item.call_id === 'string' ? item.call_id : ''
+  const itemId = typeof item.id === 'string' ? item.id : undefined
+
+  if (item.type === 'apply_patch_call') {
+    const operation = applyPatchOperation(item.operation)
+    if (!callId || !operation) return null
+    return {
+      name: 'apply_patch',
+      callId,
+      ...(itemId ? { itemId } : {}),
+      input: { operation },
+    }
+  }
+
+  if (item.type === 'local_shell_call') {
+    if (!isRecord(item.action)) return null
+    const command = stringList(item.action.command)
+    if (!callId || !command || command.length === 0) return null
+    return {
+      name: 'local_shell',
+      callId,
+      ...(itemId ? { itemId } : {}),
+      input: item.action,
+    }
+  }
+
+  if (item.type !== 'shell_call') return null
+  const environment = item.environment
+  if (isRecord(environment)) {
+    if (environment.type !== 'local') return null
+  } else if (!options.bareShell) {
+    return null
+  }
+  if (!callId || !isRecord(item.action)) return null
+  const commands = stringList(item.action.commands)
+  if (!commands || commands.length === 0) return null
+  const maxOutputLength = nullableNumber(item.action.max_output_length)
+  return {
+    name: 'shell',
+    callId,
+    ...(itemId ? { itemId } : {}),
+    input: {
+      commands,
+      max_output_length: maxOutputLength,
+      timeout_ms: nullableNumber(item.action.timeout_ms),
+    },
+    maxOutputLength,
+  }
+}
+
+function parseArguments(argumentsString: string): Record<string, unknown> {
+  try {
+    const parsed: unknown = JSON.parse(argumentsString)
+    return isRecord(parsed) ? parsed : {}
+  } catch {
+    return {}
+  }
+}
+
+function parseContent(content: unknown): unknown {
+  if (typeof content !== 'string') return content
+  try {
+    return JSON.parse(content) as unknown
+  } catch {
+    return content
+  }
+}
+
+/** Replay a user-run tool call as the Responses input item OpenAI expects. */
+export function userToolRequestItem(
+  toolCall: ToolCallLike,
+): ResponseInputItem | null {
+  const name = readUserToolName(toolCall.metadata)
+  if (!name) return null
+  const args = parseArguments(toolCall.function.arguments)
+  const itemId = readItemId(toolCall.metadata)
+
+  if (name === 'apply_patch') {
+    const operation = applyPatchOperation(args.operation)
+    if (!operation) return null
+    return {
+      type: 'apply_patch_call',
+      call_id: toolCall.id,
+      status: 'completed',
+      operation,
+      ...(itemId ? { id: itemId } : {}),
+    }
+  }
+
+  if (name === 'local_shell') {
+    const command = stringList(args.command)
+    if (!command) return null
+    return {
+      type: 'local_shell_call',
+      id: itemId ?? toolCall.id,
+      call_id: toolCall.id,
+      status: 'completed',
+      action: {
+        type: 'exec',
+        command,
+        env: stringEnv(args.env),
+        timeout_ms: nullableNumber(args.timeout_ms),
+        ...(typeof args.user === 'string' || args.user === null
+          ? { user: args.user }
+          : {}),
+        ...(typeof args.working_directory === 'string' ||
+        args.working_directory === null
+          ? { working_directory: args.working_directory }
+          : {}),
+      },
+    }
+  }
+
+  const commands = stringList(args.commands)
+  if (!commands) return null
+  return {
+    type: 'shell_call',
+    call_id: toolCall.id,
+    status: 'completed',
+    action: {
+      commands,
+      max_output_length: nullableNumber(args.max_output_length),
+      timeout_ms: nullableNumber(args.timeout_ms),
+    },
+    ...(itemId ? { id: itemId } : {}),
+  }
+}
+
+type ShellOutcome = { type: 'exit'; exit_code: number } | { type: 'timeout' }
+
+function shellOutcome(value: unknown): ShellOutcome {
+  if (isRecord(value) && value.type === 'timeout') return { type: 'timeout' }
+  const exitCode =
+    isRecord(value) && typeof value.exit_code === 'number' ? value.exit_code : 0
+  return { type: 'exit', exit_code: exitCode }
+}
+
+function shellEntry(value: unknown): {
+  stdout: string
+  stderr: string
+  outcome: ShellOutcome
+} {
+  if (typeof value === 'string') {
+    return {
+      stdout: value,
+      stderr: '',
+      outcome: { type: 'exit', exit_code: 0 },
+    }
+  }
+  const record = isRecord(value) ? value : {}
+  return {
+    stdout: typeof record.stdout === 'string' ? record.stdout : '',
+    stderr: typeof record.stderr === 'string' ? record.stderr : '',
+    outcome: shellOutcome(record.outcome),
+  }
+}
+
+function shellOutputList(content: unknown): Array<{
+  stdout: string
+  stderr: string
+  outcome: ShellOutcome
+}> {
+  const parsed = parseContent(content)
+  if (isRecord(parsed) && Array.isArray(parsed.output)) {
+    return parsed.output.map((entry) => shellEntry(entry))
+  }
+  if (isRecord(parsed) && ('stdout' in parsed || 'outcome' in parsed)) {
+    return [shellEntry(parsed)]
+  }
+  if (typeof parsed === 'string') return [shellEntry(parsed)]
+  return [shellEntry(JSON.stringify(parsed ?? ''))]
+}
+
+/** Replay the app's tool result as the matching Responses output item. */
+export function userToolResultItem(
+  toolCall: ToolCallLike,
+  content: unknown,
+): ResponseInputItem | null {
+  const name = readUserToolName(toolCall.metadata)
+  if (!name) return null
+  const parsed = parseContent(content)
+
+  if (name === 'apply_patch') {
+    const record = isRecord(parsed) ? parsed : {}
+    const failed =
+      record.status === 'failed' ||
+      (record.status !== 'completed' && typeof record.error === 'string')
+    const output =
+      typeof record.output === 'string'
+        ? record.output
+        : typeof record.error === 'string'
+          ? record.error
+          : typeof parsed === 'string'
+            ? parsed
+            : undefined
+    return {
+      type: 'apply_patch_call_output',
+      call_id: toolCall.id,
+      status: failed ? 'failed' : 'completed',
+      ...(output !== undefined ? { output } : {}),
+    }
+  }
+
+  if (name === 'local_shell') {
+    const output =
+      typeof parsed === 'string'
+        ? parsed
+        : isRecord(parsed) && typeof parsed.output === 'string'
+          ? parsed.output
+          : JSON.stringify(parsed ?? '')
+    return {
+      type: 'local_shell_call_output',
+      id: toolCall.id,
+      output,
+      status: 'completed',
+    }
+  }
+
+  const record = isRecord(parsed) ? parsed : {}
+  const fromResult = nullableNumber(record.max_output_length)
+  const fromCall = readMaxOutputLength(toolCall.metadata)
+  const maxOutputLength =
+    record.max_output_length !== undefined ? fromResult : fromCall
+  return {
+    type: 'shell_call_output',
+    call_id: toolCall.id,
+    output: shellOutputList(content),
+    ...(maxOutputLength !== undefined
+      ? { max_output_length: maxOutputLength }
+      : {}),
+  }
+}