@acentric/llm 0.0.0-stage → 0.1.0

package/src/client/responses/profile.ts

@@ -0,0 +1,55 @@
+// What one provider changes about the Responses format: where to send it, how
+// to sign it, and what is its own. The format's adapter does the rest.
+
+import type { JsonObject } from '@acentric/types';
+
+import type { LlmFailureKind } from '../../failure.js';
+import type { Credentials, ProviderOf } from '../../providers.js';
+
+/** The providers called through the Responses format. */
+export type ResponsesProvider = ProviderOf<'openai-responses'>;
+
+export type ResponsesProfile<P extends ResponsesProvider> = {
+  readonly provider: P;
+  /** Names the provider in messages, such as "OpenAI". */
+  readonly label: string;
+  /** The create-response URL. */
+  readonly url: string;
+  /** Authentication and other headers. */
+  headers(credentials: Credentials<P>): { readonly [name: string]: string };
+  /** The role mid-conversation system messages are sent with. */
+  readonly systemRole: 'system' | 'developer';
+  /** Whether file parts work. When not, they are refused, since the provider drops them without a word. */
+  readonly files: boolean;
+  /** Whether images and files may be a file ID in the provider's own storage. */
+  readonly fileIds: boolean;
+  /** Whether custom tools work. When not, they and their results are refused. */
+  readonly customTools: boolean;
+  /** Response fields that only repeat the request, left out of `native`. */
+  readonly echoFields: readonly string[];
+  /** Fields the provider writes on output items but refuses on input, left out when sending back. */
+  readonly outputOnlyFields: readonly string[];
+  /**
+   * What to multiply standard prices by for how the response was served, or
+   * undefined when that is not known, which means no cost estimate.
+   */
+  priceMultiplier(native: JsonObject, options: JsonObject): number | undefined;
+  /** The provider's own reading of an error the format does not recognize. */
+  classifyError?(error: ErrorInput): ErrorReading | undefined;
+};
+
+/** What a profile sees of an error. */
+export type ErrorInput = {
+  /** Left out for a failed response that came with HTTP 200. */
+  readonly status?: number;
+  readonly code?: string;
+  readonly type?: string;
+  readonly message?: string;
+};
+
+/** What an error means, where the provider's own code does not say. */
+export type ErrorReading = {
+  readonly kind: LlmFailureKind;
+  /** The request field at fault, when the error does not name it. */
+  readonly param?: string;
+};

package/src/client/responses/providers.ts

@@ -0,0 +1,87 @@
+// The providers on the Responses format: OpenAI and Fireworks.
+
+import type { ResponsesProfile } from './profile.js';
+
+/** OpenAI's prices for the tier that served a response. Other tiers' are not published. */
+const OPENAI_TIERS: { readonly [tier: string]: number } = { default: 1, flex: 0.5 };
+
+export const openAiProfile: ResponsesProfile<'openai'> = {
+  provider: 'openai',
+  label: 'OpenAI',
+  url: 'https://api.openai.com/v1/responses',
+  systemRole: 'developer',
+  files: true,
+  fileIds: true,
+  customTools: true,
+  echoFields: ['prompt', 'prompt_cache_retention', 'prompt_cache_options'],
+  // Written on output items and refused on input.
+  outputOnlyFields: ['created_by'],
+
+  headers({ apiKey, organization, project }) {
+    return {
+      authorization: `Bearer ${apiKey}`,
+      ...(organization === undefined ? {} : { 'openai-organization': organization }),
+      ...(project === undefined ? {} : { 'openai-project': project }),
+    };
+  },
+
+  priceMultiplier(native) {
+    const tier = native['service_tier'];
+    return OPENAI_TIERS[typeof tier === 'string' ? tier : 'default'];
+  },
+
+  classifyError({ status, message }) {
+    // Some context window errors come without the context_length_exceeded code.
+    if (
+      status === 400 &&
+      message !== undefined &&
+      /exceeds the context window|maximum context length/i.test(message)
+    ) {
+      return { kind: 'context_overflow' };
+    }
+    return undefined;
+  },
+};
+
+export const fireworksProfile: ResponsesProfile<'fireworks'> = {
+  provider: 'fireworks',
+  label: 'Fireworks',
+  url: 'https://api.fireworks.ai/inference/v1/responses',
+  systemRole: 'system',
+  // Fireworks drops file parts without an error, has no file storage, and
+  // takes custom tools its models never call.
+  files: false,
+  fileIds: false,
+  customTools: false,
+  // Fireworks repeats the reasoning settings it was sent; they say nothing new.
+  echoFields: ['reasoning'],
+  outputOnlyFields: ['created_by'],
+
+  headers({ apiKey }) {
+    return { authorization: `Bearer ${apiKey}` };
+  },
+
+  // The response does not say which tier served it. Standard prices hold
+  // unless the request asked for another tier, whose prices differ by model.
+  priceMultiplier(_native, options) {
+    const tier = options['service_tier'];
+    return tier === undefined || tier === null || tier === 'default' ? 1 : undefined;
+  },
+
+  classifyError({ status, message }) {
+    // A previous response that is unknown, expired or not stored: a bare 404 `{ detail }`.
+    if (status === 404 && message !== undefined && /^previous response not found/i.test(message)) {
+      return { kind: 'not_found', param: 'previous_response_id' };
+    }
+    if (
+      message !== undefined &&
+      /prompt is too long|exceeds maximum context length/i.test(message)
+    ) {
+      return { kind: 'context_overflow' };
+    }
+    if (status === 402) return { kind: 'quota_exceeded' };
+    // A suspended account.
+    if (status === 412) return { kind: 'auth' };
+    return undefined;
+  },
+};

package/src/client/responses/request.ts

@@ -0,0 +1,326 @@
+// The body of a Responses create-response request. Requests may come from
+// a log or, at the gateway, from anyone, so every shape is checked as it is
+// written, and anything the provider would drop without a word is refused.
+
+import type { Json, JsonObject } from '@acentric/types';
+
+import { isArray, isObject } from '../../json.js';
+import type { LlmRequest } from '../../request.js';
+import { invalidRequest, ownFields, plainText, show } from '../util.js';
+import type { ResponsesProfile, ResponsesProvider } from './profile.js';
+
+/** Request fields the request itself sets. */
+const OWNED_OPTIONS = ['model', 'input', 'instructions'];
+
+export function buildRequest<P extends ResponsesProvider>(
+  request: LlmRequest<P>,
+  profile: ResponsesProfile<P>,
+): JsonObject {
+  const { provider } = profile;
+  const options = ownFields(provider, request.options, OWNED_OPTIONS, 'request', 'options');
+  if (options['stream'] === true) {
+    throw invalidRequest(
+      provider,
+      'streaming is not supported: a call returns one finished response',
+    );
+  }
+  if (options['background'] === true) {
+    throw invalidRequest(
+      provider,
+      'background mode is not supported: a call returns one finished response',
+    );
+  }
+  const { tools: nativeTools, ...rest } = options;
+  if (nativeTools !== undefined && !isArray(nativeTools)) {
+    throw invalidRequest(provider, 'options.tools must be an array of native tool definitions');
+  }
+  const messages: Json = request.messages;
+  if (!isArray(messages)) throw invalidRequest(provider, 'messages must be an array');
+  const instructions = buildInstructions(provider, request.instructions);
+  const tools = [
+    ...(request.tools ?? []).map((tool, index) => buildTool(tool, `tools[${index}]`, profile)),
+    ...(nativeTools ?? []),
+  ];
+  return {
+    ...rest,
+    model: request.model,
+    ...(instructions === undefined ? {} : { instructions }),
+    input: messages.flatMap((message, index) => buildItems(message, `messages[${index}]`, profile)),
+    ...(tools.length > 0 ? { tools } : {}),
+  };
+}
+
+function buildInstructions(provider: string, value: Json | undefined): string | undefined {
+  if (value === undefined || typeof value === 'string') return value;
+  if (!isArray(value))
+    throw invalidRequest(provider, 'instructions must be a string or text parts');
+  const texts = value.map((part, index) => {
+    if (!isObject(part) || part['type'] !== 'text' || typeof part['text'] !== 'string') {
+      throw invalidRequest(provider, `instructions[${index}] must be a text part`);
+    }
+    if (part['providerOptions'] !== undefined) {
+      throw invalidRequest(
+        provider,
+        `instructions[${index}].providerOptions is not taken: instructions are one string here. ` +
+          'Send a system message for options such as a cache breakpoint',
+      );
+    }
+    return part['text'];
+  });
+  return texts.length > 0 ? texts.join('\n\n') : undefined;
+}
+
+function buildItems<P extends ResponsesProvider>(
+  message: Json,
+  where: string,
+  profile: ResponsesProfile<P>,
+): Json[] {
+  const { provider, label } = profile;
+  if (!isObject(message)) throw invalidRequest(provider, `${where} must be a message`);
+  switch (message['role']) {
+    case 'user':
+      return [
+        { type: 'message', role: 'user', content: buildParts(message['content'], where, profile) },
+      ];
+    case 'system':
+      return [
+        {
+          type: 'message',
+          role: profile.systemRole,
+          content: buildParts(message['content'], where, profile, true),
+        },
+      ];
+    case 'assistant':
+      if (message['provider'] !== provider) {
+        throw invalidRequest(
+          provider,
+          `${where} is from ${show(message['provider'])}: convert it before sending it to ${label}`,
+        );
+      }
+      return items(provider, message['content'], where).map((item) =>
+        outputOnly(item, profile.outputOnlyFields),
+      );
+    case 'native':
+      if (message['provider'] !== provider) {
+        throw invalidRequest(
+          provider,
+          `${where} holds ${show(message['provider'])} items, which ${label} cannot take`,
+        );
+      }
+      return items(provider, message['content'], where);
+    case 'tool_result':
+      return [buildToolResult(message, where, profile)];
+    case 'custom':
+      throw invalidRequest(
+        provider,
+        `${where} is the application's own: leave custom messages out`,
+      );
+    default:
+      throw invalidRequest(provider, `${where} has an unknown role: ${show(message['role'])}`);
+  }
+}
+
+function items(provider: string, value: Json | undefined, where: string): JsonObject[] {
+  if (!isArray(value)) throw invalidRequest(provider, `${where}.content must be an array of items`);
+  return value.map((item, index) => {
+    if (!isObject(item) || typeof item['type'] !== 'string') {
+      throw invalidRequest(provider, `${where}.content[${index}] must be an item with a type`);
+    }
+    return item;
+  });
+}
+
+/** An output item without the fields the provider writes but refuses back. */
+function outputOnly(item: JsonObject, fields: readonly string[]): JsonObject {
+  if (!fields.some((field) => field in item)) return item;
+  return Object.fromEntries(Object.entries(item).filter(([key]) => !fields.includes(key)));
+}
+
+function buildToolResult<P extends ResponsesProvider>(
+  message: JsonObject,
+  where: string,
+  profile: ResponsesProfile<P>,
+): JsonObject {
+  const { provider, label } = profile;
+  const callId = message['toolCallId'];
+  if (typeof callId !== 'string' || callId === '') {
+    throw invalidRequest(provider, `${where}.toolCallId must be a string that is not empty`);
+  }
+  const toolType = message['toolType'] ?? 'function';
+  if (toolType !== 'function' && toolType !== 'custom') {
+    throw invalidRequest(provider, `${where}.toolType must be function or custom`);
+  }
+  if (toolType === 'custom' && !profile.customTools) {
+    throw invalidRequest(provider, `${where} answers a custom tool, and ${label} has none`);
+  }
+  const fields = ownFields(
+    provider,
+    message['providerOptions'],
+    ['type', 'call_id', 'output'],
+    where,
+  );
+  return {
+    ...fields,
+    type: toolType === 'function' ? 'function_call_output' : 'custom_tool_call_output',
+    call_id: callId,
+    output: plainText(message['content']) ?? buildParts(message['content'], where, profile),
+  };
+}
+
+function buildParts<P extends ResponsesProvider>(
+  value: Json | undefined,
+  where: string,
+  profile: ResponsesProfile<P>,
+  textOnly = false,
+): JsonObject[] {
+  const { provider } = profile;
+  if (!isArray(value)) throw invalidRequest(provider, `${where}.content must be an array of parts`);
+  return value.map((part, index) => {
+    const at = `${where}.content[${index}]`;
+    if (!isObject(part)) throw invalidRequest(provider, `${at} must be a content part`);
+    if (textOnly && part['type'] !== 'text')
+      throw invalidRequest(provider, `${at} must be a text part`);
+    return buildPart(part, at, profile);
+  });
+}
+
+function buildPart<P extends ResponsesProvider>(
+  part: JsonObject,
+  at: string,
+  profile: ResponsesProfile<P>,
+): JsonObject {
+  const { provider, label } = profile;
+  const options = part['providerOptions'];
+  switch (part['type']) {
+    case 'text': {
+      const text = part['text'];
+      if (typeof text !== 'string') throw invalidRequest(provider, `${at}.text must be a string`);
+      return { ...ownFields(provider, options, ['type', 'text'], at), type: 'input_text', text };
+    }
+    case 'image': {
+      const fields = ownFields(provider, options, ['type', 'image_url', 'file_id'], at);
+      const source = checkSource(part['source'], at, profile);
+      return {
+        ...fields,
+        type: 'input_image',
+        ...(source.type === 'url'
+          ? { image_url: source.url }
+          : source.type === 'base64'
+            ? { image_url: dataUrl(source.mediaType, source.data) }
+            : { file_id: source.id }),
+      };
+    }
+    case 'file': {
+      if (!profile.files) {
+        throw invalidRequest(provider, `${at} is a file, which ${label} would drop without a word`);
+      }
+      const owned = ['type', 'file_url', 'file_data', 'file_id', 'filename'];
+      const fields = ownFields(provider, options, owned, at);
+      const source = checkSource(part['source'], at, profile);
+      const filename = part['filename'];
+      if (filename !== undefined && typeof filename !== 'string') {
+        throw invalidRequest(provider, `${at}.filename must be a string`);
+      }
+      return {
+        ...fields,
+        type: 'input_file',
+        ...(source.type === 'url'
+          ? { file_url: source.url }
+          : source.type === 'base64'
+            ? { file_data: dataUrl(source.mediaType, source.data) }
+            : { file_id: source.id }),
+        ...(filename === undefined ? {} : { filename }),
+      };
+    }
+    default:
+      throw invalidRequest(provider, `${at} has an unknown type: ${show(part['type'])}`);
+  }
+}
+
+type Source =
+  | { readonly type: 'url'; readonly url: string }
+  | { readonly type: 'base64'; readonly mediaType: string; readonly data: string }
+  | { readonly type: 'file_id'; readonly id: string };
+
+function checkSource<P extends ResponsesProvider>(
+  source: Json | undefined,
+  at: string,
+  profile: ResponsesProfile<P>,
+): Source {
+  const { provider, label } = profile;
+  if (isObject(source)) {
+    const { type, url, id, data, mediaType } = source;
+    if (type === 'url' && typeof url === 'string') return { type, url };
+    if (type === 'base64' && typeof data === 'string' && typeof mediaType === 'string') {
+      return { type, mediaType, data };
+    }
+    if (type === 'file_id' && typeof id === 'string') {
+      if (!profile.fileIds) {
+        throw invalidRequest(
+          provider,
+          `${at}.source is a file ID, which ${label} cannot take: send a URL or base64 data`,
+        );
+      }
+      return { type, id };
+    }
+  }
+  throw invalidRequest(provider, `${at}.source must be a url, base64 or file_id source`);
+}
+
+function dataUrl(mediaType: string, base64: string): string {
+  return `data:${mediaType};base64,${base64}`;
+}
+
+function buildTool<P extends ResponsesProvider>(
+  tool: Json,
+  where: string,
+  profile: ResponsesProfile<P>,
+): JsonObject {
+  const { provider, label } = profile;
+  if (
+    !isObject(tool) ||
+    typeof tool['name'] !== 'string' ||
+    typeof tool['description'] !== 'string'
+  ) {
+    throw invalidRequest(provider, `${where} needs a type, a name and a description`);
+  }
+  const { name, description, parameters, strict, format } = tool;
+  const at = `tool ${name}`;
+  if (tool['type'] === 'function') {
+    const owned = ['type', 'name', 'description', 'parameters', 'strict'];
+    const fields = ownFields(provider, tool['providerOptions'], owned, at);
+    if (!isObject(parameters))
+      throw invalidRequest(provider, `${at}: parameters must be a JSON Schema object`);
+    // A `strict` left out stays out: the provider treats it differently from false.
+    return {
+      ...fields,
+      type: 'function',
+      name,
+      description,
+      parameters,
+      ...(strict === undefined ? {} : { strict }),
+    };
+  }
+  if (tool['type'] === 'custom') {
+    if (!profile.customTools) {
+      throw invalidRequest(
+        provider,
+        `${at} is a custom tool, which ${label} cannot take: use a function tool`,
+      );
+    }
+    const fields = ownFields(
+      provider,
+      tool['providerOptions'],
+      ['type', 'name', 'description', 'format'],
+      at,
+    );
+    return {
+      ...fields,
+      type: 'custom',
+      name,
+      description,
+      ...(format === undefined ? {} : { format }),
+    };
+  }
+  throw invalidRequest(provider, `${where} has an unknown type: ${show(tool['type'])}`);
+}

package/src/client/responses/response.ts

@@ -0,0 +1,203 @@
+// A Responses answer read into an LlmResponse.
+
+import type { Json, JsonObject } from '@acentric/types';
+
+import { estimateCost } from '../../cost.js';
+import { LlmError } from '../../failure.js';
+import { isArray, isObject } from '../../json.js';
+import { findModel } from '../../models.js';
+import { pendingToolCalls } from '../../read/responses.js';
+import type { LlmRequest } from '../../request.js';
+import type { LlmResponse, StopReason } from '../../response.js';
+import type { Usage } from '../../usage.js';
+import type { JsonAnswer } from '../http.js';
+import { invalidResponse, omit, tokens, show } from '../util.js';
+import { failedResponse } from './errors.js';
+import type { ResponsesProfile, ResponsesProvider } from './profile.js';
+
+/** Response fields every Responses provider repeats from the request, or that the response already holds. */
+const ECHO_FIELDS = [
+  'id',
+  'object',
+  'model',
+  'output',
+  'output_text',
+  'usage',
+  'error',
+  'instructions',
+  'tools',
+  'tool_choice',
+  'parallel_tool_calls',
+  'max_output_tokens',
+  'max_tool_calls',
+  'temperature',
+  'top_p',
+  'top_logprobs',
+  'frequency_penalty',
+  'presence_penalty',
+  'text',
+  'truncation',
+  'store',
+  'background',
+  'previous_response_id',
+  'conversation',
+  'prompt_cache_key',
+  'safety_identifier',
+  'user',
+  'metadata',
+  'include',
+];
+
+/** Usage fields `readUsage` maps; anything else in `usage` stays in `native.usage`. */
+const TOKEN_FIELDS: { readonly [field: string]: readonly string[] | true } = {
+  input_tokens: true,
+  output_tokens: true,
+  total_tokens: true,
+  input_tokens_details: ['cached_tokens', 'cache_write_tokens'],
+  output_tokens_details: ['reasoning_tokens'],
+};
+
+export function parseResponse<P extends ResponsesProvider>(
+  request: LlmRequest<P>,
+  answer: JsonAnswer,
+  profile: ResponsesProfile<P>,
+): LlmResponse<P> {
+  const { provider, label } = profile;
+  const body = answer.body;
+  if (!isObject(body))
+    throw invalidResponse(provider, `${label} did not answer with a response`, body);
+  const id = typeof body['id'] === 'string' && body['id'] !== '' ? body['id'] : undefined;
+  const rawUsage = body['usage'];
+  const usage = isObject(rawUsage) ? readUsage(rawUsage) : undefined;
+
+  // A failure can come with HTTP 200, as a failed response or a bare error.
+  const error = body['error'];
+  if (body['status'] === 'failed' || (error !== null && error !== undefined)) {
+    throw failedResponse(profile, error ?? null, id, usage);
+  }
+  const output = body['output'];
+  const status = body['status'];
+  if (body['object'] !== 'response' || !isArray(output)) {
+    throw invalidResponse(provider, `${label} did not answer with a response`, body);
+  }
+  if (status === 'cancelled') {
+    throw new LlmError(`${label} cancelled the response`, {
+      kind: 'cancelled',
+      provider,
+      ...(id === undefined ? {} : { responseId: id }),
+    });
+  }
+  if (status !== 'completed' && status !== 'incomplete') {
+    throw invalidResponse(
+      provider,
+      `${label} answered with a ${show(status)} response, not a finished one`,
+      body,
+    );
+  }
+
+  const details = body['incomplete_details'];
+  const incomplete =
+    isObject(details) && typeof details['reason'] === 'string' ? details['reason'] : undefined;
+  const extras = isObject(rawUsage) ? usageExtras(rawUsage) : undefined;
+  const native = {
+    ...omit(body, [...ECHO_FIELDS, ...profile.echoFields]),
+    ...(extras === undefined ? {} : { usage: extras }),
+  };
+  const served = typeof body['model'] === 'string' ? body['model'] : undefined;
+  const pricing = (findModel(provider, request.model) ?? findModel(provider, served ?? ''))
+    ?.pricing;
+  const multiplier = profile.priceMultiplier(native, request.options ?? {});
+  const cost =
+    usage === undefined || multiplier === undefined
+      ? undefined
+      : estimateCost(usage, pricing, { multiplier });
+  const response = {
+    ...(id === undefined ? {} : { id }),
+    provider,
+    model: request.model,
+    ...(served === undefined || served === request.model ? {} : { responseModel: served }),
+    message: {
+      role: 'assistant',
+      provider,
+      api: 'openai-responses',
+      model: request.model,
+      ...(id === undefined ? {} : { id }),
+      timestamp: answer.timestamp,
+      content: output,
+    },
+    stopReason: stopReason(status, incomplete, output),
+    ...(incomplete === undefined ? {} : { providerStopReason: incomplete }),
+    ...(usage === undefined ? {} : { usage: cost === undefined ? usage : { ...usage, cost } }),
+    native,
+    durationMs: answer.durationMs,
+    timestamp: answer.timestamp,
+  };
+  // The output is the provider's own items, which the format's types describe.
+  // oxlint-disable-next-line typescript/no-unsafe-type-assertion
+  return response as unknown as LlmResponse<P>;
+}
+
+/** Usage as the format reports it: `input_tokens` is every prompt token, cache reads and writes among them. */
+export function readUsage(raw: JsonObject): Usage {
+  const details = isObject(raw['input_tokens_details']) ? raw['input_tokens_details'] : {};
+  const outputDetails = isObject(raw['output_tokens_details']) ? raw['output_tokens_details'] : {};
+  const totalInput = tokens(raw['input_tokens']);
+  const cacheRead = tokens(details['cached_tokens']);
+  const cacheWrite = tokens(details['cache_write_tokens']);
+  const output = tokens(raw['output_tokens']);
+  const reasoning = tokens(outputDetails['reasoning_tokens']);
+  // A model with no cache writes reports none; the uncached rest is still known.
+  const cached = (cacheRead ?? 0) + (cacheWrite ?? 0);
+  const input =
+    totalInput !== undefined && cacheRead !== undefined && cached <= totalInput
+      ? totalInput - cached
+      : undefined;
+  return {
+    ...(totalInput === undefined ? {} : { totalInput }),
+    ...(input === undefined ? {} : { input }),
+    ...(output === undefined ? {} : { output }),
+    ...(reasoning === undefined ? {} : { reasoning }),
+    ...(cacheRead === undefined ? {} : { cacheRead }),
+    ...(cacheWrite === undefined ? {} : { cacheWrite }),
+  };
+}
+
+/** What the provider reports in `usage` beyond the counts `readUsage` maps, or undefined. */
+function usageExtras(raw: JsonObject): JsonObject | undefined {
+  const extras: { [key: string]: Json } = {};
+  for (const [key, value] of Object.entries(raw)) {
+    const mapped = TOKEN_FIELDS[key];
+    if (mapped === true || value === null) continue;
+    if (mapped === undefined || !isObject(value)) {
+      extras[key] = value;
+      continue;
+    }
+    const rest = omit(value, mapped);
+    if (Object.keys(rest).length > 0) extras[key] = rest;
+  }
+  return Object.keys(extras).length > 0 ? extras : undefined;
+}
+
+/**
+ * The format has no stop reason: it reports a status, and the output shows
+ * the rest.
+ */
+function stopReason(
+  status: 'completed' | 'incomplete',
+  incomplete: string | undefined,
+  output: readonly Json[],
+): StopReason {
+  if (status === 'incomplete') return incomplete === 'content_filter' ? 'content_filter' : 'length';
+  if (pendingToolCalls(output).length > 0) return 'tool_use';
+  const messages = output.filter((item) => isObject(item) && item['type'] === 'message');
+  const refused = messages.some((message) => {
+    const content = isObject(message) ? message['content'] : undefined;
+    return isArray(content) && content.some((part) => isObject(part) && part['type'] === 'refusal');
+  });
+  if (refused) return 'refusal';
+  const answered = messages.some(
+    (message) => isObject(message) && message['phase'] !== 'commentary',
+  );
+  // Work with no final answer (hosted calls, a program, commentary) goes on when sent back.
+  return answered || output.length === 0 ? 'stop' : 'pause_turn';
+}