@core-ai/core-ai 0.18.0 → 0.20.0

package/dist/index.d.ts

@@ -9,7 +9,7 @@ type UserMessage = {
     role: 'user';
     content: string | UserContentPart[];
 };
-type UserContentPart = TextPart | ImagePart | FilePart;
+type UserContentPart = TextPart | ImagePart | FilePart | AudioPart;
 type TextPart = {
     type: 'text';
     text: string;
@@ -36,6 +36,14 @@ type FilePart = {
     mimeType: string;
     filename?: string;
 };
+type AudioPart = {
+    type: 'audio';
+    source: {
+        type: 'base64';
+        mediaType: string;
+        data: string;
+    };
+};
 type ReasoningEffort = 'minimal' | 'low' | 'medium' | 'high' | 'max';
 type ReasoningConfig = {
     effort: ReasoningEffort;
@@ -59,20 +67,35 @@ type ReasoningPart = {
     metadata?: Record<string, unknown>;
     /**
      * Provider-namespaced metadata for this reasoning block. The top-level key is
-     * the provider identifier (e.g. `'anthropic'`, `'openai'`), which also serves as
-     * the ownership discriminator: an adapter checks for the presence of its own key
-     * to detect cross-provider blocks. Cross-provider blocks are downgraded to plain
-     * text (preserving context) rather than forwarding opaque metadata that would
-     * cause an API error on the receiving provider.
+     * the provider identifier (e.g. `'anthropic'`, `'openai'`, `'azure-openai'`),
+     * which also serves as the ownership discriminator: an adapter checks for the
+     * presence of its own key to detect cross-provider blocks. Cross-provider
+     * blocks are downgraded to plain text (preserving context) rather than
+     * forwarding opaque metadata that would cause an API error on the receiving
+     * provider.
      *
-     * @example Anthropic: `{ anthropic: { signature: '...' } }`
-     * @example OpenAI:    `{ openai: { encryptedContent: '...' } }`
+     * @example Anthropic:    `{ anthropic: { signature: '...' } }`
+     * @example OpenAI:       `{ openai: { encryptedContent: '...' } }`
+     * @example Azure OpenAI: `{ 'azure-openai': { encryptedContent: '...' } }`
      */
     providerMetadata?: Record<string, Record<string, unknown>>;
 };
 type ToolCallPart = {
     type: 'tool-call';
     toolCall: ToolCall;
+    /**
+     * Provider-namespaced metadata for this tool call. The top-level key is the
+     * provider identifier and serves as the ownership discriminator, exactly as
+     * for `ReasoningPart`: an adapter only forwards metadata stored under its
+     * own key and ignores blocks produced by another provider.
+     *
+     * Some providers require this data to be replayed verbatim. Gemini 3 rejects
+     * a request when a function call from the current turn is sent back without
+     * the thought signature it was issued with.
+     *
+     * @example Google: `{ google: { thoughtSignature: '...' } }`
+     */
+    providerMetadata?: Record<string, Record<string, unknown>>;
 };
 type AssistantContentPart = AssistantTextPart | ReasoningPart | ToolCallPart;
 type AssistantMessage = {
@@ -111,6 +134,21 @@ type ToolChoice = 'auto' | 'none' | 'required' | {
     toolName: string;
 };
 type ToolChoiceMode = 'auto' | 'none' | 'required' | 'tool';
+/**
+ * Modalities a chat model can accept in user messages.
+ *
+ * `file` covers document attachments such as PDFs. `video` is reserved for a
+ * future input part.
+ */
+type ChatInputModality = 'text' | 'image' | 'file' | 'audio' | 'video';
+/**
+ * Modalities a chat model can emit as assistant content.
+ *
+ * Dedicated generators (`ImageModel`, and future audio/video models) are
+ * separate operations. Chat `output` describes native multimodal responses
+ * from `generate` / `stream`, not those dedicated APIs.
+ */
+type ChatOutputModality = 'text' | 'image' | 'audio' | 'video';
 type ModelCapabilities = {
     reasoning: {
         mode: 'unsupported' | 'optional' | 'always-on';
@@ -122,6 +160,15 @@ type ModelCapabilities = {
         restrictsSamplingParams: boolean;
         supportedToolChoices: readonly ToolChoiceMode[];
     };
+    modalities: {
+        /** Modalities accepted in user messages. Always includes `'text'`. */
+        input: readonly ChatInputModality[];
+        /**
+         * Modalities the model can emit as assistant content. Always includes
+         * `'text'`. Does not describe dedicated `ImageModel` generation.
+         */
+        output: readonly ChatOutputModality[];
+    };
 };
 type ChatModel = {
     readonly provider: string;
@@ -278,6 +325,7 @@ type StreamEvent = {
 } | {
     type: 'tool-call-end';
     toolCall: ToolCall;
+    providerMetadata?: Record<string, Record<string, unknown>>;
 } | {
     type: 'finish';
     finishReason: FinishReason;
@@ -373,6 +421,24 @@ declare class CoreAIError extends Error {
 declare class ValidationError extends CoreAIError {
     constructor(message: string, cause?: unknown, provider?: string);
 }
+type UnsupportedInputModalityErrorOptions = {
+    modelId: string;
+    providerId: string;
+    requestedModalities: readonly string[];
+    supportedModalities: readonly string[];
+    unsupportedModalities: readonly string[];
+};
+/**
+ * Thrown when user messages include content parts the model does not accept.
+ * Extends {@link ValidationError} so existing `instanceof ValidationError`
+ * checks still match.
+ */
+declare class UnsupportedInputModalityError extends ValidationError {
+    readonly requestedModalities: readonly string[];
+    readonly supportedModalities: readonly string[];
+    readonly unsupportedModalities: readonly string[];
+    constructor(options: UnsupportedInputModalityErrorOptions);
+}
 declare class AbortedError extends CoreAIError {
     constructor(cause?: unknown, provider?: string);
 }
@@ -482,6 +548,36 @@ declare function zodSchemaToJsonSchema(schema: z.ZodType): Record<string, unknow
 declare function stripModelDateSuffix(modelId: string): string;
 
 declare function clampReasoningEffort(effort: ReasoningEffort, supportedEfforts: readonly ReasoningEffort[]): ReasoningEffort;
+/** Text in, text out — the default chat modality profile. */
+declare const TEXT_ONLY_MODALITIES: {
+    readonly input: readonly ["text"];
+    readonly output: readonly ["text"];
+};
+/**
+ * Text, image, and file in; text out.
+ *
+ * Typical vision / document chat models. Audio support is advertised separately
+ * by providers whose adapters accept `AudioPart`.
+ */
+declare const MULTIMODAL_INPUT_MODALITIES: {
+    readonly input: readonly ["text", "image", "file"];
+    readonly output: readonly ["text"];
+};
+declare function supportsInputModality(capabilities: ModelCapabilities, modality: ChatInputModality): boolean;
+declare function supportsOutputModality(capabilities: ModelCapabilities, modality: ChatOutputModality): boolean;
+
+type ValidateInputModalitiesOptions = {
+    messages: Message[];
+    capabilities: ModelCapabilities;
+    modelId: string;
+    providerId: string;
+};
+/**
+ * Rejects user content parts whose modalities are not in
+ * `capabilities.modalities.input`. Part `type` values map 1:1 to input
+ * modalities (`text`, `image`, `file`, and future `audio` / `video`).
+ */
+declare function validateInputModalities({ messages, capabilities, modelId, providerId, }: ValidateInputModalitiesOptions): void;
 
 declare const UNKNOWN_MODEL: unique symbol;
 type ModelCapabilitiesRegistry<TCapabilities extends ModelCapabilities = ModelCapabilities> = Record<string, TCapabilities> & {
@@ -552,4 +648,4 @@ type GenerateImageParams = ImageGenerateOptions & {
 };
 declare function generateImage(params: GenerateImageParams): Promise<ImageGenerateResult>;
 
-export { AbortedError, type AssistantContentPart, type AssistantMessage, type AssistantTextPart, type BaseGenerateOptions, type ChatInputTokenDetails, type ChatModel, type ChatModelMiddleware, type ChatOutputTokenDetails, type ChatStream, type ChatUsage, ContextLengthExceededError, type ContextLengthExceededErrorOptions, CoreAIError, type EmbedOptions, type EmbedProviderOptions, type EmbedResult, type EmbeddingModel, type EmbeddingModelMiddleware, type EmbeddingUsage, type FilePart, type FinishReason, type GenerateObjectOptions, type GenerateObjectResult, type GenerateOptions, type GenerateProviderOptions, type GenerateResult, type GeneratedImage, type ImageGenerateOptions, type ImageGenerateResult, type ImageModel, type ImageModelMiddleware, type ImagePart, type ImageProviderOptions, type Message, type ModelCapabilities, type ModelCapabilitiesRegistry, ModelOverloadedError, type ModelOverloadedErrorOptions, type ObjectStream, type ObjectStreamEvent, ProviderError, type ProviderErrorOptions, RateLimitError, type RateLimitErrorOptions, type ReasoningConfig, type ReasoningEffort, type ReasoningPart, RetryableProviderError, ServiceUnavailableError, type ServiceUnavailableErrorOptions, StreamAbortedError, type StreamEvent, type StreamObjectOptions, StructuredOutputError, StructuredOutputNoObjectGeneratedError, StructuredOutputParseError, StructuredOutputValidationError, type SystemMessage, type TextPart, type ToolCall, type ToolCallPart, type ToolChoice, type ToolDefinition, type ToolResultMessage, type ToolSet, UNKNOWN_MODEL, type UserContentPart, type UserMessage, ValidationError, asObject, asRecord, assistantMessage, clampReasoningEffort, createChatStream, createObjectStream, defineTool, embed, generate, generateImage, generateObject, getErrorMessage, getHttpStatusCode, getProviderMetadata, getRegisteredModelCapabilities, getRetryAfterSecondsFromError, getString, isAbortErrorByName, isRateLimitStatus, isTransientUnavailableStatus, parseRetryAfterSeconds, resultToMessage, safeParseJsonObject, stream, streamObject, stripModelDateSuffix, wrapChatModel, wrapEmbeddingModel, wrapImageModel, zodSchemaToJsonSchema };
+export { AbortedError, type AssistantContentPart, type AssistantMessage, type AssistantTextPart, type AudioPart, type BaseGenerateOptions, type ChatInputModality, type ChatInputTokenDetails, type ChatModel, type ChatModelMiddleware, type ChatOutputModality, type ChatOutputTokenDetails, type ChatStream, type ChatUsage, ContextLengthExceededError, type ContextLengthExceededErrorOptions, CoreAIError, type EmbedOptions, type EmbedProviderOptions, type EmbedResult, type EmbeddingModel, type EmbeddingModelMiddleware, type EmbeddingUsage, type FilePart, type FinishReason, type GenerateObjectOptions, type GenerateObjectResult, type GenerateOptions, type GenerateProviderOptions, type GenerateResult, type GeneratedImage, type ImageGenerateOptions, type ImageGenerateResult, type ImageModel, type ImageModelMiddleware, type ImagePart, type ImageProviderOptions, MULTIMODAL_INPUT_MODALITIES, type Message, type ModelCapabilities, type ModelCapabilitiesRegistry, ModelOverloadedError, type ModelOverloadedErrorOptions, type ObjectStream, type ObjectStreamEvent, ProviderError, type ProviderErrorOptions, RateLimitError, type RateLimitErrorOptions, type ReasoningConfig, type ReasoningEffort, type ReasoningPart, RetryableProviderError, ServiceUnavailableError, type ServiceUnavailableErrorOptions, StreamAbortedError, type StreamEvent, type StreamObjectOptions, StructuredOutputError, StructuredOutputNoObjectGeneratedError, StructuredOutputParseError, StructuredOutputValidationError, type SystemMessage, TEXT_ONLY_MODALITIES, type TextPart, type ToolCall, type ToolCallPart, type ToolChoice, type ToolDefinition, type ToolResultMessage, type ToolSet, UNKNOWN_MODEL, UnsupportedInputModalityError, type UnsupportedInputModalityErrorOptions, type UserContentPart, type UserMessage, type ValidateInputModalitiesOptions, ValidationError, asObject, asRecord, assistantMessage, clampReasoningEffort, createChatStream, createObjectStream, defineTool, embed, generate, generateImage, generateObject, getErrorMessage, getHttpStatusCode, getProviderMetadata, getRegisteredModelCapabilities, getRetryAfterSecondsFromError, getString, isAbortErrorByName, isRateLimitStatus, isTransientUnavailableStatus, parseRetryAfterSeconds, resultToMessage, safeParseJsonObject, stream, streamObject, stripModelDateSuffix, supportsInputModality, supportsOutputModality, validateInputModalities, wrapChatModel, wrapEmbeddingModel, wrapImageModel, zodSchemaToJsonSchema };

package/dist/index.js

@@ -15,6 +15,25 @@ var ValidationError = class extends CoreAIError {
     this.name = "ValidationError";
   }
 };
+var UnsupportedInputModalityError = class extends ValidationError {
+  requestedModalities;
+  supportedModalities;
+  unsupportedModalities;
+  constructor(options) {
+    const unsupported = options.unsupportedModalities.join(", ");
+    const supported = options.supportedModalities.join(", ") || "(none)";
+    const modalityWord = options.unsupportedModalities.length === 1 ? "modality" : "modalities";
+    super(
+      `${options.providerId} model "${options.modelId}" does not support input ${modalityWord}: ${unsupported}. Supported: ${supported}`,
+      void 0,
+      options.providerId
+    );
+    this.name = "UnsupportedInputModalityError";
+    this.requestedModalities = options.requestedModalities;
+    this.supportedModalities = options.supportedModalities;
+    this.unsupportedModalities = options.unsupportedModalities;
+  }
+};
 var AbortedError = class extends CoreAIError {
   constructor(cause, provider) {
     super("operation aborted", cause, provider);
@@ -262,6 +281,63 @@ function clampReasoningEffort(effort, supportedEfforts) {
   }
   return best;
 }
+var TEXT_ONLY_MODALITIES = {
+  input: ["text"],
+  output: ["text"]
+};
+var MULTIMODAL_INPUT_MODALITIES = {
+  input: ["text", "image", "file"],
+  output: ["text"]
+};
+function supportsInputModality(capabilities, modality) {
+  return capabilities.modalities.input.includes(modality);
+}
+function supportsOutputModality(capabilities, modality) {
+  return capabilities.modalities.output.includes(modality);
+}
+
+// src/validate-input-modalities.ts
+function validateInputModalities({
+  messages,
+  capabilities,
+  modelId,
+  providerId
+}) {
+  const requestedModalities = collectRequestedInputModalities(messages);
+  if (requestedModalities.length === 0) {
+    return;
+  }
+  const supportedModalities = capabilities.modalities.input;
+  const unsupportedModalities = requestedModalities.filter(
+    (modality) => !supportedModalities.includes(modality)
+  );
+  if (unsupportedModalities.length === 0) {
+    return;
+  }
+  throw new UnsupportedInputModalityError({
+    modelId,
+    providerId,
+    requestedModalities,
+    supportedModalities,
+    unsupportedModalities
+  });
+}
+function collectRequestedInputModalities(messages) {
+  const requested = /* @__PURE__ */ new Set();
+  for (const message of messages) {
+    if (message.role !== "user") {
+      continue;
+    }
+    if (typeof message.content === "string") {
+      requested.add("text");
+      continue;
+    }
+    for (const part of message.content) {
+      requested.add(part.type);
+    }
+  }
+  return [...requested];
+}
 
 // src/model-capabilities-registry.ts
 var UNKNOWN_MODEL = /* @__PURE__ */ Symbol("unknown-model");
@@ -667,14 +743,15 @@ function createChatStream(source, options = {}) {
     }
     textBuffer += text;
   };
-  const appendToolCall = (toolCall) => {
+  const appendToolCall = (toolCall, providerMetadata) => {
     flushText();
     insideText = false;
     flushReasoning();
     insideReasoning = false;
     parts.push({
       type: "tool-call",
-      toolCall
+      toolCall,
+      ...providerMetadata ? { providerMetadata } : {}
     });
   };
   const setFinish = (event) => {
@@ -730,7 +807,7 @@ function createChatStream(source, options = {}) {
           endText(event.metadata);
           break;
         case "tool-call-end":
-          appendToolCall(event.toolCall);
+          appendToolCall(event.toolCall, event.providerMetadata);
           break;
         case "finish":
           setFinish(event);
@@ -889,6 +966,7 @@ export {
   AbortedError,
   ContextLengthExceededError,
   CoreAIError,
+  MULTIMODAL_INPUT_MODALITIES,
   ModelOverloadedError,
   ProviderError,
   RateLimitError,
@@ -899,7 +977,9 @@ export {
   StructuredOutputNoObjectGeneratedError,
   StructuredOutputParseError,
   StructuredOutputValidationError,
+  TEXT_ONLY_MODALITIES,
   UNKNOWN_MODEL,
+  UnsupportedInputModalityError,
   ValidationError,
   asObject,
   asRecord,
@@ -927,6 +1007,9 @@ export {
   stream,
   streamObject,
   stripModelDateSuffix,
+  supportsInputModality,
+  supportsOutputModality,
+  validateInputModalities,
   wrapChatModel,
   wrapEmbeddingModel,
   wrapImageModel,

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@core-ai/core-ai",
-    "version": "0.18.0",
+    "version": "0.20.0",
     "description": "Type-safe LLM abstraction layer over native provider SDKs",
     "license": "MIT",
     "author": "Omnifact (https://omnifact.ai)",