@gammatech/aijsx 0.8.1-dev.2024-05-24 → 0.9.0-dev.2024-05-28

package/dist/index.js

@@ -32,26 +32,22 @@ __export(src_exports, {
   AIFragment: () => AIFragment,
   AISpanProcessor: () => AISpanProcessor,
   AnthropicChatCompletion: () => AnthropicChatCompletion,
-  AnthropicChatCompletionInner: () => AnthropicChatCompletionInner,
   AnthropicClient: () => import_sdk2.default,
   AnthropicClientContext: () => AnthropicClientContext,
   AssistantMessage: () => AssistantMessage,
   BoundLogger: () => BoundLogger,
   ChatCompletionError: () => ChatCompletionError,
-  ClaudeImageBlock: () => ClaudeImageBlock,
   CombinedLogger: () => CombinedLogger,
   ConsoleLogger: () => ConsoleLogger,
-  ContentTypeImage: () => ContentTypeImage,
   DefaultMaxRetriesContext: () => DefaultMaxRetriesContext,
   EnrichingSpanProcessor: () => EnrichingSpanProcessor,
   Fallback: () => Fallback,
+  ImagePart: () => ImagePart,
   LogImplementation: () => LogImplementation,
   NoopLogImplementation: () => NoopLogImplementation,
   OpenAIChatCompletion: () => OpenAIChatCompletion,
-  OpenAIClient: () => import_openai4.OpenAI,
+  OpenAIClient: () => import_openai3.OpenAI,
   OpenAIClientContext: () => OpenAIClientContext,
-  OpenAIVisionChatCompletion: () => OpenAIVisionChatCompletion,
-  OpenAIVisionChatCompletionInner: () => OpenAIVisionChatCompletionInner,
   ParseVariablesError: () => ParseVariablesError,
   PromptInvalidOutputError: () => PromptInvalidOutputError,
   Retry: () => Retry,
@@ -59,9 +55,9 @@ __export(src_exports, {
   SystemMessage: () => SystemMessage,
   Trace: () => Trace,
   UserMessage: () => UserMessage,
+  anthropicTokenizer: () => anthropicTokenizer,
   attachedContextSymbol: () => attachedContextSymbol,
   computeUsage: () => computeUsage,
-  countAnthropicTokens: () => import_tokenizer4.countTokens,
   createAIElement: () => createAIElement,
   createContext: () => createContext,
   createFunctionChain: () => createFunctionChain,
@@ -69,15 +65,160 @@ __export(src_exports, {
   createRenderContext: () => createRenderContext,
   createStreamChain: () => createStreamChain,
   evaluatePrompt: () => evaluatePrompt,
-  tokenCountForOpenAIMessage: () => tokenCountForOpenAIMessage,
-  tokenCountForOpenAIVisionMessage: () => tokenCountForOpenAIVisionMessage,
-  tokenLimitForChatModel: () => tokenLimitForChatModel,
-  tokenizer: () => tokenizer,
+  openaiTokenizer: () => openaiTokenizer,
+  toDebugMessage: () => toDebugMessage,
   tracing: () => tracing
 });
 module.exports = __toCommonJS(src_exports);
 
-// src/chat.tsx
+// src/chat/errors.ts
+var ChatCompletionError = class extends Error {
+  constructor(message, chatCompletionRequest, status, shouldRetry3 = false) {
+    super(message);
+    this.chatCompletionRequest = chatCompletionRequest;
+    this.status = status;
+    this.shouldRetry = shouldRetry3;
+  }
+  name = "ChatCompletionError";
+};
+
+// src/chat/image.ts
+var IMAGE_RENDERED_PROPS = {
+  ImagePart: {
+    url: true,
+    data: true,
+    mediaType: true,
+    dimensions: true,
+    detail: true
+  }
+};
+var ImagePart = (_props) => {
+  return null;
+};
+async function fetchImageAndConvertToBase64(url) {
+  const response = await fetch(url);
+  const contentType = response.headers.get("content-type");
+  const allowedTypes = ["image/jpeg", "image/png", "image/gif", "image/webp"];
+  if (!contentType || !allowedTypes.includes(contentType)) {
+    throw new Error(`Unsupported media type: ${contentType}`);
+  }
+  const blob = await response.blob();
+  const buffer = Buffer.from(await blob.arrayBuffer());
+  const base64 = buffer.toString("base64");
+  return {
+    base64,
+    mediaType: contentType
+  };
+}
+async function processImageMessageProps(props, downloadUrl) {
+  const { dimensions, detail, ...rest } = props;
+  if ("url" in rest) {
+    if (downloadUrl) {
+      const { base64, mediaType } = await fetchImageAndConvertToBase64(rest.url);
+      return {
+        url: rest.url,
+        data: base64,
+        mediaType,
+        dimensions,
+        detail
+      };
+    }
+    return {
+      url: rest.url,
+      data: null,
+      mediaType: null,
+      dimensions,
+      detail
+    };
+  }
+  if ("data" in rest) {
+    return {
+      url: null,
+      mediaType: rest.mediaType,
+      data: rest.data,
+      dimensions,
+      detail
+    };
+  }
+  throw new Error(
+    `Invalid ImageMessageProps:  ${JSON.stringify(cleanProps(props))}`
+  );
+}
+var cleanProps = (props) => {
+  const res = {
+    ...props
+  };
+  if ("data" in res) {
+    res.data = res.data.slice(0, 64) + "...";
+  }
+  return res;
+};
+
+// src/chat/ChatMessage.ts
+var toDebugMessage = (message) => {
+  if (message.role === "assistant" || message.role === "system") {
+    return message;
+  }
+  const content = message.content.map((part) => {
+    if (typeof part === "string") {
+      return part;
+    } else if ("text" in part) {
+      return part.text;
+    } else if ("image" in part && typeof part.image === "object") {
+      const attributes = Object.entries(part.image).map(([key, value]) => {
+        const valueToUse = key === "data" && value != null ? value.slice(0, 22) + "..." : value;
+        return `${key}=${JSON.stringify(valueToUse)}`;
+      }).join(" ");
+      return `<ImagePart ${attributes} />`;
+    }
+    throw new Error("Invalid ImagePart param");
+  }).join("\n\n");
+  return {
+    role: message.role,
+    content
+  };
+};
+var UserChatMessageBuilder = class {
+  constructor(useBase64) {
+    this.useBase64 = useBase64;
+  }
+  content = [];
+  text(text) {
+    this.content.push(text);
+    return this;
+  }
+  image(props) {
+    this.content.push(props);
+    return this;
+  }
+  async build() {
+    const promises = this.content.map(
+      async (part) => {
+        if (typeof part === "string") {
+          return {
+            type: "text",
+            text: part
+          };
+        } else {
+          const image = await processImageMessageProps(part, this.useBase64);
+          return {
+            type: "image",
+            image
+          };
+        }
+      }
+    );
+    return {
+      role: "user",
+      content: await Promise.all(promises)
+    };
+  }
+};
+var userChatMessageBuilder = (opts) => {
+  return new UserChatMessageBuilder(opts.useBase64);
+};
+
+// src/chat/components.ts
 var SystemMessage = (props) => {
   return props.children;
 };
@@ -87,24 +228,22 @@ var UserMessage = (props) => {
 var AssistantMessage = (props) => {
   return props.children;
 };
-var computeUsage = (messages) => {
-  const prompt = messages.filter((m) => m.role === "user" || m.role === "system").reduce((acc, m) => acc + m.tokens, 0);
-  const completion = messages.filter((m) => m.role === "assistant").reduce((acc, m) => acc + m.tokens, 0);
+
+// src/chat/tokenizer.ts
+var computeUsage = (messages, tokenizer2) => {
+  const promptMessages = [...messages];
+  let assistantMessage;
+  if (promptMessages[promptMessages.length - 1].role === "assistant") {
+    assistantMessage = promptMessages.pop();
+  }
+  const prompt = promptMessages.reduce((acc, m) => acc + tokenizer2(m), 0);
+  const completion = assistantMessage ? tokenizer2(assistantMessage) : 0;
   return {
     prompt,
     completion,
     total: prompt + completion
   };
 };
-var ChatCompletionError = class extends Error {
-  constructor(message, chatCompletionRequest, status, shouldRetry4 = false) {
-    super(message);
-    this.chatCompletionRequest = chatCompletionRequest;
-    this.status = status;
-    this.shouldRetry = shouldRetry4;
-  }
-  name = "ChatCompletionError";
-};
 
 // src/createElement.ts
 function createAIElement(tag, props, ...children) {
@@ -1029,7 +1168,7 @@ var attachedContextSymbol = Symbol("AI.attachedContext");
 
 // src/xml.ts
 var import_fast_xml_parser = require("fast-xml-parser");
-var XmlNode = class {
+var XmlNode = class _XmlNode {
   constructor(parent, nodeName, attributes, value, childNodes) {
     this.parent = parent;
     this.nodeName = nodeName;
@@ -1041,6 +1180,15 @@ var XmlNode = class {
       childNodes.forEach((n) => n.parent = this);
     }
   }
+  clone() {
+    return new _XmlNode(
+      this.parent,
+      this.nodeName,
+      this.attributes,
+      this.value,
+      this.childNodes
+    );
+  }
   toObject() {
     if (this.value) {
       return {
@@ -1451,7 +1599,7 @@ function renderCloseTag(element) {
 // src/retry.tsx
 var RetryCountContext = createContext(0);
 var DefaultMaxRetriesContext = createContext(0);
-async function* Retry({ shouldRetry: shouldRetry4, retries = 0, maxRetries = 3, children }, ctx) {
+async function* Retry({ shouldRetry: shouldRetry3, retries = 0, maxRetries = 3, children }, ctx) {
   const { render } = ctx;
   let hasYieldedData = false;
   try {
@@ -1463,12 +1611,12 @@ async function* Retry({ shouldRetry: shouldRetry4, retries = 0, maxRetries = 3,
       yield value;
     }
   } catch (e) {
-    if (hasYieldedData || retries >= maxRetries || !shouldRetry4(e)) {
+    if (hasYieldedData || retries >= maxRetries || !shouldRetry3(e)) {
       throw e;
     }
     await backoff(retries);
     yield* Retry(
-      { shouldRetry: shouldRetry4, retries: retries + 1, maxRetries, children },
+      { shouldRetry: shouldRetry3, retries: retries + 1, maxRetries, children },
       ctx
     );
   }
@@ -1736,57 +1884,17 @@ var tokenizer = {
   encode: (text) => cl100kTokenizer.encode(text),
   decode: (tokens) => cl100kTokenizer.decode(tokens)
 };
-function tokenLimitForChatModel(model) {
-  const TOKENS_CONSUMED_BY_REPLY_PREFIX = 3;
-  switch (model) {
-    case "gpt-4":
-    case "gpt-4-0314":
-    case "gpt-4-0613":
-      return 8192 - TOKENS_CONSUMED_BY_REPLY_PREFIX;
-    case "gpt-4-32k":
-    case "gpt-4-32k-0314":
-    case "gpt-4-32k-0613":
-      return 32768 - TOKENS_CONSUMED_BY_REPLY_PREFIX;
-    case "gpt-4o":
-    case "gpt-4o-2024-05-13":
-    case "gpt-4-turbo-2024-04-09":
-    case "gpt-4-turbo":
-    case "gpt-4-1106-preview":
-    case "gpt-4-0125-preview":
-      return 128e3 - TOKENS_CONSUMED_BY_REPLY_PREFIX;
-    case "gpt-3.5-turbo":
-    case "gpt-3.5-turbo-0301":
-    case "gpt-3.5-turbo-0613":
-      return 4096 - TOKENS_CONSUMED_BY_REPLY_PREFIX;
-    case "gpt-3.5-turbo-16k":
-    case "gpt-3.5-turbo-16k-0613":
-    case "gpt-3.5-turbo-1106":
-    case "gpt-3.5-turbo-0125":
-      return 16384 - TOKENS_CONSUMED_BY_REPLY_PREFIX;
-    default: {
-      const _ = model;
-      return void 0;
-    }
-  }
-}
-function tokenCountForOpenAIMessage(message) {
-  const TOKENS_PER_MESSAGE = 3;
-  switch (message.role) {
-    case "assistant":
-    case "system":
-    case "user":
-      return (
-        // NOTE: this function should only be called for non vision requests,
-        // so message.content will be a string and not ChatCompletionContentPart[]
-        TOKENS_PER_MESSAGE + tokenizer.encode(message.content).length
-      );
-  }
-}
-function tokenCountForOpenAIVisionMessage(message) {
-  const TOKENS_PER_MESSAGE = 3;
-  const textCost = (content) => {
-    return TOKENS_PER_MESSAGE + tokenizer.encode(content).length;
-  };
+var TOKENS_PER_MESSAGE = 3;
+var textCost = (content) => {
+  return TOKENS_PER_MESSAGE + tokenizer.encode(content).length;
+};
+var COST_PER_LOW = 85;
+var COST_PER_512x512 = 170;
+var imageCost = (w, h) => {
+  const area = w * h;
+  return Math.ceil(area / (512 * 512)) * COST_PER_512x512 + COST_PER_LOW;
+};
+var openaiTokenizer = (message) => {
   switch (message.role) {
     case "assistant":
     case "system":
@@ -1799,34 +1907,86 @@ function tokenCountForOpenAIVisionMessage(message) {
         if (part.type === "text") {
           return acc + textCost(part.text);
         } else {
-          if (!part.image_url.detail || part.image_url.detail === "low") {
+          if (part.image.detail === "low") {
             return acc + 85;
-          } else {
-            return acc + (170 * 4 + 85);
           }
+          if (part.image.dimensions) {
+            return acc + imageCost(
+              part.image.dimensions.width,
+              part.image.dimensions.height
+            );
+          }
+          return acc + imageCost(1024, 1024);
         }
       }, 0);
   }
-}
+};
 
-// src/lib/openai/utils.ts
-var renderChatMessageContent = (content) => {
-  if (content == null) {
-    return "";
+// src/chat/buildMessages.ts
+async function toXml(ctx, children) {
+  const childrenXml = await ctx.render(children, {
+    preserveTags: true,
+    renderedProps: IMAGE_RENDERED_PROPS
+  });
+  const topLevelTags = ["UserMessage", "AssistantMessage", "SystemMessage"];
+  const chatMessageTags = [...topLevelTags, "ImagePart"];
+  const parsed = parseXml(childrenXml).collapse(chatMessageTags);
+  const topLevelValid = parsed.childNodes.every(
+    (node) => topLevelTags.includes(node.nodeName)
+  );
+  if (!topLevelValid) {
+    throw new Error("Invalid top level chat message tags");
   }
-  if (typeof content === "string") {
-    return content;
+  return parsed.childNodes;
+}
+async function iterateChatMessageXml(nodes, visitFn) {
+  const promises = [];
+  for (const node of nodes) {
+    if (node.nodeName === "UserMessage") {
+      const newNode = node.clone();
+      newNode.childNodes = collapseTextNodes(node.childNodes);
+      promises.push(visitFn(newNode));
+    } else {
+      promises.push(visitFn(node));
+    }
   }
-  return content.map((part) => {
-    if (part.type === "text") {
-      return part.text;
-    } else if (part.type === "image_url") {
-      let imageUrl = part.image_url.url.startsWith("data:image") ? part.image_url.url.slice(0, 22) + "..." : part.image_url.url;
-      return `<ContentTypeImage url="${imageUrl}" detail="${part.image_url.detail || "auto"}" />`;
+  return Promise.all(promises);
+}
+async function buildChatMessages(ctx, children, opts) {
+  const nodes = await toXml(ctx, children);
+  const handleUserMessage = async (node) => {
+    const childNodes = node.childNodes;
+    const builder = userChatMessageBuilder({ useBase64: opts.useBase64Images });
+    for (const n of childNodes) {
+      if (n.nodeName === "#text") {
+        builder.text(n.value);
+      } else if (n.nodeName === "ImagePart") {
+        builder.image(n.attributes);
+      } else {
+        throw new Error("Invalid User ChildNode, expecting Text or ImagePart");
+      }
     }
-    throw new Error("Invalid ChatCompletionContentPart type");
-  }).join("\n\n");
-};
+    return builder.build();
+  };
+  return iterateChatMessageXml(nodes, async (node) => {
+    switch (node.nodeName) {
+      case "UserMessage":
+        return handleUserMessage(node);
+      case "SystemMessage":
+        return {
+          role: "system",
+          content: node.textContent
+        };
+      case "AssistantMessage":
+        return {
+          role: "assistant",
+          content: node.textContent
+        };
+      default:
+        throw new Error("Invalid top level chat message tags");
+    }
+  });
+}
 
 // src/utils.ts
 function getEnvVar(name, shouldThrow = true) {
@@ -1858,311 +2018,86 @@ var OpenAIClientContext = createContext(() => {
   };
   return defaultClient;
 });
-function buildOpenAIMessages(childrenXml) {
-  const messages = [];
-  const chatMessageTags = ["UserMessage", "AssistantMessage", "SystemMessage"];
-  const parsed = parseXml(childrenXml).collapse(chatMessageTags);
-  const topLevelValid = parsed.childNodes.every(
-    (node) => chatMessageTags.includes(node.nodeName)
-  );
-  if (!topLevelValid) {
-    throw new Error("Invalid top level chat message tags");
-  }
-  for (const node of parsed.childNodes) {
-    if (node.nodeName === "UserMessage") {
-      messages.push({
-        content: node.textContent,
-        role: "user"
-      });
-    } else if (node.nodeName === "AssistantMessage") {
-      messages.push({
-        content: node.textContent,
-        role: "assistant"
-      });
-    } else if (node.nodeName === "SystemMessage") {
-      messages.push({
-        content: node.textContent,
-        role: "system"
-      });
-    }
-  }
-  return messages;
-}
-var shouldRetry = (error) => {
-  return error instanceof ChatCompletionError && error.shouldRetry;
-};
-function OpenAIChatCompletion(props, ctx) {
-  const defaultMaxRetries = ctx.getContext(DefaultMaxRetriesContext);
-  return /* @__PURE__ */ jsx(
-    Retry,
-    {
-      maxRetries: props.maxRetries || defaultMaxRetries,
-      shouldRetry,
-      children: /* @__PURE__ */ jsx(Trace, { name: "ai.chatCompletion", children: /* @__PURE__ */ jsx(OpenAIChatCompletionInner, { ...props }) })
-    }
-  );
-}
-async function* OpenAIChatCompletionInner(props, { logger, render, tracer, getContext }) {
-  const startTime = performance.now();
-  const retryCount = getContext(RetryCountContext);
-  const span = tracer.getActiveSpan();
-  const { client, provider, providerRegion, costFn } = getContext(OpenAIClientContext)();
-  if (!client) {
-    throw new Error("[OpenAI] must supply OpenAI model via context");
-  }
-  const openAIMessages = buildOpenAIMessages(
-    await render(props.children, {
-      preserveTags: true
-    })
-  );
-  const renderedMessages = openAIMessages.map((message) => {
-    return {
-      role: message.role,
-      content: renderChatMessageContent(message.content),
-      tokens: tokenCountForOpenAIMessage(message)
-    };
-  });
-  const chatCompletionRequest = {
-    model: props.model,
-    max_tokens: props.maxTokens,
-    temperature: props.temperature,
-    stop: props.stop,
-    messages: openAIMessages,
-    response_format: props.responseFormat ? {
-      type: props.responseFormat
-    } : void 0,
-    stream: true
-  };
-  const logRequestData = {
-    startTime,
-    model: props.model,
-    provider,
-    providerRegion,
-    inputMessages: renderedMessages,
-    request: chatCompletionRequest
-  };
-  logger.chatCompletionRequest("openai", logRequestData);
-  span.setAttributes({
-    model: props.model,
-    provider,
-    providerRegion,
-    requestType: "openai",
-    chatCompletionRequest,
-    inputMessages: renderedMessages,
-    retryCount
-  });
-  let chatResponse;
-  try {
-    chatResponse = await client.chat.completions.create(chatCompletionRequest);
-  } catch (ex) {
-    const retry = shouldRetryOpenAI(ex);
-    if (ex instanceof import_openai2.OpenAI.APIError) {
-      throw new ChatCompletionError(
-        `OpenAIClient.APIError: ${ex.message}`,
-        logRequestData,
-        ex.status,
-        retry
-      );
-    } else if (ex instanceof Error) {
-      throw new ChatCompletionError(
-        ex.message,
-        logRequestData,
-        void 0,
-        retry
-      );
-    }
-    throw ex;
-  }
-  let finishReason = void 0;
-  let content = "";
-  for await (const message of chatResponse) {
-    if (!message.choices || !message.choices[0]) {
-      continue;
-    }
-    const delta = message.choices[0].delta;
-    if (message.choices[0].finish_reason) {
-      finishReason = message.choices[0].finish_reason;
-    }
-    if (delta.content) {
-      content += delta.content;
-      yield delta.content;
+function buildOpenAIMessages(chatMessages) {
+  return chatMessages.map(({ role, content }) => {
+    if (role === "system" || role === "assistant") {
+      return {
+        role,
+        content
+      };
     }
-  }
-  const outputMessage = {
-    role: "assistant",
-    content,
-    tokens: tokenCountForOpenAIMessage({
-      role: "assistant",
-      content
-    })
-  };
-  const tokensUsed = computeUsage([...renderedMessages, outputMessage]);
-  const cost = costFn?.(props.model, tokensUsed) ?? void 0;
-  const responseData = {
-    ...logRequestData,
-    finishReason,
-    latency: performance.now() - startTime,
-    outputMessage,
-    tokensUsed
-  };
-  logger.chatCompletionResponse("openai", responseData);
-  span.setAttributes({
-    tokensUsed,
-    output: content,
-    finishReason,
-    cost
-  });
-}
-
-// src/lib/openai/OpenAIVision.tsx
-var import_openai3 = require("openai");
-var DEFAULT_MODEL = "gpt-4-vision-preview";
-var ContentTypeImage = (_props) => {
-  return null;
-};
-function buildOpenAIVisionChatMessages(childrenXml) {
-  const messages = [];
-  const chatMessageTags = [
-    "UserMessage",
-    "AssistantMessage",
-    "SystemMessage",
-    "ContentTypeImage"
-  ];
-  const parsed = parseXml(childrenXml).collapse(chatMessageTags);
-  const topLevelValid = parsed.childNodes.every(
-    (node) => chatMessageTags.includes(node.nodeName)
-  );
-  if (!topLevelValid) {
-    throw new Error("Invalid top level chat message tags");
-  }
-  const dimensions = /* @__PURE__ */ new WeakMap();
-  for (const node of parsed.childNodes) {
-    if (node.nodeName === "UserMessage") {
-      const parts = node.childNodes.map((n) => {
-        if (n.nodeName === "#text") {
+    if (role === "user") {
+      if (content.length === 1 && content[0].type === "text") {
+        return {
+          role,
+          content: content[0].text
+        };
+      }
+      const c = content.map((part) => {
+        if (part.type === "text") {
           return {
             type: "text",
-            text: n.value
+            text: part.text
           };
-        } else if (n.nodeName === "ContentTypeImage") {
-          const imagePart = {
+        }
+        if (part.type === "image") {
+          const url = part.image.url || part.image.data;
+          return {
             type: "image_url",
             image_url: {
-              url: n.attributes.url,
-              detail: n.attributes.detail || "auto"
+              url,
+              // default to auto
+              detail: part.image.detail || "auto"
             }
           };
-          dimensions.set(imagePart, n.attributes.dimensions);
-          return imagePart;
         }
-        throw new Error(
-          "Invalid ChatCompletionContentPart, expecting text or ContentTypeImage"
-        );
-      });
-      messages.push({
-        content: parts,
-        role: "user"
-      });
-    } else if (node.nodeName === "AssistantMessage") {
-      messages.push({
-        content: node.textContent,
-        role: "assistant"
-      });
-    } else if (node.nodeName === "SystemMessage") {
-      messages.push({
-        content: node.textContent,
-        role: "system"
+        throw new Error("Invalid part");
       });
+      return {
+        role: "user",
+        content: c
+      };
     }
-  }
-  return { messages, dimensions };
+    throw new Error("Invalid role");
+  });
 }
-var shouldRetry2 = (error) => {
+var shouldRetry = (error) => {
   return error instanceof ChatCompletionError && error.shouldRetry;
 };
-function OpenAIVisionChatCompletion(props, ctx) {
+function OpenAIChatCompletion(props, ctx) {
   const defaultMaxRetries = ctx.getContext(DefaultMaxRetriesContext);
   return /* @__PURE__ */ jsx(
     Retry,
     {
       maxRetries: props.maxRetries || defaultMaxRetries,
-      shouldRetry: shouldRetry2,
-      children: /* @__PURE__ */ jsx(Trace, { name: "ai.chatCompletion", children: /* @__PURE__ */ jsx(OpenAIVisionChatCompletionInner, { ...props }) })
+      shouldRetry,
+      children: /* @__PURE__ */ jsx(Trace, { name: "ai.chatCompletion", children: /* @__PURE__ */ jsx(OpenAIChatCompletionInner, { ...props }) })
     }
   );
 }
-async function* OpenAIVisionChatCompletionInner(props, { logger, render, tracer, getContext }) {
+async function* OpenAIChatCompletionInner(props, ctx) {
   const startTime = performance.now();
-  const model = props.model || DEFAULT_MODEL;
+  const { logger, tracer, getContext } = ctx;
   const retryCount = getContext(RetryCountContext);
   const span = tracer.getActiveSpan();
   const { client, provider, providerRegion, costFn } = getContext(OpenAIClientContext)();
   if (!client) {
     throw new Error("[OpenAI] must supply OpenAI model via context");
   }
-  const { messages: openAIMessages, dimensions } = buildOpenAIVisionChatMessages(
-    await render(props.children, {
-      preserveTags: true,
-      renderedProps: {
-        ContentTypeImage: {
-          url: true,
-          dimensions: true,
-          detail: true
-        }
-      }
-    })
-  );
-  const renderedMessages = openAIMessages.map((message) => {
-    if (message.role === "user") {
-      if (typeof message.content === "string") {
-        return {
-          role: message.role,
-          content: message.content,
-          tokens: tokenCountForOpenAIMessage(message)
-        };
-      }
-      const BASE_COST = 85;
-      const tokens = message.content.reduce((acc, part) => {
-        if (part.type === "text") {
-          return acc + tokenCountForOpenAIMessage({
-            role: message.role,
-            content: part.text
-          });
-        }
-        const detail = part.image_url.detail || "auto";
-        if (detail === "low") {
-          return acc + BASE_COST;
-        } else if (detail === "high") {
-          const dim = dimensions.get(part);
-          if (!dim) {
-            return acc + (170 * 4 + BASE_COST);
-          }
-          const area = dim.width * dim.height;
-          const num512Images = area / (512 * 512);
-          const highCost = num512Images * 170;
-          return acc + highCost + BASE_COST;
-        } else {
-          return acc + (170 * 4 + BASE_COST);
-        }
-      }, 0);
-      return {
-        role: message.role,
-        content: renderChatMessageContent(message.content),
-        tokens
-      };
-    }
-    return {
-      role: message.role,
-      content: renderChatMessageContent(message.content),
-      tokens: tokenCountForOpenAIMessage(message)
-    };
+  const chatMessages = await buildChatMessages(ctx, props.children, {
+    useBase64Images: false
   });
+  const openAIMessages = buildOpenAIMessages(chatMessages);
+  const inputMessages = chatMessages.map((m) => toDebugMessage(m));
   const chatCompletionRequest = {
-    model,
+    model: props.model,
     max_tokens: props.maxTokens,
     temperature: props.temperature,
     stop: props.stop,
     messages: openAIMessages,
+    response_format: props.responseFormat ? {
+      type: props.responseFormat
+    } : void 0,
     stream: true
   };
   const chatCompletionRequestToLog = cleanChatCompletionRequest(
@@ -2170,10 +2105,10 @@ async function* OpenAIVisionChatCompletionInner(props, { logger, render, tracer,
   );
   const logRequestData = {
     startTime,
-    model,
+    model: props.model,
     provider,
     providerRegion,
-    inputMessages: renderedMessages,
+    inputMessages,
     request: chatCompletionRequestToLog
   };
   logger.chatCompletionRequest("openai", logRequestData);
@@ -2183,7 +2118,7 @@ async function* OpenAIVisionChatCompletionInner(props, { logger, render, tracer,
     providerRegion,
     requestType: "openai",
     chatCompletionRequest: chatCompletionRequestToLog,
-    inputMessages: renderedMessages,
+    inputMessages,
     retryCount
   });
   let chatResponse;
@@ -2191,7 +2126,7 @@ async function* OpenAIVisionChatCompletionInner(props, { logger, render, tracer,
     chatResponse = await client.chat.completions.create(chatCompletionRequest);
   } catch (ex) {
     const retry = shouldRetryOpenAI(ex);
-    if (ex instanceof import_openai3.OpenAI.APIError) {
+    if (ex instanceof import_openai2.OpenAI.APIError) {
       throw new ChatCompletionError(
         `OpenAIClient.APIError: ${ex.message}`,
         logRequestData,
@@ -2225,14 +2160,13 @@ async function* OpenAIVisionChatCompletionInner(props, { logger, render, tracer,
   }
   const outputMessage = {
     role: "assistant",
-    content,
-    tokens: tokenCountForOpenAIMessage({
-      role: "assistant",
-      content
-    })
+    content
   };
-  const tokensUsed = computeUsage([...renderedMessages, outputMessage]);
-  const cost = costFn?.(model, tokensUsed) ?? void 0;
+  const tokensUsed = computeUsage(
+    [...chatMessages, outputMessage],
+    openaiTokenizer
+  );
+  const cost = costFn?.(props.model, tokensUsed) ?? void 0;
   const responseData = {
     ...logRequestData,
     finishReason,
@@ -2244,8 +2178,8 @@ async function* OpenAIVisionChatCompletionInner(props, { logger, render, tracer,
   span.setAttributes({
     tokensUsed,
     output: content,
-    cost,
-    finishReason
+    finishReason,
+    cost
   });
 }
 function cleanChatCompletionRequest(chatCompletionRequest) {
@@ -2262,25 +2196,22 @@ function cleanChatCompletionRequest(chatCompletionRequest) {
       return {
         ...message,
         content: message.content.map((part) => {
-          if (part.type === "text") {
+          if (typeof part === "string") {
             return part;
-          }
-          if (part.image_url.url.startsWith("data:image")) {
+          } else if (part.type === "text") {
+            return part;
+          } else if (part.type === "image_url") {
+            const url = part.image_url.url;
+            const cleanedUrl = url.startsWith("http") ? url : url.slice(0, 22) + "...";
             return {
-              type: "image_url",
+              ...part,
               image_url: {
-                url: part.image_url.url.slice(0, 22) + "...",
-                detail: part.image_url.detail
+                ...part.image_url,
+                url: cleanedUrl
               }
             };
           }
-          return {
-            type: "image_url",
-            image_url: {
-              url: part.image_url.url,
-              detail: part.image_url.detail
-            }
-          };
+          return part;
         })
       };
     })
@@ -2288,11 +2219,36 @@ function cleanChatCompletionRequest(chatCompletionRequest) {
 }
 
 // src/lib/openai/index.ts
-var import_openai4 = require("openai");
+var import_openai3 = require("openai");
 
 // src/lib/anthropic/Anthropic.tsx
 var import_sdk = __toESM(require("@anthropic-ai/sdk"));
-var import_tokenizer3 = require("@anthropic-ai/tokenizer");
+
+// src/lib/anthropic/tokenizer.ts
+var import_tokenizer4 = require("@anthropic-ai/tokenizer");
+var DEFAULT_IMAGE_TOKEN_COST = 1334;
+var imageTokens = (w, h) => {
+  return Math.ceil(w * h / 750);
+};
+var anthropicTokenizer = (message) => {
+  if (message.role === "system") {
+    return (0, import_tokenizer4.countTokens)(message.content);
+  }
+  if (message.role === "assistant") {
+    return (0, import_tokenizer4.countTokens)(message.content);
+  }
+  return message.content.reduce((carry, item) => {
+    let tokens = 0;
+    if (item.type === "text") {
+      tokens = (0, import_tokenizer4.countTokens)(item.text);
+    } else {
+      tokens = item.image.dimensions ? imageTokens(item.image.dimensions.width, item.image.dimensions.height) : DEFAULT_IMAGE_TOKEN_COST;
+    }
+    return carry + tokens;
+  }, 0);
+};
+
+// src/lib/anthropic/Anthropic.tsx
 var defaultClient2 = null;
 var AnthropicClientContext = createContext(() => {
   if (defaultClient2) {
@@ -2308,71 +2264,69 @@ var AnthropicClientContext = createContext(() => {
   return defaultClient2;
 });
 var defaultMaxTokens = 4096;
-function buildAnthropicMessages(childrenXml) {
+var buildAnthropicMessages = (chatMesssages) => {
   let system = "";
   const messages = [];
-  const chatMessageTags = [
-    "UserMessage",
-    "AssistantMessage",
-    "SystemMessage",
-    "ClaudeImageBlockParam"
-  ];
-  const parsed = parseXml(childrenXml).collapse(chatMessageTags);
-  const topLevelValid = parsed.childNodes.every(
-    (node) => chatMessageTags.includes(node.nodeName)
-  );
-  if (!topLevelValid) {
-    throw new Error("Invalid top level chat message tags");
-  }
-  for (const node of parsed.childNodes) {
-    if (node.nodeName === "UserMessage") {
-      const childNodes = collapseTextNodes(node.childNodes);
-      if (childNodes.length === 1 && childNodes[0].nodeName === "#text") {
+  chatMesssages.forEach(({ role, content }) => {
+    if (role === "system") {
+      system = content;
+      return;
+    }
+    if (role === "user") {
+      const userContent = content;
+      if (userContent.length === 1 && userContent[0].type === "text") {
         messages.push({
-          content: childNodes[0].value,
-          role: "user"
+          role,
+          content: userContent[0].text
         });
-        continue;
+        return;
       }
-      const parts = childNodes.map((n) => {
-        if (n.nodeName === "#text") {
-          return {
-            type: "text",
-            text: n.value
-          };
-        } else if (n.nodeName === "ClaudeImageBlockParam") {
+      const c = userContent.map((part) => {
+        if (part.type === "text") {
+          return part;
+        } else if (part.type === "image") {
           const imagePart = {
             type: "image",
             source: {
               type: "base64",
-              media_type: n.attributes.mediaType,
-              data: n.attributes.data
+              // NOTE: these `!` rely on the fact that we used `useBase64: true` in `userMessageBuilder``
+              media_type: part.image.mediaType,
+              data: part.image.data
             }
           };
           return imagePart;
         }
-        throw new Error(
-          "Invalid ChatCompletionContentPart, expecting text or ContentTypeImage"
-        );
-      }).filter((n) => n.type !== "text" || n.text.trim().length > 0);
+        throw new Error("Invalid part");
+      }).filter(
+        (a) => (
+          // allow image messages
+          a.type === "image" || a.type === "text" && a.text.trim().length > 0
+        )
+      );
       messages.push({
-        content: parts,
-        role: "user"
+        role,
+        content: c
       });
-    } else if (node.nodeName === "AssistantMessage") {
+      return;
+    }
+    if (role === "assistant") {
       messages.push({
-        content: node.textContent,
-        role: "assistant"
+        role,
+        content
       });
-    } else if (node.nodeName === "SystemMessage") {
-      system = node.textContent;
+      return;
     }
-  }
-  return { messages, system };
-}
-var shouldRetry3 = (error) => {
+    throw new Error(`Invalid role: ${role}`);
+  });
+  return {
+    system,
+    messages
+  };
+};
+var shouldRetry2 = (error) => {
   return error instanceof ChatCompletionError && error.shouldRetry;
 };
+var shouldRetryFromStatus = (status) => Boolean(status && [424, 429, 500].includes(status));
 var RE_INTERNAL_SERVER_MESSASGE = /The system encountered an unexpected error during processing/i;
 var RE_RATE_LIMIT_MESSAGE = /Too many requests, please wait before trying again/;
 var extractStatusFromError = (error) => {
@@ -2406,13 +2360,14 @@ function AnthropicChatCompletion(props, ctx) {
     Retry,
     {
       maxRetries: props.maxRetries || defaultMaxRetries,
-      shouldRetry: shouldRetry3,
+      shouldRetry: shouldRetry2,
       children: /* @__PURE__ */ jsx(Trace, { name: "ai.chatCompletion", children: /* @__PURE__ */ jsx(AnthropicChatCompletionInner, { ...props }) })
     }
   );
 }
-async function* AnthropicChatCompletionInner(props, { render, logger, tracer, getContext }) {
+async function* AnthropicChatCompletionInner(props, ctx) {
   const startTime = performance.now();
+  const { logger, tracer, getContext } = ctx;
   const retryCount = getContext(RetryCountContext);
   const span = tracer.getActiveSpan();
   const { client, provider, providerRegion, costFn } = getContext(
@@ -2423,18 +2378,11 @@ async function* AnthropicChatCompletionInner(props, { render, logger, tracer, ge
       "[AnthropicChatCompletion] must supply AnthropicClient via context"
     );
   }
-  const { system, messages } = buildAnthropicMessages(
-    await render(props.children, {
-      preserveTags: true,
-      renderedProps: {
-        ClaudeImageBlockParam: {
-          data: true,
-          mediaType: true
-        }
-      }
-    })
-  );
-  const inputMessages = getRenderedMessages(system, messages);
+  const chatMessages = await buildChatMessages(ctx, props.children, {
+    useBase64Images: true
+  });
+  const { system, messages } = buildAnthropicMessages(chatMessages);
+  const inputMessages = chatMessages.map((m) => toDebugMessage(m));
   let stopSequences;
   if (props.stop && typeof props.stop === "string") {
     stopSequences = [props.stop];
@@ -2475,25 +2423,24 @@ async function* AnthropicChatCompletionInner(props, { render, logger, tracer, ge
     response = client.messages.stream(anthropicCompletionRequest);
   } catch (err) {
     if (err instanceof import_sdk.default.APIError) {
-      const retry = "status" in err && typeof err.status === "number" && err.status !== 400;
+      const status = extractStatusFromError(err);
+      const retry = shouldRetryFromStatus(status);
       throw new ChatCompletionError(
         `AnthropicClient.APIError: ${err.message}`,
         logRequestData,
-        extractStatusFromError(err),
+        status,
         retry
       );
     } else if (err instanceof Error) {
-      throw new ChatCompletionError(
-        err.message,
-        logRequestData,
-        extractStatusFromError(err)
-      );
+      const status = extractStatusFromError(err);
+      const retry = shouldRetryFromStatus(status);
+      throw new ChatCompletionError(err.message, logRequestData, status, retry);
     }
     throw err;
   }
   let content = "";
-  let outputUsage = 0;
-  let inputUsage = 0;
+  let outputUsage;
+  let inputUsage;
   let finishReason = null;
   try {
     for await (const event of response) {
@@ -2512,25 +2459,18 @@ async function* AnthropicChatCompletionInner(props, { render, logger, tracer, ge
     }
   } catch (e) {
     const status = extractStatusFromError(e);
-    const retry = status === 429 || status === 500;
+    const retry = shouldRetryFromStatus(status);
     throw new ChatCompletionError(e.message, logRequestData, status, retry);
   }
   const outputMessage = {
     role: "assistant",
-    content,
-    tokens: outputUsage
+    content
   };
-  const userMessage = inputMessages.find(
-    (m) => m.role === "user" && m.tokens === 0
-  );
-  const restMessages = inputMessages.filter(
-    (m) => m.role !== "assistant" && m.tokens > 0
-  );
-  const restMessagesTokens = restMessages.reduce((acc, m) => acc + m.tokens, 0);
-  if (userMessage && inputUsage) {
-    userMessage.tokens = inputUsage - restMessagesTokens;
-  }
-  const tokensUsed = computeUsage([...inputMessages, outputMessage]);
+  const tokensUsed = inputUsage !== void 0 && outputUsage !== void 0 ? {
+    prompt: inputUsage,
+    completion: outputUsage,
+    total: inputUsage + outputUsage
+  } : computeUsage([...chatMessages, outputMessage], anthropicTokenizer);
   const cost = costFn?.(props.model, tokensUsed) ?? void 0;
   const responseData = {
     ...logRequestData,
@@ -2551,47 +2491,6 @@ async function* AnthropicChatCompletionInner(props, { render, logger, tracer, ge
     finishReason
   });
 }
-function getRenderedMessages(system, messages) {
-  const systemMessage = {
-    role: "system",
-    content: system,
-    tokens: (0, import_tokenizer3.countTokens)(system)
-  };
-  const renderedMessages = messages.map((message) => {
-    if (message.role === "user") {
-      return {
-        role: message.role,
-        content: renderChatMessageContent2(message.content),
-        // we keep user message tokens 0 and just let anthropic tell us
-        tokens: 0
-      };
-    }
-    return {
-      role: message.role,
-      content: renderChatMessageContent2(message.content),
-      tokens: (0, import_tokenizer3.countTokens)(message.content)
-    };
-  });
-  return [systemMessage, ...renderedMessages];
-}
-var renderChatMessageContent2 = (content) => {
-  if (content == null) {
-    return "";
-  }
-  if (typeof content === "string") {
-    return content;
-  }
-  return content.map((part) => {
-    if (typeof part === "string") {
-      return part;
-    } else if ("text" in part) {
-      return part.text;
-    } else if ("source" in part && typeof part.source === "object") {
-      return `<ImageBlockParam data="base64..." />`;
-    }
-    throw new Error("Invalid ImageBlockParam type");
-  }).join("\n\n");
-};
 function cleanChatCompletionRequest2(chatCompletionRequest) {
   const { messages, ...rest } = chatCompletionRequest;
   return {
@@ -2626,62 +2525,29 @@ function cleanChatCompletionRequest2(chatCompletionRequest) {
   };
 }
 
-// src/lib/anthropic/ClaudeImageBlock.tsx
-var ClaudeImageBlockParam = (_props) => {
-  return null;
-};
-async function fetchImageAndConvertToBase64(url) {
-  const response = await fetch(url);
-  const contentType = response.headers.get("content-type");
-  const allowedTypes = ["image/jpeg", "image/png", "image/gif", "image/webp"];
-  if (!contentType || !allowedTypes.includes(contentType)) {
-    throw new Error(`Unsupported media type: ${contentType}`);
-  }
-  const blob = await response.blob();
-  const buffer = Buffer.from(await blob.arrayBuffer());
-  const base64 = buffer.toString("base64");
-  return {
-    base64,
-    mediaType: contentType
-  };
-}
-var ClaudeImageBlock = async (props) => {
-  if ("data" in props) {
-    return /* @__PURE__ */ jsx(ClaudeImageBlockParam, { ...props });
-  }
-  const { url } = props;
-  const { base64, mediaType } = await fetchImageAndConvertToBase64(url);
-  return /* @__PURE__ */ jsx(ClaudeImageBlockParam, { mediaType, data: base64 });
-};
-
 // src/lib/anthropic/index.ts
 var import_sdk2 = __toESM(require("@anthropic-ai/sdk"));
-var import_tokenizer4 = require("@anthropic-ai/tokenizer");
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {
   AIFragment,
   AISpanProcessor,
   AnthropicChatCompletion,
-  AnthropicChatCompletionInner,
   AnthropicClient,
   AnthropicClientContext,
   AssistantMessage,
   BoundLogger,
   ChatCompletionError,
-  ClaudeImageBlock,
   CombinedLogger,
   ConsoleLogger,
-  ContentTypeImage,
   DefaultMaxRetriesContext,
   EnrichingSpanProcessor,
   Fallback,
+  ImagePart,
   LogImplementation,
   NoopLogImplementation,
   OpenAIChatCompletion,
   OpenAIClient,
   OpenAIClientContext,
-  OpenAIVisionChatCompletion,
-  OpenAIVisionChatCompletionInner,
   ParseVariablesError,
   PromptInvalidOutputError,
   Retry,
@@ -2689,9 +2555,9 @@ var import_tokenizer4 = require("@anthropic-ai/tokenizer");
   SystemMessage,
   Trace,
   UserMessage,
+  anthropicTokenizer,
   attachedContextSymbol,
   computeUsage,
-  countAnthropicTokens,
   createAIElement,
   createContext,
   createFunctionChain,
@@ -2699,9 +2565,7 @@ var import_tokenizer4 = require("@anthropic-ai/tokenizer");
   createRenderContext,
   createStreamChain,
   evaluatePrompt,
-  tokenCountForOpenAIMessage,
-  tokenCountForOpenAIVisionMessage,
-  tokenLimitForChatModel,
-  tokenizer,
+  openaiTokenizer,
+  toDebugMessage,
   tracing
 });