@gammatech/aijsx 0.1.2 → 0.2.0-vision.2

package/dist/{createElement-Q_LxUYf8.d.mts → createElement-y5NHsp6D.d.mts}

@@ -112,8 +112,8 @@ declare class CombinedLogger extends LogImplementation {
     private readonly loggers;
     constructor(loggers: LogImplementation[]);
     log(...args: Parameters<LogImplementation['log']>): void;
-    chatCompletionRequest<K extends keyof ChatCompletionRequestPayloads>(...args: Parameters<LogImplementation['chatCompletionRequest']>): void;
-    chatCompletionResponse<K extends keyof ChatCompletionRequestPayloads>(...args: Parameters<LogImplementation['chatCompletionResponse']>): void;
+    chatCompletionRequest<_K extends keyof ChatCompletionRequestPayloads>(...args: Parameters<LogImplementation['chatCompletionRequest']>): void;
+    chatCompletionResponse<_K extends keyof ChatCompletionRequestPayloads>(...args: Parameters<LogImplementation['chatCompletionResponse']>): void;
 }
 
 type ChatCompletionRole = 'user' | 'system' | 'assistant';
@@ -156,4 +156,4 @@ declare function AIFragment({ children }: {
     children: AINode;
 }): Renderable;
 
-export { type AIElement as A, BoundLogger as B, type Context as C, LogImplementation as L, NoopLogImplementation as N, type PropsOfAIComponent as P, type RenderContext as R, SystemMessage as S, UserMessage as U, type RenderedConversationMessage as a, AIFragment as b, createAIElement as c, LoggerContext as d, createContext as e, AssistantMessage as f, type ConversationMessage as g, childrenToConversationMessage as h, computeUsage as i, ChatCompletionError as j, type ChatCompletionRequestPayloads as k, type LogChatCompletionRequest as l, type LogChatCompletionResponse as m, type LogLevel as n, type Logger as o, ConsoleLogger as p, CombinedLogger as q, type Literal as r, type RenderableStream as s, type RenderResult as t, type AIComponent as u, attachedContextSymbol as v, type AINode as w, type Renderable as x };
+export { type AIComponent as A, BoundLogger as B, type Context as C, LogImplementation as L, NoopLogImplementation as N, type PropsOfAIComponent as P, type RenderContext as R, SystemMessage as S, UserMessage as U, type AINode as a, type AIElement as b, createAIElement as c, AIFragment as d, LoggerContext as e, createContext as f, AssistantMessage as g, type ConversationMessage as h, type RenderedConversationMessage as i, childrenToConversationMessage as j, computeUsage as k, ChatCompletionError as l, type ChatCompletionRequestPayloads as m, type LogChatCompletionRequest as n, type LogChatCompletionResponse as o, type LogLevel as p, type Logger as q, ConsoleLogger as r, CombinedLogger as s, type Literal as t, type RenderableStream as u, type RenderResult as v, attachedContextSymbol as w, type Renderable as x };

package/dist/{createElement-Q_LxUYf8.d.ts → createElement-y5NHsp6D.d.ts}

@@ -112,8 +112,8 @@ declare class CombinedLogger extends LogImplementation {
     private readonly loggers;
     constructor(loggers: LogImplementation[]);
     log(...args: Parameters<LogImplementation['log']>): void;
-    chatCompletionRequest<K extends keyof ChatCompletionRequestPayloads>(...args: Parameters<LogImplementation['chatCompletionRequest']>): void;
-    chatCompletionResponse<K extends keyof ChatCompletionRequestPayloads>(...args: Parameters<LogImplementation['chatCompletionResponse']>): void;
+    chatCompletionRequest<_K extends keyof ChatCompletionRequestPayloads>(...args: Parameters<LogImplementation['chatCompletionRequest']>): void;
+    chatCompletionResponse<_K extends keyof ChatCompletionRequestPayloads>(...args: Parameters<LogImplementation['chatCompletionResponse']>): void;
 }
 
 type ChatCompletionRole = 'user' | 'system' | 'assistant';
@@ -156,4 +156,4 @@ declare function AIFragment({ children }: {
     children: AINode;
 }): Renderable;
 
-export { type AIElement as A, BoundLogger as B, type Context as C, LogImplementation as L, NoopLogImplementation as N, type PropsOfAIComponent as P, type RenderContext as R, SystemMessage as S, UserMessage as U, type RenderedConversationMessage as a, AIFragment as b, createAIElement as c, LoggerContext as d, createContext as e, AssistantMessage as f, type ConversationMessage as g, childrenToConversationMessage as h, computeUsage as i, ChatCompletionError as j, type ChatCompletionRequestPayloads as k, type LogChatCompletionRequest as l, type LogChatCompletionResponse as m, type LogLevel as n, type Logger as o, ConsoleLogger as p, CombinedLogger as q, type Literal as r, type RenderableStream as s, type RenderResult as t, type AIComponent as u, attachedContextSymbol as v, type AINode as w, type Renderable as x };
+export { type AIComponent as A, BoundLogger as B, type Context as C, LogImplementation as L, NoopLogImplementation as N, type PropsOfAIComponent as P, type RenderContext as R, SystemMessage as S, UserMessage as U, type AINode as a, type AIElement as b, createAIElement as c, AIFragment as d, LoggerContext as e, createContext as f, AssistantMessage as g, type ConversationMessage as h, type RenderedConversationMessage as i, childrenToConversationMessage as j, computeUsage as k, ChatCompletionError as l, type ChatCompletionRequestPayloads as m, type LogChatCompletionRequest as n, type LogChatCompletionResponse as o, type LogLevel as p, type Logger as q, ConsoleLogger as r, CombinedLogger as s, type Literal as t, type RenderableStream as u, type RenderResult as v, attachedContextSymbol as w, type Renderable as x };

package/dist/index.d.mts

@@ -1,7 +1,8 @@
-import { L as LogImplementation, R as RenderContext, C as Context, A as AIElement, a as RenderedConversationMessage } from './createElement-Q_LxUYf8.mjs';
-export { u as AIComponent, b as AIFragment, w as AINode, f as AssistantMessage, B as BoundLogger, j as ChatCompletionError, k as ChatCompletionRequestPayloads, q as CombinedLogger, p as ConsoleLogger, g as ConversationMessage, r as Literal, l as LogChatCompletionRequest, m as LogChatCompletionResponse, n as LogLevel, o as Logger, d as LoggerContext, N as NoopLogImplementation, P as PropsOfAIComponent, t as RenderResult, x as Renderable, s as RenderableStream, S as SystemMessage, U as UserMessage, v as attachedContextSymbol, h as childrenToConversationMessage, i as computeUsage, c as createAIElement, e as createContext } from './createElement-Q_LxUYf8.mjs';
+import { L as LogImplementation, R as RenderContext, A as AIComponent, C as Context, a as AINode, b as AIElement } from './createElement-y5NHsp6D.mjs';
+export { d as AIFragment, g as AssistantMessage, B as BoundLogger, l as ChatCompletionError, m as ChatCompletionRequestPayloads, s as CombinedLogger, r as ConsoleLogger, h as ConversationMessage, t as Literal, n as LogChatCompletionRequest, o as LogChatCompletionResponse, p as LogLevel, q as Logger, e as LoggerContext, N as NoopLogImplementation, P as PropsOfAIComponent, v as RenderResult, x as Renderable, u as RenderableStream, i as RenderedConversationMessage, S as SystemMessage, U as UserMessage, w as attachedContextSymbol, j as childrenToConversationMessage, k as computeUsage, c as createAIElement, f as createContext } from './createElement-y5NHsp6D.mjs';
 import { OpenAI } from 'openai';
 export { OpenAI as OpenAIClient } from 'openai';
+import { ChatCompletionSystemMessageParam, ChatCompletionUserMessageParam, ChatCompletionAssistantMessageParam } from 'openai/resources';
 import AnthropicClient from '@anthropic-ai/sdk';
 export { default as AnthropicClient } from '@anthropic-ai/sdk';
 export { countTokens as countAnthropicTokens } from '@anthropic-ai/tokenizer';
@@ -12,12 +13,21 @@ declare function createRenderContext({ logger, rootRenderId, }?: {
 }): RenderContext;
 
 type OpenAIChatCompletionRequest = OpenAI.Chat.Completions.ChatCompletionCreateParamsStreaming;
+type OpenAIChatMessage = ChatCompletionSystemMessageParam | ChatCompletionUserMessageParam | ChatCompletionAssistantMessageParam;
 declare module '@gammatech/aijsx' {
     interface ChatCompletionRequestPayloads {
         openai: OpenAIChatCompletionRequest;
     }
 }
-type ValidOpenAIChatModel = 'gpt-4' | 'gpt-4-0314' | 'gpt-4-0613' | 'gpt-4-32k' | 'gpt-4-32k-0314' | 'gpt-4-32k-0613' | 'gpt-4-1106-preview' | 'gpt-3.5-turbo' | 'gpt-3.5-turbo-0301' | 'gpt-3.5-turbo-0613' | 'gpt-3.5-turbo-16k' | 'gpt-3.5-turbo-16k-0613' | 'gpt-3.5-turbo-1106';
+type UserMessageWithImagesProps = {
+    children: AINode;
+    images: {
+        url: string;
+        detail?: 'low' | 'high' | 'auto';
+    }[];
+};
+declare const UserMessageWithImages: AIComponent<UserMessageWithImagesProps>;
+type ValidOpenAIChatModel = 'gpt-4' | 'gpt-4-0314' | 'gpt-4-0613' | 'gpt-4-32k' | 'gpt-4-32k-0314' | 'gpt-4-32k-0613' | 'gpt-4-1106-preview' | 'gpt-3.5-turbo' | 'gpt-3.5-turbo-0301' | 'gpt-3.5-turbo-0613' | 'gpt-3.5-turbo-16k' | 'gpt-3.5-turbo-16k-0613' | 'gpt-3.5-turbo-1106' | 'gpt-4-vision-preview';
 declare const OpenAIClientContext: Context<() => OpenAI>;
 type OpenAIChatCompletionProps = {
     model: ValidOpenAIChatModel;
@@ -34,7 +44,7 @@ declare const tokenizer: {
     decode: (tokens: number[]) => string;
 };
 declare function tokenLimitForChatModel(model: ValidOpenAIChatModel): number | undefined;
-declare function tokenCountForConversationMessage(message: Pick<RenderedConversationMessage, 'type' | 'content'>): number;
+declare function tokenCountForChatMessage(message: OpenAIChatMessage): number;
 
 type AnthropicChatCompletionRequest = AnthropicClient.CompletionCreateParams;
 declare module '@gammatech/aijsx' {
@@ -69,4 +79,4 @@ type AnthropicChatCompletionProps = {
  */
 declare function AnthropicChatCompletion(props: AnthropicChatCompletionProps, { render, logger, getContext }: RenderContext): AsyncGenerator<string, void, unknown>;
 
-export { AIElement, AnthropicChatCompletion, type AnthropicChatCompletionRequest, AnthropicClientContext, Context, LogImplementation, OpenAIChatCompletion, type OpenAIChatCompletionRequest, OpenAIClientContext, RenderContext, RenderedConversationMessage, type ValidAnthropicChatModel, type ValidOpenAIChatModel, createRenderContext, defaultMaxTokens, tokenCountForConversationMessage, tokenLimitForChatModel, tokenizer };
+export { AIComponent, AIElement, AINode, AnthropicChatCompletion, type AnthropicChatCompletionRequest, AnthropicClientContext, Context, LogImplementation, OpenAIChatCompletion, type OpenAIChatCompletionRequest, type OpenAIChatMessage, OpenAIClientContext, RenderContext, UserMessageWithImages, type ValidAnthropicChatModel, type ValidOpenAIChatModel, createRenderContext, defaultMaxTokens, tokenCountForChatMessage, tokenLimitForChatModel, tokenizer };

package/dist/index.d.ts

@@ -1,7 +1,8 @@
-import { L as LogImplementation, R as RenderContext, C as Context, A as AIElement, a as RenderedConversationMessage } from './createElement-Q_LxUYf8.js';
-export { u as AIComponent, b as AIFragment, w as AINode, f as AssistantMessage, B as BoundLogger, j as ChatCompletionError, k as ChatCompletionRequestPayloads, q as CombinedLogger, p as ConsoleLogger, g as ConversationMessage, r as Literal, l as LogChatCompletionRequest, m as LogChatCompletionResponse, n as LogLevel, o as Logger, d as LoggerContext, N as NoopLogImplementation, P as PropsOfAIComponent, t as RenderResult, x as Renderable, s as RenderableStream, S as SystemMessage, U as UserMessage, v as attachedContextSymbol, h as childrenToConversationMessage, i as computeUsage, c as createAIElement, e as createContext } from './createElement-Q_LxUYf8.js';
+import { L as LogImplementation, R as RenderContext, A as AIComponent, C as Context, a as AINode, b as AIElement } from './createElement-y5NHsp6D.js';
+export { d as AIFragment, g as AssistantMessage, B as BoundLogger, l as ChatCompletionError, m as ChatCompletionRequestPayloads, s as CombinedLogger, r as ConsoleLogger, h as ConversationMessage, t as Literal, n as LogChatCompletionRequest, o as LogChatCompletionResponse, p as LogLevel, q as Logger, e as LoggerContext, N as NoopLogImplementation, P as PropsOfAIComponent, v as RenderResult, x as Renderable, u as RenderableStream, i as RenderedConversationMessage, S as SystemMessage, U as UserMessage, w as attachedContextSymbol, j as childrenToConversationMessage, k as computeUsage, c as createAIElement, f as createContext } from './createElement-y5NHsp6D.js';
 import { OpenAI } from 'openai';
 export { OpenAI as OpenAIClient } from 'openai';
+import { ChatCompletionSystemMessageParam, ChatCompletionUserMessageParam, ChatCompletionAssistantMessageParam } from 'openai/resources';
 import AnthropicClient from '@anthropic-ai/sdk';
 export { default as AnthropicClient } from '@anthropic-ai/sdk';
 export { countTokens as countAnthropicTokens } from '@anthropic-ai/tokenizer';
@@ -12,12 +13,21 @@ declare function createRenderContext({ logger, rootRenderId, }?: {
 }): RenderContext;
 
 type OpenAIChatCompletionRequest = OpenAI.Chat.Completions.ChatCompletionCreateParamsStreaming;
+type OpenAIChatMessage = ChatCompletionSystemMessageParam | ChatCompletionUserMessageParam | ChatCompletionAssistantMessageParam;
 declare module '@gammatech/aijsx' {
     interface ChatCompletionRequestPayloads {
         openai: OpenAIChatCompletionRequest;
     }
 }
-type ValidOpenAIChatModel = 'gpt-4' | 'gpt-4-0314' | 'gpt-4-0613' | 'gpt-4-32k' | 'gpt-4-32k-0314' | 'gpt-4-32k-0613' | 'gpt-4-1106-preview' | 'gpt-3.5-turbo' | 'gpt-3.5-turbo-0301' | 'gpt-3.5-turbo-0613' | 'gpt-3.5-turbo-16k' | 'gpt-3.5-turbo-16k-0613' | 'gpt-3.5-turbo-1106';
+type UserMessageWithImagesProps = {
+    children: AINode;
+    images: {
+        url: string;
+        detail?: 'low' | 'high' | 'auto';
+    }[];
+};
+declare const UserMessageWithImages: AIComponent<UserMessageWithImagesProps>;
+type ValidOpenAIChatModel = 'gpt-4' | 'gpt-4-0314' | 'gpt-4-0613' | 'gpt-4-32k' | 'gpt-4-32k-0314' | 'gpt-4-32k-0613' | 'gpt-4-1106-preview' | 'gpt-3.5-turbo' | 'gpt-3.5-turbo-0301' | 'gpt-3.5-turbo-0613' | 'gpt-3.5-turbo-16k' | 'gpt-3.5-turbo-16k-0613' | 'gpt-3.5-turbo-1106' | 'gpt-4-vision-preview';
 declare const OpenAIClientContext: Context<() => OpenAI>;
 type OpenAIChatCompletionProps = {
     model: ValidOpenAIChatModel;
@@ -34,7 +44,7 @@ declare const tokenizer: {
     decode: (tokens: number[]) => string;
 };
 declare function tokenLimitForChatModel(model: ValidOpenAIChatModel): number | undefined;
-declare function tokenCountForConversationMessage(message: Pick<RenderedConversationMessage, 'type' | 'content'>): number;
+declare function tokenCountForChatMessage(message: OpenAIChatMessage): number;
 
 type AnthropicChatCompletionRequest = AnthropicClient.CompletionCreateParams;
 declare module '@gammatech/aijsx' {
@@ -69,4 +79,4 @@ type AnthropicChatCompletionProps = {
  */
 declare function AnthropicChatCompletion(props: AnthropicChatCompletionProps, { render, logger, getContext }: RenderContext): AsyncGenerator<string, void, unknown>;
 
-export { AIElement, AnthropicChatCompletion, type AnthropicChatCompletionRequest, AnthropicClientContext, Context, LogImplementation, OpenAIChatCompletion, type OpenAIChatCompletionRequest, OpenAIClientContext, RenderContext, RenderedConversationMessage, type ValidAnthropicChatModel, type ValidOpenAIChatModel, createRenderContext, defaultMaxTokens, tokenCountForConversationMessage, tokenLimitForChatModel, tokenizer };
+export { AIComponent, AIElement, AINode, AnthropicChatCompletion, type AnthropicChatCompletionRequest, AnthropicClientContext, Context, LogImplementation, OpenAIChatCompletion, type OpenAIChatCompletionRequest, type OpenAIChatMessage, OpenAIClientContext, RenderContext, UserMessageWithImages, type ValidAnthropicChatModel, type ValidOpenAIChatModel, createRenderContext, defaultMaxTokens, tokenCountForChatMessage, tokenLimitForChatModel, tokenizer };

package/dist/index.js

@@ -46,6 +46,7 @@ __export(src_exports, {
   OpenAIClientContext: () => OpenAIClientContext,
   SystemMessage: () => SystemMessage,
   UserMessage: () => UserMessage,
+  UserMessageWithImages: () => UserMessageWithImages,
   attachedContextSymbol: () => attachedContextSymbol,
   childrenToConversationMessage: () => childrenToConversationMessage,
   computeUsage: () => computeUsage,
@@ -54,7 +55,7 @@ __export(src_exports, {
   createContext: () => createContext,
   createRenderContext: () => createRenderContext,
   defaultMaxTokens: () => defaultMaxTokens,
-  tokenCountForConversationMessage: () => tokenCountForConversationMessage,
+  tokenCountForChatMessage: () => tokenCountForChatMessage,
   tokenLimitForChatModel: () => tokenLimitForChatModel,
   tokenizer: () => tokenizer
 });
@@ -73,7 +74,7 @@ var AssistantMessage = (props) => {
 var childrenToConversationMessage = (c) => {
   const children = Array.isArray(c) ? c : [c];
   return children.map((child) => {
-    if (child.tag.name === "UserMessage") {
+    if (child.tag.name.startsWith("UserMessage")) {
       return {
         type: "user",
         element: child
@@ -568,6 +569,7 @@ function tokenLimitForChatModel(model) {
     case "gpt-4-0314":
     case "gpt-4-0613":
       return 8192 - TOKENS_CONSUMED_BY_REPLY_PREFIX;
+    case "gpt-4-vision-preview":
     case "gpt-4-32k":
     case "gpt-4-32k-0314":
     case "gpt-4-32k-0613":
@@ -588,14 +590,23 @@ function tokenLimitForChatModel(model) {
     }
   }
 }
-function tokenCountForConversationMessage(message) {
+var DETAIL_COST_PLACEHOLDERS = {
+  // auto and high are based on 1024x1024
+  auto: 765,
+  high: 765,
+  low: 85
+};
+function tokenCountForChatMessage(message) {
   const TOKENS_PER_MESSAGE = 3;
-  switch (message.type) {
-    case "assistant":
-    case "system":
-    case "user":
-      return TOKENS_PER_MESSAGE + tokenizer.encode(message.content).length;
-  }
+  const contentTokens = Array.isArray(message.content) ? message.content.reduce((acc, cur) => {
+    if ("text" in cur) {
+      return acc + tokenizer.encode(cur.text).length;
+    }
+    const detail = cur.image_url.detail || "auto";
+    const detailCost = DETAIL_COST_PLACEHOLDERS[detail];
+    return acc + detailCost;
+  }, 0) : tokenizer.encode(message.content || "").length;
+  return TOKENS_PER_MESSAGE + contentTokens;
 }
 
 // src/jsx-runtime.ts
@@ -607,6 +618,10 @@ function jsx(type, config, maybeKey) {
 var jsxs = jsx;
 
 // src/lib/openai/OpenAI.tsx
+var VISION_MODELS = ["gpt-4-vision-preview"];
+var UserMessageWithImages = (props) => {
+  return /* @__PURE__ */ jsx(UserMessage, { children: props.children });
+};
 var defaultClient = null;
 var OpenAIClientContext = createContext(() => {
   if (defaultClient) {
@@ -622,24 +637,78 @@ async function* OpenAIChatCompletion(props, { logger, render, getContext }) {
   if (!client) {
     throw new Error("[OpenAI] must supply OpenAI model via context");
   }
-  const renderedMessages = await Promise.all(
-    childrenToConversationMessage(props.children).map(async (message) => {
-      const partiallyRendered = {
-        ...message,
-        content: await render(message.element)
-      };
+  const conversationMessages = childrenToConversationMessage(props.children);
+  const chatMessagesWithElement = await Promise.all(
+    conversationMessages.map(async (m) => {
+      const content2 = await render(m.element);
+      if (VISION_MODELS.includes(props.model) && m.element.tag.name === "UserMessageWithImages") {
+        const element = m.element;
+        return {
+          content: [
+            {
+              type: "text",
+              text: content2
+            },
+            ...element.props.images.map((image) => ({
+              type: "image_url",
+              image_url: {
+                url: image.url,
+                detail: image.detail || "auto"
+              }
+            }))
+          ],
+          role: "user",
+          element
+        };
+      }
       return {
-        ...partiallyRendered,
-        tokens: tokenCountForConversationMessage(partiallyRendered)
+        content: content2,
+        role: m.type,
+        element: m.element
       };
     })
   );
-  const chatMessages = renderedMessages.map((m) => {
+  const renderedMessages = chatMessagesWithElement.map((chatMessage) => {
+    const stringContent = typeof chatMessage.content === "string" ? chatMessage.content : (chatMessage.content || []).map((c) => {
+      if (c.type === "text") {
+        return c.text;
+      }
+      if (c.type === "image_url") {
+        return `<ContentTypeImage url="${c.image_url.url.slice(
+          0,
+          256
+          // in case of base64
+        )}" detail="${c.image_url.detail}" />`;
+      }
+      return "";
+    }).join("\n");
     return {
-      content: m.content,
-      role: m.type
+      type: chatMessage.role,
+      element: chatMessage.element,
+      content: stringContent,
+      tokens: tokenCountForChatMessage(chatMessage)
     };
   });
+  const chatMessages = chatMessagesWithElement.map(
+    ({ content: content2, role }) => {
+      if (role === "user") {
+        return {
+          content: content2,
+          role
+        };
+      }
+      if (role === "system") {
+        return {
+          content: content2,
+          role
+        };
+      }
+      return {
+        content: content2,
+        role
+      };
+    }
+  );
   const chatCompletionRequest = {
     model: props.model,
     max_tokens: props.maxTokens,
@@ -689,8 +758,8 @@ async function* OpenAIChatCompletion(props, { logger, render, getContext }) {
     type: "assistant",
     element: /* @__PURE__ */ jsx(AssistantMessage, { children: content }),
     content,
-    tokens: tokenCountForConversationMessage({
-      type: "assistant",
+    tokens: tokenCountForChatMessage({
+      role: "assistant",
       content
     })
   };
@@ -845,6 +914,7 @@ var import_tokenizer3 = require("@anthropic-ai/tokenizer");
   OpenAIClientContext,
   SystemMessage,
   UserMessage,
+  UserMessageWithImages,
   attachedContextSymbol,
   childrenToConversationMessage,
   computeUsage,
@@ -853,7 +923,7 @@ var import_tokenizer3 = require("@anthropic-ai/tokenizer");
   createContext,
   createRenderContext,
   defaultMaxTokens,
-  tokenCountForConversationMessage,
+  tokenCountForChatMessage,
   tokenLimitForChatModel,
   tokenizer
 });

package/dist/index.mjs

@@ -20,7 +20,7 @@ var AssistantMessage = (props) => {
 var childrenToConversationMessage = (c) => {
   const children = Array.isArray(c) ? c : [c];
   return children.map((child) => {
-    if (child.tag.name === "UserMessage") {
+    if (child.tag.name.startsWith("UserMessage")) {
       return {
         type: "user",
         element: child
@@ -489,6 +489,7 @@ function tokenLimitForChatModel(model) {
     case "gpt-4-0314":
     case "gpt-4-0613":
       return 8192 - TOKENS_CONSUMED_BY_REPLY_PREFIX;
+    case "gpt-4-vision-preview":
     case "gpt-4-32k":
     case "gpt-4-32k-0314":
     case "gpt-4-32k-0613":
@@ -509,17 +510,30 @@ function tokenLimitForChatModel(model) {
     }
   }
 }
-function tokenCountForConversationMessage(message) {
+var DETAIL_COST_PLACEHOLDERS = {
+  // auto and high are based on 1024x1024
+  auto: 765,
+  high: 765,
+  low: 85
+};
+function tokenCountForChatMessage(message) {
   const TOKENS_PER_MESSAGE = 3;
-  switch (message.type) {
-    case "assistant":
-    case "system":
-    case "user":
-      return TOKENS_PER_MESSAGE + tokenizer.encode(message.content).length;
-  }
+  const contentTokens = Array.isArray(message.content) ? message.content.reduce((acc, cur) => {
+    if ("text" in cur) {
+      return acc + tokenizer.encode(cur.text).length;
+    }
+    const detail = cur.image_url.detail || "auto";
+    const detailCost = DETAIL_COST_PLACEHOLDERS[detail];
+    return acc + detailCost;
+  }, 0) : tokenizer.encode(message.content || "").length;
+  return TOKENS_PER_MESSAGE + contentTokens;
 }
 
 // src/lib/openai/OpenAI.tsx
+var VISION_MODELS = ["gpt-4-vision-preview"];
+var UserMessageWithImages = (props) => {
+  return /* @__PURE__ */ jsx(UserMessage, { children: props.children });
+};
 var defaultClient = null;
 var OpenAIClientContext = createContext(() => {
   if (defaultClient) {
@@ -535,24 +549,78 @@ async function* OpenAIChatCompletion(props, { logger, render, getContext }) {
   if (!client) {
     throw new Error("[OpenAI] must supply OpenAI model via context");
   }
-  const renderedMessages = await Promise.all(
-    childrenToConversationMessage(props.children).map(async (message) => {
-      const partiallyRendered = {
-        ...message,
-        content: await render(message.element)
-      };
+  const conversationMessages = childrenToConversationMessage(props.children);
+  const chatMessagesWithElement = await Promise.all(
+    conversationMessages.map(async (m) => {
+      const content2 = await render(m.element);
+      if (VISION_MODELS.includes(props.model) && m.element.tag.name === "UserMessageWithImages") {
+        const element = m.element;
+        return {
+          content: [
+            {
+              type: "text",
+              text: content2
+            },
+            ...element.props.images.map((image) => ({
+              type: "image_url",
+              image_url: {
+                url: image.url,
+                detail: image.detail || "auto"
+              }
+            }))
+          ],
+          role: "user",
+          element
+        };
+      }
       return {
-        ...partiallyRendered,
-        tokens: tokenCountForConversationMessage(partiallyRendered)
+        content: content2,
+        role: m.type,
+        element: m.element
       };
     })
   );
-  const chatMessages = renderedMessages.map((m) => {
+  const renderedMessages = chatMessagesWithElement.map((chatMessage) => {
+    const stringContent = typeof chatMessage.content === "string" ? chatMessage.content : (chatMessage.content || []).map((c) => {
+      if (c.type === "text") {
+        return c.text;
+      }
+      if (c.type === "image_url") {
+        return `<ContentTypeImage url="${c.image_url.url.slice(
+          0,
+          256
+          // in case of base64
+        )}" detail="${c.image_url.detail}" />`;
+      }
+      return "";
+    }).join("\n");
     return {
-      content: m.content,
-      role: m.type
+      type: chatMessage.role,
+      element: chatMessage.element,
+      content: stringContent,
+      tokens: tokenCountForChatMessage(chatMessage)
     };
   });
+  const chatMessages = chatMessagesWithElement.map(
+    ({ content: content2, role }) => {
+      if (role === "user") {
+        return {
+          content: content2,
+          role
+        };
+      }
+      if (role === "system") {
+        return {
+          content: content2,
+          role
+        };
+      }
+      return {
+        content: content2,
+        role
+      };
+    }
+  );
   const chatCompletionRequest = {
     model: props.model,
     max_tokens: props.maxTokens,
@@ -602,8 +670,8 @@ async function* OpenAIChatCompletion(props, { logger, render, getContext }) {
     type: "assistant",
     element: /* @__PURE__ */ jsx(AssistantMessage, { children: content }),
     content,
-    tokens: tokenCountForConversationMessage({
-      type: "assistant",
+    tokens: tokenCountForChatMessage({
+      role: "assistant",
       content
     })
   };
@@ -757,6 +825,7 @@ export {
   OpenAIClientContext,
   SystemMessage,
   UserMessage,
+  UserMessageWithImages,
   attachedContextSymbol,
   childrenToConversationMessage,
   computeUsage,
@@ -765,7 +834,7 @@ export {
   createContext,
   createRenderContext,
   defaultMaxTokens,
-  tokenCountForConversationMessage,
+  tokenCountForChatMessage,
   tokenLimitForChatModel,
   tokenizer
 };

package/dist/jsx-dev-runtime.d.mts

@@ -1,2 +1,2 @@
 export { Fragment, JSX, jsx, jsxDEV, jsxs } from './jsx-runtime.mjs';
-import './createElement-Q_LxUYf8.mjs';
+import './createElement-y5NHsp6D.mjs';

package/dist/jsx-dev-runtime.d.ts

@@ -1,2 +1,2 @@
 export { Fragment, JSX, jsx, jsxDEV, jsxs } from './jsx-runtime.js';
-import './createElement-Q_LxUYf8.js';
+import './createElement-y5NHsp6D.js';

package/dist/jsx-runtime.d.mts

@@ -1,4 +1,4 @@
-import { u as AIComponent, A as AIElement, b as AIFragment } from './createElement-Q_LxUYf8.mjs';
+import { A as AIComponent, b as AIElement, d as AIFragment } from './createElement-y5NHsp6D.mjs';
 
 /**
  * The is used as an import source for ts/js files as the JSX transpile functinos

package/dist/jsx-runtime.d.ts

@@ -1,4 +1,4 @@
-import { u as AIComponent, A as AIElement, b as AIFragment } from './createElement-Q_LxUYf8.js';
+import { A as AIComponent, b as AIElement, d as AIFragment } from './createElement-y5NHsp6D.js';
 
 /**
  * The is used as an import source for ts/js files as the JSX transpile functinos

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@gammatech/aijsx",
-  "version": "0.1.2",
+  "version": "0.2.0-vision.2",
   "description": "Rewrite of aijsx",
   "author": "Jordan Garcia",
   "license": "MIT",
@@ -13,7 +13,6 @@
     "test:watch": "jest --watch --verbose",
     "build": "yarn check-types && yarn clean-symlinks && tsup",
     "clean-symlinks": "rm ./jsx-* || true",
-    "symlink": "ln -s ./dist/jsx-runtime.js . && ln -s ./dist/jsx-runtime.d.ts && ln -s ./dist/jsx-runtime.js ./jsx-dev-runtime.js && ln -s ./dist/jsx-runtime.d.ts ./jsx-dev-runtime.d.ts",
     "prepublishOnly": "yarn build",
     "lint": "eslint \"{src,test}/**/*.ts\" && yarn check-types",
     "check-types": "tsc --skipLibCheck --noEmit"