@gammatech/aijsx 0.2.0-beta.3 → 0.2.0-beta.5

package/dist/index.d.mts

@@ -34,6 +34,10 @@ declare function OpenAIChatCompletion(props: OpenAIChatCompletionProps, { logger
 type ValidOpenAIVisionModel = 'gpt-4-vision-preview';
 declare const ContentTypeImage: (_props: {
     url: string;
+    dimensions?: {
+        width: number;
+        height: number;
+    };
     detail?: 'auto' | 'high' | 'low';
 }) => null;
 type OpenAIVisionChatCompletionProps = {

package/dist/index.d.ts

@@ -34,6 +34,10 @@ declare function OpenAIChatCompletion(props: OpenAIChatCompletionProps, { logger
 type ValidOpenAIVisionModel = 'gpt-4-vision-preview';
 declare const ContentTypeImage: (_props: {
     url: string;
+    dimensions?: {
+        width: number;
+        height: number;
+    };
     detail?: 'auto' | 'high' | 'low';
 }) => null;
 type OpenAIVisionChatCompletionProps = {

package/dist/index.js

@@ -740,7 +740,8 @@ function tokenCountForOpenAIMessage(message) {
     case "system":
     case "user":
       return (
-        // TODO this isn't working for vision
+        // NOTE: this function should only be called for non vision requests,
+        // so message.content will be a string and not ChatCompletionContentPart[]
         TOKENS_PER_MESSAGE + tokenizer.encode(message.content).length
       );
   }
@@ -772,6 +773,24 @@ function tokenCountForOpenAIVisionMessage(message) {
   }
 }
 
+// src/lib/openai/utils.ts
+var renderChatMessageContent = (content) => {
+  if (content == null) {
+    return "";
+  }
+  if (typeof content === "string") {
+    return content;
+  }
+  return content.map((part) => {
+    if (part.type === "text") {
+      return part.text;
+    } else if (part.type === "image_url") {
+      return `<ContentTypeImage url="${part.image_url.url}" detail="${part.image_url.detail || "auto"}" />`;
+    }
+    throw new Error("Invalid ChatCompletionContentPart type");
+  }).join("\n\n");
+};
+
 // src/lib/openai/OpenAI.tsx
 var defaultClient = null;
 var OpenAIClientContext = createContext(() => {
@@ -826,8 +845,7 @@ async function* OpenAIChatCompletion(props, { logger, render, getContext }) {
   const renderedMessages = openAIMessages.map((message) => {
     return {
       role: message.role,
-      // TODO support gpt4 vision
-      content: message.content,
+      content: renderChatMessageContent(message.content),
       tokens: tokenCountForOpenAIMessage(message)
     };
   });
@@ -915,6 +933,7 @@ function buildOpenAIVisionChatMessages(childrenXml) {
   if (!topLevelValid) {
     throw new Error("Invalid top level chat message tags");
   }
+  const dimensions = /* @__PURE__ */ new WeakMap();
   for (const node of parsed.childNodes) {
     if (node.nodeName === "UserMessage") {
       const parts = node.childNodes.map((n) => {
@@ -924,13 +943,15 @@ function buildOpenAIVisionChatMessages(childrenXml) {
             text: n.value
           };
         } else if (n.nodeName === "ContentTypeImage") {
-          return {
+          const imagePart = {
             type: "image_url",
             image_url: {
               url: n.attributes.url,
               detail: n.attributes.detail || "auto"
             }
           };
+          dimensions.set(imagePart, n.attributes.dimensions);
+          return imagePart;
         }
         throw new Error(
           "Invalid ChatCompletionContentPart, expecting text or ContentTypeImage"
@@ -952,7 +973,7 @@ function buildOpenAIVisionChatMessages(childrenXml) {
       });
     }
   }
-  return messages;
+  return { messages, dimensions };
 }
 async function* OpenAIVisionChatCompletion(props, { logger, render, getContext }) {
   const startTime = performance.now();
@@ -961,38 +982,61 @@ async function* OpenAIVisionChatCompletion(props, { logger, render, getContext }
   if (!client) {
     throw new Error("[OpenAI] must supply OpenAI model via context");
   }
-  const openAIMessages = buildOpenAIVisionChatMessages(
+  const { messages: openAIMessages, dimensions } = buildOpenAIVisionChatMessages(
     await render(props.children, {
       preserveTags: true,
       renderedProps: {
         ContentTypeImage: {
           url: true,
+          dimensions: true,
           detail: true
         }
       }
     })
   );
   const renderedMessages = openAIMessages.map((message) => {
-    const renderContent = (content2) => {
-      if (content2 == null) {
-        return "";
-      }
-      if (typeof content2 === "string") {
-        return content2;
+    if (message.role === "user") {
+      if (typeof message.content === "string") {
+        return {
+          role: message.role,
+          content: message.content,
+          tokens: tokenCountForOpenAIMessage(message)
+        };
       }
-      return content2.map((part) => {
+      const BASE_COST = 85;
+      const tokens = message.content.reduce((acc, part) => {
         if (part.type === "text") {
-          return part.text;
-        } else if (part.type === "image_url") {
-          return `<ContentTypeImage url="${part.image_url.url}" detail="${part.image_url.detail || "auto"}" />`;
+          return acc + tokenCountForOpenAIMessage({
+            role: message.role,
+            content: part.text
+          });
         }
-        throw new Error("Invalid ChatCompletionContentPart type");
-      }).join(" ");
-    };
+        const detail = part.image_url.detail || "auto";
+        if (detail === "low") {
+          return acc + BASE_COST;
+        } else if (detail === "high") {
+          const dim = dimensions.get(part);
+          if (!dim) {
+            return acc + (170 * 4 + BASE_COST);
+          }
+          const area = dim.width * dim.height;
+          const num512Images = area / (512 * 512);
+          const highCost = num512Images * 170;
+          return acc + highCost + BASE_COST;
+        } else {
+          return acc + (170 * 4 + BASE_COST);
+        }
+      }, 0);
+      return {
+        role: message.role,
+        content: renderChatMessageContent(message.content),
+        tokens
+      };
+    }
     return {
       role: message.role,
-      content: renderContent(message.content),
-      tokens: tokenCountForOpenAIVisionMessage(message)
+      content: renderChatMessageContent(message.content),
+      tokens: tokenCountForOpenAIMessage(message)
     };
   });
   const chatCompletionRequest = {

package/dist/index.mjs

@@ -657,7 +657,8 @@ function tokenCountForOpenAIMessage(message) {
     case "system":
     case "user":
       return (
-        // TODO this isn't working for vision
+        // NOTE: this function should only be called for non vision requests,
+        // so message.content will be a string and not ChatCompletionContentPart[]
         TOKENS_PER_MESSAGE + tokenizer.encode(message.content).length
       );
   }
@@ -689,6 +690,24 @@ function tokenCountForOpenAIVisionMessage(message) {
   }
 }
 
+// src/lib/openai/utils.ts
+var renderChatMessageContent = (content) => {
+  if (content == null) {
+    return "";
+  }
+  if (typeof content === "string") {
+    return content;
+  }
+  return content.map((part) => {
+    if (part.type === "text") {
+      return part.text;
+    } else if (part.type === "image_url") {
+      return `<ContentTypeImage url="${part.image_url.url}" detail="${part.image_url.detail || "auto"}" />`;
+    }
+    throw new Error("Invalid ChatCompletionContentPart type");
+  }).join("\n\n");
+};
+
 // src/lib/openai/OpenAI.tsx
 var defaultClient = null;
 var OpenAIClientContext = createContext(() => {
@@ -743,8 +762,7 @@ async function* OpenAIChatCompletion(props, { logger, render, getContext }) {
   const renderedMessages = openAIMessages.map((message) => {
     return {
       role: message.role,
-      // TODO support gpt4 vision
-      content: message.content,
+      content: renderChatMessageContent(message.content),
       tokens: tokenCountForOpenAIMessage(message)
     };
   });
@@ -832,6 +850,7 @@ function buildOpenAIVisionChatMessages(childrenXml) {
   if (!topLevelValid) {
     throw new Error("Invalid top level chat message tags");
   }
+  const dimensions = /* @__PURE__ */ new WeakMap();
   for (const node of parsed.childNodes) {
     if (node.nodeName === "UserMessage") {
       const parts = node.childNodes.map((n) => {
@@ -841,13 +860,15 @@ function buildOpenAIVisionChatMessages(childrenXml) {
             text: n.value
           };
         } else if (n.nodeName === "ContentTypeImage") {
-          return {
+          const imagePart = {
             type: "image_url",
             image_url: {
               url: n.attributes.url,
               detail: n.attributes.detail || "auto"
             }
           };
+          dimensions.set(imagePart, n.attributes.dimensions);
+          return imagePart;
         }
         throw new Error(
           "Invalid ChatCompletionContentPart, expecting text or ContentTypeImage"
@@ -869,7 +890,7 @@ function buildOpenAIVisionChatMessages(childrenXml) {
       });
     }
   }
-  return messages;
+  return { messages, dimensions };
 }
 async function* OpenAIVisionChatCompletion(props, { logger, render, getContext }) {
   const startTime = performance.now();
@@ -878,38 +899,61 @@ async function* OpenAIVisionChatCompletion(props, { logger, render, getContext }
   if (!client) {
     throw new Error("[OpenAI] must supply OpenAI model via context");
   }
-  const openAIMessages = buildOpenAIVisionChatMessages(
+  const { messages: openAIMessages, dimensions } = buildOpenAIVisionChatMessages(
     await render(props.children, {
       preserveTags: true,
       renderedProps: {
         ContentTypeImage: {
           url: true,
+          dimensions: true,
           detail: true
         }
       }
     })
   );
   const renderedMessages = openAIMessages.map((message) => {
-    const renderContent = (content2) => {
-      if (content2 == null) {
-        return "";
-      }
-      if (typeof content2 === "string") {
-        return content2;
+    if (message.role === "user") {
+      if (typeof message.content === "string") {
+        return {
+          role: message.role,
+          content: message.content,
+          tokens: tokenCountForOpenAIMessage(message)
+        };
       }
-      return content2.map((part) => {
+      const BASE_COST = 85;
+      const tokens = message.content.reduce((acc, part) => {
         if (part.type === "text") {
-          return part.text;
-        } else if (part.type === "image_url") {
-          return `<ContentTypeImage url="${part.image_url.url}" detail="${part.image_url.detail || "auto"}" />`;
+          return acc + tokenCountForOpenAIMessage({
+            role: message.role,
+            content: part.text
+          });
         }
-        throw new Error("Invalid ChatCompletionContentPart type");
-      }).join(" ");
-    };
+        const detail = part.image_url.detail || "auto";
+        if (detail === "low") {
+          return acc + BASE_COST;
+        } else if (detail === "high") {
+          const dim = dimensions.get(part);
+          if (!dim) {
+            return acc + (170 * 4 + BASE_COST);
+          }
+          const area = dim.width * dim.height;
+          const num512Images = area / (512 * 512);
+          const highCost = num512Images * 170;
+          return acc + highCost + BASE_COST;
+        } else {
+          return acc + (170 * 4 + BASE_COST);
+        }
+      }, 0);
+      return {
+        role: message.role,
+        content: renderChatMessageContent(message.content),
+        tokens
+      };
+    }
     return {
       role: message.role,
-      content: renderContent(message.content),
-      tokens: tokenCountForOpenAIVisionMessage(message)
+      content: renderChatMessageContent(message.content),
+      tokens: tokenCountForOpenAIMessage(message)
     };
   });
   const chatCompletionRequest = {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@gammatech/aijsx",
-  "version": "0.2.0-beta.3",
+  "version": "0.2.0-beta.5",
   "description": "Rewrite of aijsx",
   "author": "Jordan Garcia",
   "license": "MIT",