@gammatech/aijsx 0.2.0-beta.1 → 0.2.0-beta.3

package/dist/index.d.mts

@@ -31,6 +31,21 @@ type OpenAIChatCompletionProps = {
 };
 declare function OpenAIChatCompletion(props: OpenAIChatCompletionProps, { logger, render, getContext }: RenderContext): AsyncGenerator<string, void, unknown>;
 
+type ValidOpenAIVisionModel = 'gpt-4-vision-preview';
+declare const ContentTypeImage: (_props: {
+    url: string;
+    detail?: 'auto' | 'high' | 'low';
+}) => null;
+type OpenAIVisionChatCompletionProps = {
+    model?: ValidOpenAIVisionModel;
+    maxTokens?: number;
+    temperature?: number;
+    children: AINode;
+    provider?: string;
+    providerRegion?: string;
+};
+declare function OpenAIVisionChatCompletion(props: OpenAIVisionChatCompletionProps, { logger, render, getContext }: RenderContext): AsyncGenerator<string, void, unknown>;
+
 declare const tokenizer: {
     encode: (text: string) => number[];
     decode: (tokens: number[]) => string;
@@ -72,4 +87,4 @@ type AnthropicChatCompletionProps = {
  */
 declare function AnthropicChatCompletion(props: AnthropicChatCompletionProps, { render, logger, getContext }: RenderContext): AsyncGenerator<string, void, unknown>;
 
-export { AINode, AnthropicChatCompletion, type AnthropicChatCompletionRequest, AnthropicClientContext, Context, LogImplementation, OpenAIChatCompletion, type OpenAIChatCompletionRequest, type OpenAIChatMessage, OpenAIClientContext, RenderContext, type ValidAnthropicChatModel, type ValidOpenAIChatModel, createRenderContext, defaultMaxTokens, tokenCountForOpenAIMessage, tokenCountForOpenAIVisionMessage, tokenLimitForChatModel, tokenizer };
+export { AINode, AnthropicChatCompletion, type AnthropicChatCompletionRequest, AnthropicClientContext, ContentTypeImage, Context, LogImplementation, OpenAIChatCompletion, type OpenAIChatCompletionRequest, type OpenAIChatMessage, OpenAIClientContext, OpenAIVisionChatCompletion, RenderContext, type ValidAnthropicChatModel, type ValidOpenAIChatModel, type ValidOpenAIVisionModel, createRenderContext, defaultMaxTokens, tokenCountForOpenAIMessage, tokenCountForOpenAIVisionMessage, tokenLimitForChatModel, tokenizer };

package/dist/index.d.ts

@@ -31,6 +31,21 @@ type OpenAIChatCompletionProps = {
 };
 declare function OpenAIChatCompletion(props: OpenAIChatCompletionProps, { logger, render, getContext }: RenderContext): AsyncGenerator<string, void, unknown>;
 
+type ValidOpenAIVisionModel = 'gpt-4-vision-preview';
+declare const ContentTypeImage: (_props: {
+    url: string;
+    detail?: 'auto' | 'high' | 'low';
+}) => null;
+type OpenAIVisionChatCompletionProps = {
+    model?: ValidOpenAIVisionModel;
+    maxTokens?: number;
+    temperature?: number;
+    children: AINode;
+    provider?: string;
+    providerRegion?: string;
+};
+declare function OpenAIVisionChatCompletion(props: OpenAIVisionChatCompletionProps, { logger, render, getContext }: RenderContext): AsyncGenerator<string, void, unknown>;
+
 declare const tokenizer: {
     encode: (text: string) => number[];
     decode: (tokens: number[]) => string;
@@ -72,4 +87,4 @@ type AnthropicChatCompletionProps = {
  */
 declare function AnthropicChatCompletion(props: AnthropicChatCompletionProps, { render, logger, getContext }: RenderContext): AsyncGenerator<string, void, unknown>;
 
-export { AINode, AnthropicChatCompletion, type AnthropicChatCompletionRequest, AnthropicClientContext, Context, LogImplementation, OpenAIChatCompletion, type OpenAIChatCompletionRequest, type OpenAIChatMessage, OpenAIClientContext, RenderContext, type ValidAnthropicChatModel, type ValidOpenAIChatModel, createRenderContext, defaultMaxTokens, tokenCountForOpenAIMessage, tokenCountForOpenAIVisionMessage, tokenLimitForChatModel, tokenizer };
+export { AINode, AnthropicChatCompletion, type AnthropicChatCompletionRequest, AnthropicClientContext, ContentTypeImage, Context, LogImplementation, OpenAIChatCompletion, type OpenAIChatCompletionRequest, type OpenAIChatMessage, OpenAIClientContext, OpenAIVisionChatCompletion, RenderContext, type ValidAnthropicChatModel, type ValidOpenAIChatModel, type ValidOpenAIVisionModel, createRenderContext, defaultMaxTokens, tokenCountForOpenAIMessage, tokenCountForOpenAIVisionMessage, tokenLimitForChatModel, tokenizer };

package/dist/index.js

@@ -38,17 +38,19 @@ __export(src_exports, {
   ChatCompletionError: () => ChatCompletionError,
   CombinedLogger: () => CombinedLogger,
   ConsoleLogger: () => ConsoleLogger,
+  ContentTypeImage: () => ContentTypeImage,
   LogImplementation: () => LogImplementation,
   LoggerContext: () => LoggerContext,
   NoopLogImplementation: () => NoopLogImplementation,
   OpenAIChatCompletion: () => OpenAIChatCompletion,
-  OpenAIClient: () => import_openai2.OpenAI,
+  OpenAIClient: () => import_openai3.OpenAI,
   OpenAIClientContext: () => OpenAIClientContext,
+  OpenAIVisionChatCompletion: () => OpenAIVisionChatCompletion,
   SystemMessage: () => SystemMessage,
   UserMessage: () => UserMessage,
   attachedContextSymbol: () => attachedContextSymbol,
   computeUsage: () => computeUsage,
-  countAnthropicTokens: () => import_tokenizer3.countTokens,
+  countAnthropicTokens: () => import_tokenizer4.countTokens,
   createAIElement: () => createAIElement,
   createContext: () => createContext,
   createRenderContext: () => createRenderContext,
@@ -390,7 +392,7 @@ function parseXml(input) {
   return constructNode(null, parsed[0]);
 }
 function escape(html) {
-  return html.replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;").replace(/"/g, "&quot;").replace(/'/g, "&#039;");
+  return html.replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;").replace(/"/g, "&quot;");
 }
 
 // src/context.ts
@@ -892,12 +894,176 @@ async function* OpenAIChatCompletion(props, { logger, render, getContext }) {
   logger.chatCompletionResponse("openai", responseData);
 }
 
-// src/lib/openai/index.ts
+// src/lib/openai/OpenAIVision.tsx
 var import_openai2 = require("openai");
+var DEFAULT_MODEL = "gpt-4-vision-preview";
+var ContentTypeImage = (_props) => {
+  return null;
+};
+function buildOpenAIVisionChatMessages(childrenXml) {
+  const messages = [];
+  const chatMessageTags2 = [
+    "UserMessage",
+    "AssistantMessage",
+    "SystemMessage",
+    "ContentTypeImage"
+  ];
+  const parsed = parseXml(childrenXml).collapse(chatMessageTags2);
+  const topLevelValid = parsed.childNodes.every(
+    (node) => chatMessageTags2.includes(node.nodeName)
+  );
+  if (!topLevelValid) {
+    throw new Error("Invalid top level chat message tags");
+  }
+  for (const node of parsed.childNodes) {
+    if (node.nodeName === "UserMessage") {
+      const parts = node.childNodes.map((n) => {
+        if (n.nodeName === "#text") {
+          return {
+            type: "text",
+            text: n.value
+          };
+        } else if (n.nodeName === "ContentTypeImage") {
+          return {
+            type: "image_url",
+            image_url: {
+              url: n.attributes.url,
+              detail: n.attributes.detail || "auto"
+            }
+          };
+        }
+        throw new Error(
+          "Invalid ChatCompletionContentPart, expecting text or ContentTypeImage"
+        );
+      });
+      messages.push({
+        content: parts,
+        role: "user"
+      });
+    } else if (node.nodeName === "AssistantMessage") {
+      messages.push({
+        content: node.textContent,
+        role: "assistant"
+      });
+    } else if (node.nodeName === "SystemMessage") {
+      messages.push({
+        content: node.textContent,
+        role: "system"
+      });
+    }
+  }
+  return messages;
+}
+async function* OpenAIVisionChatCompletion(props, { logger, render, getContext }) {
+  const startTime = performance.now();
+  const model = props.model || DEFAULT_MODEL;
+  const client = getContext(OpenAIClientContext)();
+  if (!client) {
+    throw new Error("[OpenAI] must supply OpenAI model via context");
+  }
+  const openAIMessages = buildOpenAIVisionChatMessages(
+    await render(props.children, {
+      preserveTags: true,
+      renderedProps: {
+        ContentTypeImage: {
+          url: true,
+          detail: true
+        }
+      }
+    })
+  );
+  const renderedMessages = openAIMessages.map((message) => {
+    const renderContent = (content2) => {
+      if (content2 == null) {
+        return "";
+      }
+      if (typeof content2 === "string") {
+        return content2;
+      }
+      return content2.map((part) => {
+        if (part.type === "text") {
+          return part.text;
+        } else if (part.type === "image_url") {
+          return `<ContentTypeImage url="${part.image_url.url}" detail="${part.image_url.detail || "auto"}" />`;
+        }
+        throw new Error("Invalid ChatCompletionContentPart type");
+      }).join(" ");
+    };
+    return {
+      role: message.role,
+      content: renderContent(message.content),
+      tokens: tokenCountForOpenAIVisionMessage(message)
+    };
+  });
+  const chatCompletionRequest = {
+    model,
+    max_tokens: props.maxTokens,
+    temperature: props.temperature,
+    messages: openAIMessages,
+    stream: true
+  };
+  const logRequestData = {
+    startTime,
+    model,
+    provider: props.provider,
+    providerRegion: props.providerRegion,
+    inputMessages: renderedMessages,
+    request: chatCompletionRequest
+  };
+  logger.chatCompletionRequest("openai", logRequestData);
+  let chatResponse;
+  try {
+    chatResponse = await client.chat.completions.create(chatCompletionRequest);
+  } catch (ex) {
+    if (ex instanceof import_openai2.OpenAI.APIError) {
+      throw new ChatCompletionError(
+        `OpenAIClient.APIError: ${ex.message}`,
+        logRequestData
+      );
+    } else if (ex instanceof Error) {
+      throw new ChatCompletionError(ex.message, logRequestData);
+    }
+    throw ex;
+  }
+  let finishReason = void 0;
+  let content = "";
+  for await (const message of chatResponse) {
+    if (!message.choices || !message.choices[0]) {
+      continue;
+    }
+    const delta = message.choices[0].delta;
+    if (message.choices[0].finish_reason) {
+      finishReason = message.choices[0].finish_reason;
+    }
+    if (delta.content) {
+      content += delta.content;
+      yield delta.content;
+    }
+  }
+  const outputMessage = {
+    role: "assistant",
+    content,
+    tokens: tokenCountForOpenAIMessage({
+      role: "assistant",
+      content
+    })
+  };
+  const responseData = {
+    ...logRequestData,
+    finishReason,
+    latency: performance.now() - startTime,
+    outputMessage,
+    tokensUsed: computeUsage([...renderedMessages, outputMessage])
+  };
+  logger.chatCompletionResponse("openai", responseData);
+}
+
+// src/lib/openai/index.ts
+var import_openai3 = require("openai");
 
 // src/lib/anthropic/Anthropic.tsx
 var import_sdk = __toESM(require("@anthropic-ai/sdk"));
-var import_tokenizer2 = require("@anthropic-ai/tokenizer");
+var import_tokenizer3 = require("@anthropic-ai/tokenizer");
 var defaultClient2 = null;
 var AnthropicClientContext = createContext(
   () => {
@@ -927,27 +1093,27 @@ function buildChatMessages(childrenXml) {
       messages.push({
         role: "user",
         content,
-        tokens: (0, import_tokenizer2.countTokens)(content)
+        tokens: (0, import_tokenizer3.countTokens)(content)
       });
     } else if (node.nodeName === "AssistantMessage") {
       const content = `${import_sdk.default.AI_PROMPT} ${node.textContent}`;
       messages.push({
         role: "assistant",
         content,
-        tokens: (0, import_tokenizer2.countTokens)(content)
+        tokens: (0, import_tokenizer3.countTokens)(content)
       });
     } else if (node.nodeName === "SystemMessage") {
       const userContent = `${import_sdk.default.HUMAN_PROMPT} For subsequent replies you will adhere to the following instructions: ${node.textContent}`;
       messages.push({
         role: "user",
         content: userContent,
-        tokens: (0, import_tokenizer2.countTokens)(userContent)
+        tokens: (0, import_tokenizer3.countTokens)(userContent)
       });
       const assistantContent = `${import_sdk.default.AI_PROMPT} Okay, I will do that.`;
       messages.push({
         role: "assistant",
         content: assistantContent,
-        tokens: (0, import_tokenizer2.countTokens)(assistantContent)
+        tokens: (0, import_tokenizer3.countTokens)(assistantContent)
       });
     }
   }
@@ -1016,7 +1182,7 @@ async function* AnthropicChatCompletion(props, { render, logger, getContext }) {
   const outputMessage = {
     role: "assistant",
     content,
-    tokens: (0, import_tokenizer2.countTokens)(content)
+    tokens: (0, import_tokenizer3.countTokens)(content)
   };
   const responseData = {
     ...logRequestData,
@@ -1030,7 +1196,7 @@ async function* AnthropicChatCompletion(props, { render, logger, getContext }) {
 
 // src/lib/anthropic/index.ts
 var import_sdk2 = __toESM(require("@anthropic-ai/sdk"));
-var import_tokenizer3 = require("@anthropic-ai/tokenizer");
+var import_tokenizer4 = require("@anthropic-ai/tokenizer");
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {
   AIFragment,
@@ -1042,12 +1208,14 @@ var import_tokenizer3 = require("@anthropic-ai/tokenizer");
   ChatCompletionError,
   CombinedLogger,
   ConsoleLogger,
+  ContentTypeImage,
   LogImplementation,
   LoggerContext,
   NoopLogImplementation,
   OpenAIChatCompletion,
   OpenAIClient,
   OpenAIClientContext,
+  OpenAIVisionChatCompletion,
   SystemMessage,
   UserMessage,
   attachedContextSymbol,

package/dist/index.mjs

@@ -309,7 +309,7 @@ function parseXml(input) {
   return constructNode(null, parsed[0]);
 }
 function escape(html) {
-  return html.replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;").replace(/"/g, "&quot;").replace(/'/g, "&#039;");
+  return html.replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;").replace(/"/g, "&quot;");
 }
 
 // src/context.ts
@@ -811,8 +811,172 @@ async function* OpenAIChatCompletion(props, { logger, render, getContext }) {
   logger.chatCompletionResponse("openai", responseData);
 }
 
-// src/lib/openai/index.ts
+// src/lib/openai/OpenAIVision.tsx
 import { OpenAI as OpenAIClient2 } from "openai";
+var DEFAULT_MODEL = "gpt-4-vision-preview";
+var ContentTypeImage = (_props) => {
+  return null;
+};
+function buildOpenAIVisionChatMessages(childrenXml) {
+  const messages = [];
+  const chatMessageTags2 = [
+    "UserMessage",
+    "AssistantMessage",
+    "SystemMessage",
+    "ContentTypeImage"
+  ];
+  const parsed = parseXml(childrenXml).collapse(chatMessageTags2);
+  const topLevelValid = parsed.childNodes.every(
+    (node) => chatMessageTags2.includes(node.nodeName)
+  );
+  if (!topLevelValid) {
+    throw new Error("Invalid top level chat message tags");
+  }
+  for (const node of parsed.childNodes) {
+    if (node.nodeName === "UserMessage") {
+      const parts = node.childNodes.map((n) => {
+        if (n.nodeName === "#text") {
+          return {
+            type: "text",
+            text: n.value
+          };
+        } else if (n.nodeName === "ContentTypeImage") {
+          return {
+            type: "image_url",
+            image_url: {
+              url: n.attributes.url,
+              detail: n.attributes.detail || "auto"
+            }
+          };
+        }
+        throw new Error(
+          "Invalid ChatCompletionContentPart, expecting text or ContentTypeImage"
+        );
+      });
+      messages.push({
+        content: parts,
+        role: "user"
+      });
+    } else if (node.nodeName === "AssistantMessage") {
+      messages.push({
+        content: node.textContent,
+        role: "assistant"
+      });
+    } else if (node.nodeName === "SystemMessage") {
+      messages.push({
+        content: node.textContent,
+        role: "system"
+      });
+    }
+  }
+  return messages;
+}
+async function* OpenAIVisionChatCompletion(props, { logger, render, getContext }) {
+  const startTime = performance.now();
+  const model = props.model || DEFAULT_MODEL;
+  const client = getContext(OpenAIClientContext)();
+  if (!client) {
+    throw new Error("[OpenAI] must supply OpenAI model via context");
+  }
+  const openAIMessages = buildOpenAIVisionChatMessages(
+    await render(props.children, {
+      preserveTags: true,
+      renderedProps: {
+        ContentTypeImage: {
+          url: true,
+          detail: true
+        }
+      }
+    })
+  );
+  const renderedMessages = openAIMessages.map((message) => {
+    const renderContent = (content2) => {
+      if (content2 == null) {
+        return "";
+      }
+      if (typeof content2 === "string") {
+        return content2;
+      }
+      return content2.map((part) => {
+        if (part.type === "text") {
+          return part.text;
+        } else if (part.type === "image_url") {
+          return `<ContentTypeImage url="${part.image_url.url}" detail="${part.image_url.detail || "auto"}" />`;
+        }
+        throw new Error("Invalid ChatCompletionContentPart type");
+      }).join(" ");
+    };
+    return {
+      role: message.role,
+      content: renderContent(message.content),
+      tokens: tokenCountForOpenAIVisionMessage(message)
+    };
+  });
+  const chatCompletionRequest = {
+    model,
+    max_tokens: props.maxTokens,
+    temperature: props.temperature,
+    messages: openAIMessages,
+    stream: true
+  };
+  const logRequestData = {
+    startTime,
+    model,
+    provider: props.provider,
+    providerRegion: props.providerRegion,
+    inputMessages: renderedMessages,
+    request: chatCompletionRequest
+  };
+  logger.chatCompletionRequest("openai", logRequestData);
+  let chatResponse;
+  try {
+    chatResponse = await client.chat.completions.create(chatCompletionRequest);
+  } catch (ex) {
+    if (ex instanceof OpenAIClient2.APIError) {
+      throw new ChatCompletionError(
+        `OpenAIClient.APIError: ${ex.message}`,
+        logRequestData
+      );
+    } else if (ex instanceof Error) {
+      throw new ChatCompletionError(ex.message, logRequestData);
+    }
+    throw ex;
+  }
+  let finishReason = void 0;
+  let content = "";
+  for await (const message of chatResponse) {
+    if (!message.choices || !message.choices[0]) {
+      continue;
+    }
+    const delta = message.choices[0].delta;
+    if (message.choices[0].finish_reason) {
+      finishReason = message.choices[0].finish_reason;
+    }
+    if (delta.content) {
+      content += delta.content;
+      yield delta.content;
+    }
+  }
+  const outputMessage = {
+    role: "assistant",
+    content,
+    tokens: tokenCountForOpenAIMessage({
+      role: "assistant",
+      content
+    })
+  };
+  const responseData = {
+    ...logRequestData,
+    finishReason,
+    latency: performance.now() - startTime,
+    outputMessage,
+    tokensUsed: computeUsage([...renderedMessages, outputMessage])
+  };
+  logger.chatCompletionResponse("openai", responseData);
+}
+
+// src/lib/openai/index.ts
+import { OpenAI as OpenAIClient3 } from "openai";
 
 // src/lib/anthropic/Anthropic.tsx
 import AnthropicClient from "@anthropic-ai/sdk";
@@ -960,12 +1124,14 @@ export {
   ChatCompletionError,
   CombinedLogger,
   ConsoleLogger,
+  ContentTypeImage,
   LogImplementation,
   LoggerContext,
   NoopLogImplementation,
   OpenAIChatCompletion,
-  OpenAIClient2 as OpenAIClient,
+  OpenAIClient3 as OpenAIClient,
   OpenAIClientContext,
+  OpenAIVisionChatCompletion,
   SystemMessage,
   UserMessage,
   attachedContextSymbol,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@gammatech/aijsx",
-  "version": "0.2.0-beta.1",
+  "version": "0.2.0-beta.3",
   "description": "Rewrite of aijsx",
   "author": "Jordan Garcia",
   "license": "MIT",