@saltcorn/large-language-model 1.0.0 → 1.0.1

package/generate.js

@@ -291,6 +291,58 @@ const toolResponse = async (
   }
 };
 
+const addImageMesssage = async (
+  { backend, apiKey, api_key, provider, ai_sdk_provider, responses_api },
+  opts,
+) => {
+  let chat = opts.chat;
+  let result = opts.prompt;
+  //console.log("chat", JSON.stringify(chat, null, 2));
+  let imageurl = opts.prompt;
+  switch (opts.backend || backend) {
+    case "OpenAI":
+      {
+        const new_chat_item = responses_api
+          ? {
+              role: "user",
+              content: [
+                {
+                  type: "input_image",
+                  image_url: imageurl,
+                },
+              ],
+            }
+          : {
+              role: "user",
+              content: [
+                {
+                  type: "image_url",
+                  image_url: {
+                    url: imageurl,
+                  },
+                },
+              ],
+            };
+
+        chat.push(new_chat_item);
+      }
+      break;
+    case "AI SDK":
+      chat.push({
+        role: "user",
+        content: [
+          {
+            type: "image",
+            image: imageurl,
+          },
+        ],
+      });
+
+      break;
+    default:
+  }
+};
+
 const getCompletion = async (config, opts) => {
   switch (config.backend) {
     case "AI SDK":
@@ -1110,4 +1162,5 @@ module.exports = {
   getImageGeneration,
   getAudioTranscription,
   toolResponse,
+  addImageMesssage,
 };

package/index.js

@@ -11,7 +11,8 @@ const {
   getEmbedding,
   getImageGeneration,
   getAudioTranscription,
-  toolResponse
+  toolResponse,
+  addImageMesssage,
 } = require("./generate");
 const { OPENAI_MODELS } = require("./constants.js");
 const { eval_expression } = require("@saltcorn/data/models/expression");
@@ -422,14 +423,26 @@ const functions = (config) => {
         { name: "options", type: "JSON", tstype: "any", required: true },
       ],
     },
-    llm_add_tool_response: {
-      run: async (prompt, opts) => {
-        const result = await toolResponse(config, { prompt, ...opts });
-        return result;
+    llm_add_message: {
+      run: async (what, prompt, opts) => {
+        switch (what) {
+          case "tool_response":
+            return await toolResponse(config, { prompt, ...opts });
+          case "image":
+            return await addImageMesssage(config, { prompt, ...opts });
+          default:
+            break;
+        }
       },
       isAsync: true,
-      description: "Insert the response to a tool call into a chat",
+      description: "Insert a tool response or an image in a chat",
       arguments: [
+        {
+          name: "what",
+          type: "String",
+          tstype: '"tool_response"|"image"',
+          required: true,
+        },
         { name: "prompt", type: "String", required: true },
         { name: "options", type: "JSON", tstype: "any" },
       ],

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@saltcorn/large-language-model",
-  "version": "1.0.0",
+  "version": "1.0.1",
   "description": "Large language models and functionality for Saltcorn",
   "main": "index.js",
   "dependencies": {

package/tests/llm.test.js

@@ -133,7 +133,12 @@ for (const nameconfig of require("./configs")) {
       const chat = [];
       const answer = await getState().functions.llm_generate.run(
         "Generate a list of EU capitals in a structured format using the provided tool",
-        { chat, appendToChat: true, ...cities_tool, streamCallback() {} },
+        {
+          chat,
+          appendToChat: true,
+          ...cities_tool,
+          //streamCallback() {}
+        },
       );
       expect(typeof answer).toBe("object");
 
@@ -142,10 +147,14 @@ for (const nameconfig of require("./configs")) {
       const cities = tc.input.cities;
       expect(cities.length).toBe(27);
 
-      await getState().functions.llm_add_tool_response.run("List received", {
-        chat,
-        tool_call: tc,
-      });
+      await getState().functions.llm_add_message.run(
+        "tool_response",
+        "List received",
+        {
+          chat,
+          tool_call: tc,
+        },
+      );
 
       const answer1 = await getState().functions.llm_generate.run(
         "Make the same list in a structured format using the provided tool but for the original 12 member countries of the EU",