@saltcorn/large-language-model 1.0.8 → 1.0.9

package/generate.js

@@ -331,12 +331,17 @@ const addImageMesssage = async (
       }
       break;
     case "AI SDK":
+      let aisdk_image;
+      if (imageurl.startsWith("data:") && imageurl.includes("base64,")) {
+        const [_pre, b64] = imageurl.split("base64,");
+        aisdk_image = b64;
+      } else aisdk_image = imageurl;
       chat.push({
         role: "user",
         content: [
           {
             type: "image",
-            image: imageurl,
+            image: aisdk_image,
           },
         ],
       });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@saltcorn/large-language-model",
-  "version": "1.0.8",
+  "version": "1.0.9",
   "description": "Large language models and functionality for Saltcorn",
   "main": "index.js",
   "dependencies": {

package/tests/llm.test.js

@@ -2,7 +2,8 @@ const { getState } = require("@saltcorn/data/db/state");
 const View = require("@saltcorn/data/models/view");
 const Table = require("@saltcorn/data/models/table");
 const Plugin = require("@saltcorn/data/models/plugin");
-
+const fs = require("fs");
+const path = require("path");
 const { mockReqRes } = require("@saltcorn/data/tests/mocks");
 const { afterAll, beforeAll, describe, it, expect } = require("@jest/globals");
 
@@ -97,14 +98,14 @@ for (const nameconfig of require("./configs")) {
     });
     it("uses tools", async () => {
       const answer = await getState().functions.llm_generate.run(
-        "Generate a list of all the EU capitals in a structured format using the provided tool",
+        "Generate a list of all the EU capitals in a structured format using the cities tool",
         cities_tool,
       );
       expect(typeof answer).toBe("object");
-      const cities = answer.ai_sdk
-        ? answer.tool_calls[0].input?.cities
-        : JSON.parse(answer.tool_calls[0].function.arguments).cities;
+      const cities = answer.getToolCalls()[0].input?.cities;
       expect(cities.length).toBe(27);
+      expect(typeof cities[0].country_name).toBe("string");
+      expect(typeof cities[0].city_name).toBe("string");
     });
     it("appends to chat history", async () => {
       const chat = [];
@@ -131,7 +132,7 @@ for (const nameconfig of require("./configs")) {
     it("tool use sequence", async () => {
       const chat = [];
       const answer = await getState().functions.llm_generate.run(
-        "Generate a list of all the EU capitals in a structured format using the provided tool",
+        "Generate a list of all the EU capitals in a structured format using the cities tool",
         {
           chat,
           appendToChat: true,
@@ -185,6 +186,24 @@ for (const nameconfig of require("./configs")) {
       expect(!!json_answer.cities[0].city_name).toBe(true);
       expect(!!json_answer.cities[0].country_name).toBe(true);
     });
+    it("reads images", async () => {
+      const chat = [];
+      const b64 = fs
+          .readFileSync(path.join(__dirname, "enjoy.png"))
+          .toString("base64"),
+        imageurl = `data:image/png;base64,${b64}`;
+      await getState().functions.llm_add_message.run("image", imageurl, {
+        chat,
+      });
+      const answer = await getState().functions.llm_generate.run(
+        "What is written in this picture?",
+        {
+          chat,
+        },
+      );
+      expect(typeof answer).toBe("string");
+      expect(answer.toLowerCase()).toContain("coffee");
+    });
     if (name !== "AI SDK Anthropic")
       it("gets embedding", async () => {
         const v = await getState().functions.llm_embedding.run(