@saltcorn/large-language-model 1.0.8 → 1.0.10

package/generate.js

@@ -331,12 +331,17 @@ const addImageMesssage = async (
       }
       break;
     case "AI SDK":
+      let aisdk_image;
+      if (imageurl.startsWith("data:") && imageurl.includes("base64,")) {
+        const [_pre, b64] = imageurl.split("base64,");
+        aisdk_image = b64;
+      } else aisdk_image = imageurl;
       chat.push({
         role: "user",
         content: [
           {
             type: "image",
-            image: imageurl,
+            image: aisdk_image,
           },
         ],
       });
@@ -548,7 +553,12 @@ const getCompletionAISDK = async (
   const debugRequest = { ...body, model: use_model_name };
   if (debugResult)
     console.log("AI SDK request", JSON.stringify(debugRequest, null, 2));
-  getState().log(6, `AI SDK request ${JSON.stringify(debugRequest)} `);
+  getState().log(
+    6,
+    `AI SDK request`,
+    { tools: Object.keys(debugRequest.tools || {}), model: debugRequest.model },
+    JSON.stringify(debugRequest.messages, null, 2),
+  );
   if (debugCollector) debugCollector.request = debugRequest;
   const reqTimeStart = Date.now();
 
@@ -570,15 +580,18 @@ const getCompletionAISDK = async (
     chat.push(...results.response.messages);
   }
 
-  if (debugResult)
-    console.log("AI SDK response", JSON.stringify(results, null, 2));
-  else getState().log(6, `AI SDK response ${JSON.stringify(results)}`);
   if (debugCollector) {
     debugCollector.response = results;
     debugCollector.response_time_ms = Date.now() - reqTimeStart;
   }
   const allToolCalls = (await results.steps).flatMap((step) => step.toolCalls);
-
+  if (debugResult)
+    console.log("AI SDK response", JSON.stringify(results, null, 2));
+  else
+    getState().log(6, `AI SDK response`, {
+      text: results.text,
+      tool_calls: allToolCalls,
+    });
   if (allToolCalls.length) {
     return {
       tool_calls: allToolCalls,
@@ -833,10 +846,29 @@ const getCompletionOpenAICompatible = async (
       if (streamToolCalls) debugCollector.response = streamToolCalls;
       debugCollector.response_time_ms = Date.now() - reqTimeStart;
     }
+    if (appendToChat && chat && streamToolCalls) {
+      chat.push({
+        role: "assistant",
+        content: streamParts.join("") || null,
+        tool_calls: streamToolCalls.tool_calls,
+      });
+    } else if (appendToChat && chat && streamParts.length > 0) {
+      chat.push({ role: "assistant", content: streamParts.join("") });
+    }
     return streamToolCalls
       ? {
           content: streamParts.join(""),
           tool_calls: streamToolCalls.tool_calls,
+          hasToolCalls: streamToolCalls.tool_calls?.length,
+          getToolCalls() {
+            return streamToolCalls.tool_calls.map((tc) => ({
+              tool_name: tc.function.name,
+              input: tc.function.arguments
+                ? JSON.parse(tc.function.arguments)
+                : {},
+              tool_call_id: tc.id,
+            }));
+          },
         }
       : streamParts.join("");
   }
@@ -890,7 +922,9 @@ const getCompletionOpenAICompatible = async (
           getToolCalls() {
             return tool_calls.map((tc) => ({
               tool_name: tc.function.name,
-              input: JSON.parse(tc.function.arguments),
+              input: tc.function.arguments
+                ? JSON.parse(tc.function.arguments)
+                : {},
               tool_call_id: tc.call_id,
             }));
           },
@@ -905,7 +939,9 @@ const getCompletionOpenAICompatible = async (
           getToolCalls() {
             return results?.choices?.[0]?.message?.tool_calls.map((tc) => ({
               tool_name: tc.function.name,
-              input: JSON.parse(tc.function.arguments),
+              input: tc.function.arguments
+                ? JSON.parse(tc.function.arguments)
+                : {},
               tool_call_id: tc.id,
             }));
           },

package/index.js

@@ -658,6 +658,7 @@ module.exports = {
   functions,
   modelpatterns: require("./model.js"),
   routes,
+  ready_for_mobile: false,
   actions: (config) => ({
     llm_function_call: require("./function-insert-action.js")(config),
     llm_generate: {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@saltcorn/large-language-model",
-  "version": "1.0.8",
+  "version": "1.0.10",
   "description": "Large language models and functionality for Saltcorn",
   "main": "index.js",
   "dependencies": {

package/tests/llm.test.js

@@ -2,7 +2,8 @@ const { getState } = require("@saltcorn/data/db/state");
 const View = require("@saltcorn/data/models/view");
 const Table = require("@saltcorn/data/models/table");
 const Plugin = require("@saltcorn/data/models/plugin");
-
+const fs = require("fs");
+const path = require("path");
 const { mockReqRes } = require("@saltcorn/data/tests/mocks");
 const { afterAll, beforeAll, describe, it, expect } = require("@jest/globals");
 
@@ -97,14 +98,14 @@ for (const nameconfig of require("./configs")) {
     });
     it("uses tools", async () => {
       const answer = await getState().functions.llm_generate.run(
-        "Generate a list of all the EU capitals in a structured format using the provided tool",
+        "Generate a list of all the EU capitals in a structured format using the cities tool",
         cities_tool,
       );
       expect(typeof answer).toBe("object");
-      const cities = answer.ai_sdk
-        ? answer.tool_calls[0].input?.cities
-        : JSON.parse(answer.tool_calls[0].function.arguments).cities;
+      const cities = answer.getToolCalls()[0].input?.cities;
       expect(cities.length).toBe(27);
+      expect(typeof cities[0].country_name).toBe("string");
+      expect(typeof cities[0].city_name).toBe("string");
     });
     it("appends to chat history", async () => {
       const chat = [];
@@ -131,7 +132,7 @@ for (const nameconfig of require("./configs")) {
     it("tool use sequence", async () => {
       const chat = [];
       const answer = await getState().functions.llm_generate.run(
-        "Generate a list of all the EU capitals in a structured format using the provided tool",
+        "Generate a list of all the EU capitals in a structured format using the cities tool",
         {
           chat,
           appendToChat: true,
@@ -185,6 +186,24 @@ for (const nameconfig of require("./configs")) {
       expect(!!json_answer.cities[0].city_name).toBe(true);
       expect(!!json_answer.cities[0].country_name).toBe(true);
     });
+    it("reads images", async () => {
+      const chat = [];
+      const b64 = fs
+          .readFileSync(path.join(__dirname, "enjoy.png"))
+          .toString("base64"),
+        imageurl = `data:image/png;base64,${b64}`;
+      await getState().functions.llm_add_message.run("image", imageurl, {
+        chat,
+      });
+      const answer = await getState().functions.llm_generate.run(
+        "What is written in this picture?",
+        {
+          chat,
+        },
+      );
+      expect(typeof answer).toBe("string");
+      expect(answer.toLowerCase()).toContain("coffee");
+    });
     if (name !== "AI SDK Anthropic")
       it("gets embedding", async () => {
         const v = await getState().functions.llm_embedding.run(