@saltcorn/agents 0.8.5 → 0.8.7

package/consolidate_agent_memory.js

@@ -0,0 +1,65 @@
+const FieldRepeat = require("@saltcorn/data/models/fieldrepeat");
+const Table = require("@saltcorn/data/models/table");
+const View = require("@saltcorn/data/models/view");
+const { p } = require("@saltcorn/markup/tags");
+const { get_skills, process_interaction, wrapSegment } = require("./common");
+const { applyAsync } = require("@saltcorn/data/utils");
+const WorkflowRun = require("@saltcorn/data/models/workflow_run");
+const { interpolate, escapeHtml } = require("@saltcorn/data/utils");
+const { getState } = require("@saltcorn/data/db/state");
+
+module.exports = {
+  //configFields: async ({ table, mode }) => [],
+  run: async ({ configuration, user, table, req, ...rest }) => {
+    const memtable = Table.findOne("AgentLongTermMemory");
+    if (!memtable) return;
+    // write personal preferences
+
+    // get all personal observations
+    const personalmems = await memtable.getRows(
+      { personal: true },
+      { orderBy: "written_at" },
+    );
+
+    const user_ids = new Set(personalmems.map((p) => p.user_id));
+
+    for (const uid of user_ids.values()) {
+      const all_umems = personalmems.filter((m) => m.user_id === uid);
+      const existingPP = all_umems.find(
+        (m) => m.memory_type === "PersonalPreferences",
+      );
+      const nonPPs = all_umems.filter(
+        (m) => m.memory_type !== "PersonalPreferences",
+      );
+
+      if (
+        existingPP &&
+        nonPPs.every((m) => m.written_at < existingPP.written_at)
+      )
+        continue; // no new memories
+      const prompt = `This is a set of observations about a user:
+${nonPPs.map((m) => `* ${m.contents}`).join("\n")}
+
+They are in ascending chronological order so if there are any contradictions, the latter observation take precedence.
+
+Write a succinct summary of these observations which captures all the essential facts. Start the summary with the words
+"The user..." (in the language in which the observations appear) and then write what you have learned about the user.`;
+
+      const answer = await getState().functions.llm_generate.run(prompt);
+
+      if (answer && typeof answer === "string") {
+        await memtable.deleteRows({
+          memory_type: "PersonalPreferences",
+          user_id: uid,
+        });
+        await memtable.insertRow({
+          user_id: uid,
+          written_at: new Date(),
+          memory_type: "PersonalPreferences",
+          contents: answer,
+          personal: true,
+        });
+      }
+    }
+  },
+};

package/index.js

@@ -36,6 +36,7 @@ module.exports = {
   ],
   actions: {
     Agent: require("./action"),
+    consolidate_agent_memory: require("./consolidate_agent_memory"),
   },
   functions: {
     inspect_agent: {
@@ -69,6 +70,7 @@ module.exports = {
         };
       },
       isAsync: true,
+      hidden: true,
       description: "Return system prompt, tools and action of an agent",
     },
     agent_generate: {
@@ -115,6 +117,7 @@ module.exports = {
         };
       },
       isAsync: true,
+      hidden: true,
       description: "Run an agent on a prompt",
       arguments: [
         { name: "agent_name", type: "String" },

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@saltcorn/agents",
-  "version": "0.8.5",
+  "version": "0.8.7",
   "description": "AI agents for Saltcorn",
   "main": "index.js",
   "dependencies": {

package/skills/GenerateImage.js

@@ -1,19 +1,11 @@
-const { div, pre } = require("@saltcorn/markup/tags");
-const Workflow = require("@saltcorn/data/models/workflow");
-const Form = require("@saltcorn/data/models/form");
-const Table = require("@saltcorn/data/models/table");
-const File = require("@saltcorn/data/models/file");
-const View = require("@saltcorn/data/models/view");
 const { getState } = require("@saltcorn/data/db/state");
-const db = require("@saltcorn/data/db");
-const { eval_expression } = require("@saltcorn/data/models/expression");
-const { interpolate } = require("@saltcorn/data/utils");
+const File = require("@saltcorn/data/models/file");
 
 class GenerateImage {
   static skill_name = "Image generation";
 
   get skill_label() {
-    return `Image generation`;
+    return "Image generation";
   }
 
   constructor(cfg) {
@@ -23,20 +15,31 @@ class GenerateImage {
   static async configFields() {
     return [
       {
-        name: "quality",
-        label: "Quality",
+        name: "size",
+        label: "Size",
         type: "String",
         required: true,
-        attributes: { options: ["auto", "low", "medium", "high"] },
+        attributes: {
+          options: [
+            "auto",
+            "1024x1024",
+            "1536x1024",
+            "1024x1536",
+            "1792x1024",
+            "1024x1792",
+          ],
+        },
+        default: "auto",
       },
       {
-        name: "size",
-        label: "Size",
+        name: "quality",
+        label: "Quality",
         type: "String",
         required: true,
         attributes: {
-          options: ["auto", "1024x1024", "1536x1024", "1024x1536"],
+          options: ["auto", "low", "medium", "high", "standard", "hd"],
         },
+        default: "auto",
       },
       {
         name: "format",
@@ -44,54 +47,102 @@ class GenerateImage {
         type: "String",
         required: true,
         attributes: { options: ["png", "jpeg", "webp"] },
+        default: "png",
       },
       {
         name: "transparent",
-        label: "Transparent",
+        label: "Transparent background",
         type: "Bool",
-      },
-      {
-        name: "save_file",
-        label: "Save file",
-        type: "String",
-        required: true,
-        attributes: { options: ["Always", "Never"] }, //, "Button"] },
+        sublabel: "Only supported with gpt-image-1.",
       },
     ];
   }
 
   provideTools() {
-    const tool = {
-      type: "image_generation",
-      size: this.size,
-      quality: this.quality,
-      process: async (v, { req }) => {
-        if (this.save_file === "Always") {
-          const buf = Buffer.from(v.result, "base64");
-          const file = await File.from_contents(
-            `genimg.${v.output_format}`,
-            `image/${v.output_format}`,
-            buf,
-            req?.user?.id,
-            100
+    const skill = this;
+    return {
+      type: "function",
+      function: {
+        name: "generate_image",
+        description:
+          "Generate an image from a text prompt. The image is automatically " +
+          "displayed to the user in the chat — DO NOT include the image URL or " +
+          "any markdown image syntax in your reply. Just confirm in words what " +
+          "was generated.",
+        parameters: {
+          type: "object",
+          properties: {
+            prompt: {
+              type: "string",
+              description: "Detailed description of the image to generate.",
+            },
+          },
+          required: ["prompt"],
+        },
+      },
+      process: async (input, { req }) => {
+        const fn = getState().functions.llm_image_generate;
+        if (!fn)
+          throw new Error(
+            "LLM plugin not available (llm_image_generate not registered)",
           );
-          return { filename: file.path_to_serve, result: null };
-        }
+        const opts = {};
+        if (skill.size && skill.size !== "auto") opts.size = skill.size;
+        if (skill.quality && skill.quality !== "auto")
+          opts.quality = skill.quality;
+        if (skill.format) opts.output_format = skill.format;
+        if (skill.transparent) opts.background = "transparent";
+        // gpt-image-* family supports the rich options above (quality, format,
+        // background). The LLM-plugin's image gen will pick them up.
+        const result = await fn.run(input.prompt, opts);
+        const b64 = result?.b64_json;
+        if (!b64) throw new Error("Image generation returned no data");
+        const ext = result?.output_format || skill.format || "png";
+        const buf = Buffer.from(b64, "base64");
+        const file = await File.from_contents(
+          `genimg.${ext}`,
+          `image/${ext}`,
+          buf,
+          req?.user?.id,
+          100,
+        );
+        const sizeOut =
+          skill.size && skill.size !== "auto" ? skill.size : "1024x1024";
+        // Two response paths:
+        //   - add_response: rendered HTML the user actually sees in chat
+        //     (with download overlay). Saved in interactions.
+        //   - return shape consumed by the LLM tool_response — kept terse
+        //     and free of any URL so the model cannot leak a broken markdown
+        //     image link into its textual reply.
+        return {
+          ok: true,
+          message:
+            "Image generated and displayed to the user. Do NOT include the image URL or any markdown image syntax in your textual reply.",
+          // Hidden internal fields used only by renderToolResponse:
+          filename: file.path_to_serve,
+          output_format: ext,
+          size: sizeOut,
+        };
       },
       renderToolResponse: (v) => {
-        const [ws, hs] = v.size.split("x");
-        if (v.filename)
-          return `<img height="${+hs / 4}" width="${
-            +ws / 4
-          }" src="/files/serve/${v.filename}" />`;
-        else
-          return `<img height="${+hs / 4}" width="${+ws / 4}" src="data:image/${
-            v.output_format
-          };base64, ${v.result}" />`;
+        const sz = v?.size || "1024x1024";
+        const [ws, hs] = sz.split("x");
+        const w = +ws / 4;
+        const h = +hs / 4;
+        if (!v?.filename) return "";
+        const ext = v.output_format || "png";
+        const url = "/files/serve/" + v.filename;
+        const dlName = "image-" + Date.now() + "." + ext;
+        return (
+          `<div class="agent-generated-image">` +
+          `<img height="${h}" width="${w}" src="${url}" />` +
+          `<a href="${url}" download="${dlName}" ` +
+          `class="agent-image-download" title="Download image">` +
+          `<i class="fas fa-download"></i></a>` +
+          `</div>`
+        );
       },
     };
-    if (this.transparent) tool.background = "transparent";
-    return tool;
   }
 }
 

package/skills/LongTermMemory.js

@@ -20,12 +20,23 @@ class LongTermMemory {
     Object.assign(this, cfg);
   }
 
-  systemPrompt() {
+  async systemPrompt({ user }) {
+    let pp = "";
+    if (this.inject_personal_prefs && user?.id) {
+      const table = Table.findOne("AgentLongTermMemory");
+      if (table) {
+        const ppRow = await table.getRow({
+          memory_type: "PersonalPreferences",
+          user_id: user.id,
+        });
+        if (ppRow) pp = "\n\n" + ppRow.contents;
+      }
+    }
     return `You have access to a memory bank you can read from or write to. You should search the memory bank with the search_memory tool with any search terms that might be relevant to the user's query or the result of a tool call. When you learn something noteworthy (from the user or from the result of a tool call) store it in memory with the store_in_memory tool. Mark it as personal if it is only true or relevant for the specific user. Don't tell the user when you are storing to and retrieving from memory. ${
       this.add_sys_prompt
         ? ` Additional instructions for the memory tools: ${this.add_sys_prompt}`
         : ""
-    }`;
+    }${pp}`;
   }
 
   static async configFields() {
@@ -36,6 +47,13 @@ class LongTermMemory {
         type: "String",
         fieldview: "textarea",
       },
+      {
+        name: "inject_personal_prefs",
+        label: "Personal preferences",
+        type: "Bool",
+        sublabel:
+          "Inject personal preferences generated by the consolidate_agent_memory action in the system prompt",
+      },
     ];
   }
 

package/skills/WebSearch.js

@@ -37,7 +37,7 @@ class WebSearchSkill {
         label: "Search provider",
         type: "String",
         required: true,
-        attributes: { options: ["By URL template"] },
+        attributes: { options: ["By URL template", "Firecrawl", "Tavily"] },
       },
       {
         name: "url_template",
@@ -47,6 +47,13 @@ class WebSearchSkill {
         required: true,
         showIf: { search_provider: "By URL template" },
       },
+      {
+        name: "api_key",
+        label: "API key",
+        type: "String",
+        required: true,
+        showIf: { search_provider: ["Firecrawl", "Tavily"] },
+      },
       {
         name: "header",
         label: "Header",
@@ -64,16 +71,73 @@ class WebSearchSkill {
     return {
       type: "function",
       process: async (row) => {
-        const fOpts = { method: "GET" };
-        if (this.header) {
-          const [key, val] = this.header.split(":");
-          const myHeaders = new Headers();
-          myHeaders.append(key, val.trim());
-          fOpts.headers = myHeaders;
+        switch (this.search_provider) {
+          case "Firecrawl":
+            {
+              const url = "https://api.firecrawl.dev/v2/search";
+              const options = {
+                method: "POST",
+                headers: {
+                  Authorization: "Bearer " + this.api_key,
+                  "Content-Type": "application/json",
+                },
+                body: JSON.stringify({
+                  query: row.search_phrase,
+                  sources: ["web"],
+                  categories: [],
+                  limit: 10,
+                  scrapeOptions: {
+                    onlyMainContent: false,
+                    maxAge: 172800000,
+                    parsers: ["pdf"],
+                    formats: [],
+                  },
+                }),
+              };
+
+              const response = await fetch(url, options);
+              const data = await response.json();
+              return data.data.web;
+            }
+            break;
+          case "Tavily":
+            {
+              const url = "https://api.tavily.com/search";
+              const options = {
+                method: "POST",
+                headers: {
+                  Authorization: "Bearer " + this.api_key,
+                  "Content-Type": "application/json",
+                },
+                body: JSON.stringify({
+                  query: row.search_phrase,
+                  search_depth: "advanced",
+                }),
+              };
+
+              const response = await fetch(url, options);
+              const data = await response.json();
+              return data.results;
+            }
+            break;
+          case "By URL template":
+          default:
+            {
+              const fOpts = { method: "GET" };
+              if (this.header) {
+                const [key, val] = this.header.split(":");
+                const myHeaders = new Headers();
+                myHeaders.append(key, val.trim());
+                fOpts.headers = myHeaders;
+              }
+              const url = interpolate(this.url_template, {
+                q: row.search_phrase,
+              });
+              const resp = await fetch(url);
+              return await resp.text();
+            }
+            break;
         }
-        const url = interpolate(this.url_template, { q: row.search_phrase });
-        const resp = await fetch(url);
-        return await resp.text();
       },
       function: {
         name: "web_search",