@saltcorn/agents 0.3.0 → 0.3.1

package/agent-view.js

@@ -39,6 +39,7 @@ const {
   wrapCard,
   wrapSegment,
   process_interaction,
+  find_image_tool,
 } = require("./common");
 const MarkdownIt = require("markdown-it"),
   md = new MarkdownIt();
@@ -157,7 +158,6 @@ const run = async (
     for (const interact of run.context.interactions) {
       switch (interact.role) {
         case "user":
-          console.log(interact.content);
           if (interact.content?.[0]?.type === "image_url") {
             const image_url = interact.content[0].image_url.url;
             if (image_url.startsWith("data"))
@@ -187,58 +187,74 @@ const run = async (
           break;
         case "assistant":
         case "system":
-          if (interact.tool_calls) {
-            if (interact.content) {
-              interactMarkups.push(
-                div(
-                  { class: "interaction-segment" },
-                  span({ class: "badge bg-secondary" }, action.name),
-                  typeof interact.content === "string"
-                    ? md.render(interact.content)
-                    : interact.content
-                )
-              );
+
+          for (const tool_call of interact.tool_calls || []) {
+            const toolSkill = find_tool(
+              tool_call.function.name,
+              action.configuration
+            );
+            if (toolSkill) {
+              const row = JSON.parse(tool_call.function.arguments);
+              if (toolSkill.tool.renderToolCall) {
+                const rendered = await toolSkill.tool.renderToolCall(row, {
+                  req,
+                });
+                if (rendered)
+                  interactMarkups.push(
+                    wrapSegment(
+                      wrapCard(
+                        toolSkill.skill.skill_label ||
+                          toolSkill.skill.constructor.skill_name,
+                        rendered
+                      ),
+                      action.name
+                    )
+                  );
+              }
             }
-            for (const tool_call of interact.tool_calls) {
-              const toolSkill = find_tool(
-                tool_call.function.name,
-                action.configuration
-              );
-              if (toolSkill) {
-                const row = JSON.parse(tool_call.function.arguments);
-                if (toolSkill.tool.renderToolCall) {
-                  const rendered = await toolSkill.tool.renderToolCall(row, {
+          }
+          for (const image_call of interact.content?.image_calls || []) {
+
+            const toolSkill = find_image_tool(action.configuration);
+            if (toolSkill) {
+              if (toolSkill.tool.renderToolResponse) {
+                const rendered = await toolSkill.tool.renderToolResponse(
+                  image_call,
+                  {
                     req,
-                  });
-                  if (rendered)
-                    interactMarkups.push(
-                      wrapSegment(
-                        wrapCard(
-                          toolSkill.skill.skill_label ||
-                            toolSkill.skill.constructor.skill_name,
-                          rendered
-                        ),
-                        action.name
-                      )
-                    );
-                }
+                  }
+                );
+
+                if (rendered)
+                  interactMarkups.push(
+                    wrapSegment(
+                      wrapCard(
+                        toolSkill.skill.skill_label ||
+                          toolSkill.skill.constructor.skill_name,
+                        rendered
+                      ),
+                      action.name
+                    )
+                  );
               }
             }
-          } else
-            interactMarkups.push(
-              div(
-                { class: "interaction-segment" },
-                span({ class: "badge bg-secondary" }, action.name),
-                typeof interact.content === "string"
-                  ? md.render(interact.content)
-                  : interact.content
-              )
-            );
+          }
+
+          interactMarkups.push(
+            div(
+              { class: "interaction-segment" },
+              span({ class: "badge bg-secondary" }, action.name),
+              typeof interact.content === "string"
+                ? md.render(interact.content)
+                : typeof interact.content?.content === "string"
+                ? md.render(interact.content.content)
+                : interact.content
+            )
+          );
           break;
         case "tool":
           if (interact.content !== "Action run") {
             let markupContent;
-            console.log("interact", interact);
             const toolSkill = find_tool(interact.name, action.configuration);
             try {
               if (toolSkill?.tool?.renderToolResponse)

package/common.js

@@ -12,6 +12,7 @@ const get_skills = () => {
     require("./skills/Trigger"),
     require("./skills/Table"),
     require("./skills/PreloadData"),
+    require("./skills/GenerateImage"),
     //require("./skills/AdaptiveFeedback"),
   ];
 };
@@ -34,7 +35,7 @@ const get_skill_instances = (config) => {
 const find_tool = (name, config) => {
   const skills = get_skill_instances(config);
   for (const skill of skills) {
-    const skillTools = skill.provideTools();
+    const skillTools = skill.provideTools?.();
     const tools = !skillTools
       ? []
       : Array.isArray(skillTools)
@@ -45,6 +46,20 @@ const find_tool = (name, config) => {
   }
 };
 
+const find_image_tool = (config) => {
+  const skills = get_skill_instances(config);
+  for (const skill of skills) {
+    const skillTools = skill.provideTools?.();
+    const tools = !skillTools
+      ? []
+      : Array.isArray(skillTools)
+      ? skillTools
+      : [skillTools];
+    const found = tools.find((t) => t?.type === "image_generation");
+    if (found) return { tool: found, skill };
+  }
+};
+
 const getCompletionArguments = async (config, user) => {
   let tools = [];
 
@@ -52,7 +67,7 @@ const getCompletionArguments = async (config, user) => {
 
   const skills = get_skill_instances(config);
   for (const skill of skills) {
-    const sysPr = await skill.systemPrompt({ user });
+    const sysPr = await skill.systemPrompt?.({ user });
     if (sysPr) sysPrompts.push(sysPr);
     const skillTools = skill.provideTools?.();
     if (skillTools && Array.isArray(skillTools)) tools.push(...skillTools);
@@ -133,10 +148,43 @@ const process_interaction = async (
   const complArgs = await getCompletionArguments(config, req.user);
   complArgs.chat = run.context.interactions;
   //complArgs.debugResult = true;
-  console.log("complArgs", JSON.stringify(complArgs, null, 2));
+  //console.log("complArgs", JSON.stringify(complArgs, null, 2));
 
   const answer = await getState().functions.llm_generate.run("", complArgs);
-  console.log("answer", answer);
+  //console.log("answer", answer);
+
+  const responses = [];
+  if (typeof answer === "object" && answer.image_calls) {
+    for (const image_call of answer.image_calls) {
+      const tool = find_image_tool(config);
+      let prcRes;
+      if (tool?.tool.process) {
+        prcRes = await tool.tool.process(image_call, { req });
+        if (prcRes?.result === null) delete image_call.result;
+        if (prcRes?.filename) image_call.filename = prcRes?.filename;
+      }
+      if (tool?.tool.renderToolResponse) {
+        const rendered = await tool.tool.renderToolResponse(
+          { ...image_call, ...(prcRes || {}) },
+          {
+            req,
+          }
+        );
+        if (rendered)
+          responses.push(
+            wrapSegment(
+              wrapCard(
+                tool.skill.skill_label || tool.skill.constructor.skill_name,
+                rendered
+              ),
+              agent_label
+            )
+          );
+      }
+    }
+    if (answer.content && !answer.tool_calls)
+      responses.push(wrapSegment(md.render(answer.content), agent_label));
+  }
   await addToContext(run, {
     interactions:
       typeof answer === "object" && answer.tool_calls
@@ -149,8 +197,6 @@ const process_interaction = async (
           ]
         : [{ role: "assistant", content: answer }],
   });
-  const responses = [];
-
   if (typeof answer === "object" && answer.tool_calls) {
     if (answer.content)
       responses.push(wrapSegment(md.render(answer.content), agent_label));
@@ -213,6 +259,7 @@ const process_interaction = async (
             {
               role: "tool",
               tool_call_id: tool_call.id,
+              call_id: tool_call.call_id,
               name: tool_call.function.name,
               content:
                 result && typeof result !== "string"
@@ -228,7 +275,8 @@ const process_interaction = async (
         ...prevResponses,
         ...responses,
       ]);
-  } else responses.push(wrapSegment(md.render(answer), agent_label));
+  } else if (typeof answer === "string")
+    responses.push(wrapSegment(md.render(answer), agent_label));
 
   return {
     json: {
@@ -251,4 +299,5 @@ module.exports = {
   wrapCard,
   wrapSegment,
   process_interaction,
+  find_image_tool,
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@saltcorn/agents",
-  "version": "0.3.0",
+  "version": "0.3.1",
   "description": "AI agents for Saltcorn",
   "main": "index.js",
   "dependencies": {

package/skills/GenerateImage.js

@@ -0,0 +1,98 @@
+const { div, pre } = require("@saltcorn/markup/tags");
+const Workflow = require("@saltcorn/data/models/workflow");
+const Form = require("@saltcorn/data/models/form");
+const Table = require("@saltcorn/data/models/table");
+const File = require("@saltcorn/data/models/file");
+const View = require("@saltcorn/data/models/view");
+const { getState } = require("@saltcorn/data/db/state");
+const db = require("@saltcorn/data/db");
+const { eval_expression } = require("@saltcorn/data/models/expression");
+const { interpolate } = require("@saltcorn/data/utils");
+
+class GenerateImage {
+  static skill_name = "Image generation";
+
+  get skill_label() {
+    return `Image generation`;
+  }
+
+  constructor(cfg) {
+    Object.assign(this, cfg);
+  }
+
+  static async configFields() {
+    return [
+      {
+        name: "quality",
+        label: "Quality",
+        type: "String",
+        required: true,
+        attributes: { options: ["auto", "low", "medium", "high"] },
+      },
+      {
+        name: "size",
+        label: "Size",
+        type: "String",
+        required: true,
+        attributes: {
+          options: ["auto", "1024x1024", "1536x1024", "1024x1536"],
+        },
+      },
+      {
+        name: "format",
+        label: "Format",
+        type: "String",
+        required: true,
+        attributes: { options: ["png", "jpeg", "webp"] },
+      },
+      {
+        name: "transparent",
+        label: "Transparent",
+        type: "Bool",
+      },
+      {
+        name: "save_file",
+        label: "Save file",
+        type: "String",
+        required: true,
+        attributes: { options: ["Always", "Never"] }, //, "Button"] },
+      },
+    ];
+  }
+
+  provideTools() {
+    const tool = {
+      type: "image_generation",
+      size: this.size,
+      quality: this.quality,
+      process: async (v, { req }) => {
+        if (this.save_file === "Always") {
+          const buf = Buffer.from(v.result, "base64");
+          const file = await File.from_contents(
+            `genimg.${v.output_format}`,
+            `image/${v.output_format}`,
+            buf,
+            req?.user?.id,
+            100
+          );
+          return { filename: file.path_to_serve, result: null };
+        }
+      },
+      renderToolResponse: (v) => {
+        const [ws, hs] = v.size.split("x");
+        if (v.filename)
+          return `<img height="${+hs / 4}" width="${
+            +ws / 4
+          }" src="/files/serve/${v.filename}" />`;
+        else
+          return `<img height="${+hs / 4}" width="${+ws / 4}" src="data:image/${
+            v.output_format
+          };base64, ${v.result}" />`;
+      },
+    };
+    if (this.transparent) tool.background = "transparent";
+    return tool;
+  }
+}
+
+module.exports = GenerateImage;

package/skills/PreloadData.js

@@ -31,7 +31,6 @@ class PreloadData {
     );
 
     const rows = await table.getRows(q);
-    console.log("preload data rows", rows);
     if (this.contents_expr) {
       for (const row of rows)
         prompts.push(interpolate(this.contents_expr, row, user));