@saltcorn/agents 0.1.2 → 0.2.0

package/agent-view.js

@@ -2,6 +2,7 @@ const Field = require("@saltcorn/data/models/field");
 const Table = require("@saltcorn/data/models/table");
 const Form = require("@saltcorn/data/models/form");
 const View = require("@saltcorn/data/models/view");
+const File = require("@saltcorn/data/models/file");
 const Trigger = require("@saltcorn/data/models/trigger");
 const FieldRepeat = require("@saltcorn/data/models/fieldrepeat");
 const db = require("@saltcorn/data/db");
@@ -26,6 +27,7 @@ const {
   small,
   form,
   textarea,
+  label,
   a,
 } = require("@saltcorn/markup/tags");
 const { getState } = require("@saltcorn/data/db/state");
@@ -45,7 +47,7 @@ const configuration_workflow = (req) =>
   new Workflow({
     steps: [
       {
-        name: req.__("Agent action"),
+        name: "Agent action",
         form: async (context) => {
           const agent_actions = await Trigger.find({ action: "Agent" });
           return new Form({
@@ -68,7 +70,7 @@ const configuration_workflow = (req) =>
                       "data-dyn-href": `\`/actions/configure/\${action_id}\``,
                       target: "_blank",
                     },
-                    req.__("Configure")
+                    "Configure"
                   ),
               },
               {
@@ -80,7 +82,27 @@ const configuration_workflow = (req) =>
                 name: "placeholder",
                 label: "Placeholder",
                 type: "String",
-                default: "How can I help you?"
+                default: "How can I help you?",
+              },
+              {
+                name: "explainer",
+                label: "Explainer",
+                type: "String",
+                sublabel:
+                  "Appears below the input box. Use for additional instructions.",
+              },
+              {
+                name: "image_upload",
+                label: "Upload images",
+                sublabel: "Allow the user to upload images",
+                type: "Bool",
+              },
+              {
+                name: "image_base64",
+                label: "base64 encode",
+                sublabel: "Use base64 encoding in the OpenAI API",
+                type: "Bool",
+                showIf: { image_upload: true },
               },
             ],
           });
@@ -91,10 +113,30 @@ const configuration_workflow = (req) =>
 
 const get_state_fields = () => [];
 
+const uploadForm = (viewname, req) =>
+  span(
+    {
+      class: "attach_agent_image_wrap",
+    },
+    label(
+      { class: "btn-link", for: "attach_agent_image" },
+      i({ class: "fas fa-paperclip" })
+    ),
+    input({
+      id: "attach_agent_image",
+      name: "file",
+      type: "file",
+      class: "d-none",
+      accept: "image/*",
+      onchange: `agent_file_attach(event)`,
+    }),
+    span({ class: "ms-2 filename-label" })
+  );
+
 const run = async (
   table_id,
   viewname,
-  { action_id, show_prev_runs, placeholder },
+  { action_id, show_prev_runs, placeholder, explainer, image_upload },
   state,
   { res, req }
 ) => {
@@ -115,13 +157,33 @@ const run = async (
     for (const interact of run.context.interactions) {
       switch (interact.role) {
         case "user":
-          interactMarkups.push(
-            div(
-              { class: "interaction-segment" },
-              span({ class: "badge bg-secondary" }, "You"),
-              md.render(interact.content)
-            )
-          );
+          console.log(interact.content);
+          if (interact.content?.[0]?.type === "image_url") {
+            const image_url = interact.content[0].image_url.url;
+            if (image_url.startsWith("data"))
+              interactMarkups.push(
+                div(
+                  { class: "interaction-segment" },
+                  span({ class: "badge bg-secondary" }, "You"),
+                  "File"
+                )
+              );
+            else
+              interactMarkups.push(
+                div(
+                  { class: "interaction-segment" },
+                  span({ class: "badge bg-secondary" }, "You"),
+                  a({ href: image_url, target: "_blank" }, "File")
+                )
+              );
+          } else
+            interactMarkups.push(
+              div(
+                { class: "interaction-segment" },
+                span({ class: "badge bg-secondary" }, "You"),
+                md.render(interact.content)
+              )
+            );
           break;
         case "assistant":
         case "system":
@@ -209,7 +271,7 @@ const run = async (
   }
   const input_form = form(
     {
-      onsubmit: `event.preventDefault();spin_send_button();view_post('${viewname}', 'interact', $(this).serialize(), processCopilotResponse);return false;`,
+      onsubmit: `event.preventDefault();spin_send_button();view_post('${viewname}', 'interact', new FormData(this), processCopilotResponse);return false;`,
       class: "form-namespace copilot mt-2",
       method: "post",
     },
@@ -238,7 +300,9 @@ const run = async (
       span(
         { class: "submit-button p-2", onclick: "$('form.copilot').submit()" },
         i({ id: "sendbuttonicon", class: "far fa-paper-plane" })
-      )
+      ),
+      image_upload && uploadForm(viewname, req),
+      explainer && small({ class: "explainer" }, i(explainer))
     )
   );
 
@@ -283,17 +347,6 @@ const run = async (
     { class: "card" },
     div(
       { class: "card-body" },
-      script({
-        src: `/static_assets/${db.connectObj.version_tag}/mermaid.min.js`,
-      }),
-      script(
-        { type: "module" },
-        `mermaid.initialize({securityLevel: 'loose'${
-          getState().getLightDarkMode(req.user) === "dark"
-            ? ",theme: 'dark',"
-            : ""
-        }});`
-      ),
       div({ id: "copilotinteractions" }, runInteractions),
       input_form,
       style(
@@ -305,11 +358,22 @@ const run = async (
             div.prevcopilotrun i.fa-trash-alt {display: none;}
             div.prevcopilotrun:hover i.fa-trash-alt {display: block;}
             .copilot-entry .submit-button:hover { cursor: pointer}
+            .copilot-entry span.attach_agent_image_wrap i:hover { cursor: pointer}
 
             .copilot-entry .submit-button {
               position: relative; 
               top: -1.8rem;
               left: 0.1rem;              
+            }
+              .copilot-entry span.attach_agent_image_wrap {
+              position: relative; 
+              top: -1.8rem;
+              left: 0.2rem;              
+            }
+              .copilot-entry .explainer {
+              position: relative; 
+              top: -1.2rem;    
+              display: block;                        
             }
             .copilot-entry {margin-bottom: -1.25rem; margin-top: 1rem;}
             p.prevrun_content {
@@ -320,17 +384,25 @@ const run = async (
     text-overflow: ellipsis;}`
       ),
       script(`function processCopilotResponse(res) {
+        const hadFile = $("input#attach_agent_image").val();
+        $("span.filename-label").text("");
+        $("input#attach_agent_image").val(null);
         $("#sendbuttonicon").attr("class","far fa-paper-plane");
         const $runidin= $("input[name=run_id")
         if(res.run_id && (!$runidin.val() || $runidin.val()=="undefined"))
           $runidin.val(res.run_id);
         const wrapSegment = (html, who) => '<div class="interaction-segment"><span class="badge bg-secondary">'+who+'</span>'+html+'</div>'
         $("#copilotinteractions").append(wrapSegment('<p>'+$("textarea[name=userinput]").val()+'</p>', "You"))
-        $("textarea[name=userinput]").val("")      
+        if(hadFile)
+          $("#copilotinteractions").append(wrapSegment('File', "You"))
+        $("textarea[name=userinput]").val("")
 
         if(res.response)
             $("#copilotinteractions").append(res.response)
     }
+    function agent_file_attach(e) {
+        $(".attach_agent_image_wrap span.filename-label").text(e.target.files[0].name)
+    }
     function restore_old_button_elem(btn) {
         const oldText = $(btn).data("old-text");
         btn.html(oldText);
@@ -390,12 +462,6 @@ const run = async (
     : main_chat;
 };
 
-/*
-
-build a workflow that asks the user for their name and age
-
-*/
-
 const interact = async (table_id, viewname, config, body, { req, res }) => {
   const { userinput, run_id } = body;
   let run;
@@ -416,6 +482,41 @@ const interact = async (table_id, viewname, config, body, { req, res }) => {
       interactions: [{ role: "user", content: userinput }],
     });
   }
+  if (config.image_upload && req.files?.file) {
+    const file = await File.from_req_files(
+      req.files.file,
+      req.user ? req.user.id : null,
+      100
+      // file_field?.attributes?.folder
+    );
+    const baseUrl = getState().getConfig("base_url").replace(/\/$/, "");
+    let imageurl;
+    if (
+      !config.image_base64 &&
+      baseUrl &&
+      !baseUrl.includes("http://localhost:")
+    ) {
+      imageurl = `${baseUrl}/files/serve/${file.path_to_serve}`;
+    } else {
+      const b64 = await file.get_contents("base64");
+      imageurl = `data:${file.mimetype};base64,${b64}`;
+    }
+    await addToContext(run, {
+      interactions: [
+        {
+          role: "user",
+          content: [
+            {
+              type: "image_url",
+              image_url: {
+                url: imageurl,
+              },
+            },
+          ],
+        },
+      ],
+    });
+  }
   const action = await Trigger.findOne({ id: config.action_id });
 
   return await process_interaction(run, action.configuration, req, action.name);

package/common.js

@@ -9,6 +9,8 @@ const get_skills = () => {
   return [
     require("./skills/FTSRetrieval"),
     require("./skills/EmbeddingRetrieval"),
+    require("./skills/Trigger"),
+    require("./skills/Table"),
     //require("./skills/AdaptiveFeedback"),
   ];
 };
@@ -181,7 +183,8 @@ const process_interaction = async (
         }
         hasResult = true;
         const result = await tool.tool.process(
-          JSON.parse(tool_call.function.arguments)
+          JSON.parse(tool_call.function.arguments),
+          { req }
         );
         if (
           (typeof result === "object" && Object.keys(result || {}).length) ||

package/index.js

@@ -93,8 +93,9 @@ module.exports = {
 /* 
 TODO
 
--embedding retrieval test
--promote triggers to skills (optional user confirm)
+-embedding retrieval list view
+-optional user confirm: action, insert
+-Preload data
 -sql access
 -memory
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@saltcorn/agents",
-  "version": "0.1.2",
+  "version": "0.2.0",
   "description": "AI agents for Saltcorn",
   "main": "index.js",
   "dependencies": {

package/skills/EmbeddingRetrieval.js

@@ -37,17 +37,26 @@ class RetrievalByEmbedding {
     const allTables = await Table.find();
     const tableOpts = [];
     const relation_opts = {};
+    const list_view_opts = {};
     for (const table of allTables) {
-      table.fields
-        .filter((f) => f.type?.name === "PGVector")
-        .forEach((f) => {
-          const relNm = `${table.name}.${f.name}`;
-          tableOpts.push(relNm);
-          const fkeys = table.fields
-            .filter((f) => f.is_fkey)
-            .map((f) => f.name);
-          relation_opts[relNm] = ["", ...fkeys];
-        });
+      for (const f of table.fields) {
+        if (f.type?.name !== "PGVector") continue;
+        const relNm = `${table.name}.${f.name}`;
+        tableOpts.push(relNm);
+        list_view_opts[relNm] = [""];
+        const fkeys = table.fields.filter((f) => f.is_fkey).map((f) => f.name);
+        relation_opts[relNm] = ["", ...fkeys];
+        for (const fkeyField of table.fields.filter((f) => f.is_fkey)) {
+          const t = Table.findOne(fkeyField.reftable_name);
+          const lviews = await View.find_table_views_where(
+            t.id,
+            ({ state_fields, viewrow }) =>
+              viewrow.viewtemplate !== "Edit" &&
+              state_fields.every((sf) => !sf.required)
+          );
+          list_view_opts[relNm].push(...lviews.map((v) => v.name));
+        }
+      }
     }
     return [
       {
@@ -74,6 +83,14 @@ class RetrievalByEmbedding {
         required: true,
         attributes: { calcOptions: ["vec_field", relation_opts] },
       },
+      {
+        name: "list_view",
+        label: "List view",
+        type: "String",
+        attributes: {
+          calcOptions: ["table_name", list_view_opts],
+        },
+      },
       {
         name: "hidden_fields",
         label: "Hide fields",
@@ -90,6 +107,7 @@ class RetrievalByEmbedding {
         name: "add_sys_prompt",
         label: "Additional prompt",
         type: "String",
+        fieldview: "textarea",
       },
     ];
   }
@@ -152,6 +170,24 @@ class RetrievalByEmbedding {
           return { rows: docs };
         }
       },
+      renderToolResponse: async ({ response, rows }, { req }) => {
+        if (rows) {
+          const view = View.findOne({ name: this.list_view });
+
+          if (view) {
+            const viewRes = await view.run(
+              {
+                [table_docs.pk_name]: {
+                  in: rows.map((r) => r[table_docs.pk_name]),
+                },
+              },
+              { req }
+            );
+            return viewRes;
+          } else return "";
+        }
+        return div({ class: "border border-success p-2 m-2" }, response);
+      },
       function: {
         name: this.toolName,
         description: `Search the ${table_docs.name``} archive${

package/skills/FTSRetrieval.js

@@ -26,6 +26,10 @@ class RetrievalByFullTextSearch {
       return `Use the ${this.toolName} tool to search the ${
         this.table_name
       } database by a search phrase which will locate rows where any field match that query.${
+        this.list_view
+          ? ` When the tool call returns rows, do not describe them or repeat the information to the user. The results are already displayed to the user automatically.`
+          : ""
+      }${
         this.add_sys_prompt
           ? ` Additional information for the ${this.toolName} tool: ${this.add_sys_prompt}`
           : ""
@@ -78,6 +82,7 @@ class RetrievalByFullTextSearch {
         name: "add_sys_prompt",
         label: "Additional prompt",
         type: "String",
+        fieldview: "textarea",
       },
       /*{
         name: "contents_expr",

package/skills/Table.js

@@ -0,0 +1,241 @@
+const { div, pre } = require("@saltcorn/markup/tags");
+const Workflow = require("@saltcorn/data/models/workflow");
+const Form = require("@saltcorn/data/models/form");
+const Table = require("@saltcorn/data/models/table");
+const View = require("@saltcorn/data/models/view");
+const { getState } = require("@saltcorn/data/db/state");
+const db = require("@saltcorn/data/db");
+const { fieldProperties } = require("./helpers");
+
+class TableToSkill {
+  static skill_name = "Table";
+
+  get skill_label() {
+    return `${this.table_name} table`;
+  }
+
+  constructor(cfg) {
+    Object.assign(this, cfg);
+  }
+
+  systemPrompt() {
+    return `Use the query_${this.table_name} tool to search the ${
+      this.table_name
+    } database by a search phrase which will locate rows where any field match that query.${
+      this.add_sys_prompt ? ` ${this.add_sys_prompt}` : ""
+    }`;
+  }
+
+  static async configFields() {
+    const allTables = await Table.find();
+    const list_view_opts = {};
+    for (const t of allTables) {
+      const lviews = await View.find_table_views_where(
+        t.id,
+        ({ state_fields, viewrow }) =>
+          viewrow.viewtemplate !== "Edit" &&
+          state_fields.every((sf) => !sf.required)
+      );
+      list_view_opts[t.name] = ["", ...lviews.map((v) => v.name)];
+    }
+    return [
+      {
+        name: "table_name",
+        label: "Table",
+        sublabel: "Which table to link to the agent",
+        type: "String",
+        required: true,
+        attributes: { options: allTables.map((t) => t.name) },
+      },
+      {
+        name: "query",
+        label: "Query",
+        type: "Bool",
+        sublabel: "Allow the agent to query from this table",
+      },
+      {
+        name: "insert",
+        label: "Insert",
+        type: "Bool",
+        sublabel: "Allow the agent to insert new rows into this table",
+      },
+      {
+        name: "list_view",
+        label: "List view",
+        type: "String",
+        attributes: {
+          calcOptions: ["table_name", list_view_opts],
+        },
+      },
+      {
+        name: "hidden_fields",
+        label: "Hide fields",
+        type: "String",
+        sublabel: "Comma-separated list of fields to hide from the prompt",
+        showIf: { query: true },
+      },
+      {
+        name: "add_sys_prompt",
+        label: "Additional system prompt",
+        type: "String",
+        fieldview: "textarea"
+      },
+    ];
+  }
+
+  provideTools() {
+    const table = Table.findOne(this.table_name);
+    const tools = [];
+    let queryProperties = {};
+    let required = [];
+
+    table.fields
+      .filter((f) => !f.primary_key)
+      .forEach((field) => {
+        if (field.required && !field.primary_key) required.push(field.name);
+        queryProperties[field.name] = {
+          description: field.label + " " + field.description || "",
+          ...fieldProperties(field),
+        };
+      });
+
+    if (this.query)
+      tools.push({
+        type: "function",
+        process: async (q) => {
+          console.log("Table search", q);
+
+          let rows = [];
+          if (q.query?.full_text_search || q.full_text_search) {
+            const scState = getState();
+            const language = scState.pg_ts_config;
+            const use_websearch = scState.getConfig(
+              "search_use_websearch",
+              false
+            );
+            rows = await table.getRows({
+              _fts: {
+                fields: table.fields,
+                searchTerm: q.query?.full_text_search || q.full_text_search,
+                language,
+                use_websearch,
+                table: table.name,
+                schema: db.isSQLite ? undefined : db.getTenantSchema(),
+              },
+            });
+          } else {
+            rows = await table.getRows(q.query || q);
+          }
+          if (this.hidden_fields) {
+            const hidden_fields = this.hidden_fields
+              .split(",")
+              .map((s) => s.trim());
+            rows.forEach((r) => {
+              hidden_fields.forEach((k) => {
+                delete r[k];
+              });
+            });
+          }
+
+          if (rows.length) return { rows };
+          else
+            return {
+              response: "No rows found",
+            };
+        },
+        /*renderToolCall({ phrase }, { req }) {
+        return div({ class: "border border-primary p-2 m-2" }, phrase);
+      },*/
+        renderToolResponse: async ({ response, rows }, { req }) => {
+          if (rows) {
+            const view = View.findOne({ name: this.list_view });
+
+            if (view) {
+              const viewRes = await view.run(
+                { [table.pk_name]: { in: rows.map((r) => r[table.pk_name]) } },
+                { req }
+              );
+              return viewRes;
+            } else return "";
+          }
+          return div({ class: "border border-success p-2 m-2" }, response);
+        },
+        function: {
+          name: `query_${this.table_name}`,
+          description: `Search the ${this.table_name} database table${
+            table.description ? ` (${table.description})` : ""
+          } by a search phrase matched against all fields in the table with full text search or a more specific query. The retrieved rows will be returned`,
+          parameters: {
+            type: "object",
+            properties: {
+              query: {
+                anyOf: [
+                  {
+                    type: "object",
+                    description:
+                      "Search the table by a phrase matched against any string field",
+                    properties: {
+                      full_text_search: {
+                        type: "string",
+                        description: "A phrase to search the table with",
+                      },
+                    },
+                  },
+                  {
+                    type: "object",
+                    description: "Search the table by individual fields",
+                    properties: queryProperties,
+                  },
+                ],
+              },
+            },
+          },
+        },
+      });
+
+    if (this.insert)
+      tools.push({
+        type: "function",
+        process: async (rows, { req }) => {
+          const ids = [];
+          if (Array.isArray(rows)) {
+            for (const row of rows)
+              ids.push(await table.insertRow(row, req?.user));
+          } else ids.push(await table.insertRow(rows, req?.user));
+          return { created_ids: ids };
+        },
+        /*renderToolCall({ phrase }, { req }) {
+        return div({ class: "border border-primary p-2 m-2" }, phrase);
+      },*/
+        renderToolResponse: async ({ created_ids }, { req }) => {
+          if (created_ids) {
+            const view = View.findOne({ name: this.list_view });
+
+            if (view) {
+              const viewRes = await view.run(
+                { [table.pk_name]: { in: created_ids } },
+                { req }
+              );
+              return viewRes;
+            } else return "";
+          }
+        },
+        function: {
+          name: `insert_${this.table_name}`,
+          description: `Insert rows into the  ${
+            this.table_name
+          } database table${
+            table.description ? ` (${table.description})` : ""
+          }`,
+          parameters: {
+            type: "object",
+            required,
+            properties: queryProperties,
+          },
+        },
+      });
+    return tools;
+  }
+}
+
+module.exports = TableToSkill;

package/skills/Trigger.js

@@ -0,0 +1,101 @@
+const { div, pre, a } = require("@saltcorn/markup/tags");
+const Workflow = require("@saltcorn/data/models/workflow");
+const Form = require("@saltcorn/data/models/form");
+const Table = require("@saltcorn/data/models/table");
+const View = require("@saltcorn/data/models/view");
+const Trigger = require("@saltcorn/data/models/trigger");
+const { getState } = require("@saltcorn/data/db/state");
+const db = require("@saltcorn/data/db");
+const { fieldProperties } = require("./helpers");
+
+class TriggerToSkill {
+  static skill_name = "Trigger";
+
+  get skill_label() {
+    return this.trigger_name;
+  }
+
+  constructor(cfg) {
+    Object.assign(this, cfg);
+  }
+
+  systemPrompt() {
+    const trigger = Trigger.findOne({ name: this.trigger_name });
+
+    return `${this.trigger_name} tool: ${trigger.description}`;
+  }
+
+  static async configFields() {
+    const actions = (await Trigger.find({})).filter(
+      (action) => action.description
+    );
+    const hasTable = actions.filter((a) => a.table_id).map((a) => a.name);
+    const confirm_view_opts = {};
+    for (const a of actions) {
+      if (!a.table_id) continue;
+      const views = await View.find({ table_id: a.table_id });
+      confirm_view_opts[a.name] = views.map((v) => v.name);
+    }
+    return [
+      {
+        name: "trigger_name",
+        label: "Action",
+        sublabel:
+          "Only actions with a description can be enabled. " +
+          a(
+            {
+              "data-dyn-href": `\`/actions/configure/\${trigger_name}\``,
+              target: "_blank",
+            },
+            "Configure"
+          ),
+        type: "String",
+        required: true,
+        attributes: { options: actions.map((a) => a.name) },
+      },
+      // TODO: confirm, show response, show argument
+    ];
+  }
+
+  provideTools = () => {
+    let properties = {};
+
+    const trigger = Trigger.findOne({ name: this.trigger_name });
+    if (trigger.table_id) {
+      const table = Table.findOne({ id: trigger.table_id });
+
+      table.fields
+        .filter((f) => !f.primary_key)
+        .forEach((field) => {
+          properties[field.name] = {
+            description: field.label + " " + field.description || "",
+            ...fieldProperties(field),
+          };
+        });
+    }
+    return {
+      type: "function",
+      process: async (row, { req }) => {
+        const result = await trigger.runWithoutRow({ user: req?.user, row });
+        return result;
+      },
+      /*renderToolCall({ phrase }, { req }) {
+        return div({ class: "border border-primary p-2 m-2" }, phrase);
+      },*/
+      renderToolResponse: async (response, { req }) => {
+        return div({ class: "border border-success p-2 m-2" }, response);
+      },
+      function: {
+        name: trigger.name,
+        description: trigger.description,
+        parameters: {
+          type: "object",
+          //required: ["action_javascript_code", "action_name"],
+          properties,
+        },
+      },
+    };
+  };
+}
+
+module.exports = TriggerToSkill;

package/skills/helpers.js

@@ -0,0 +1,54 @@
+const toArrayOfStrings = (opts) => {
+  if (typeof opts === "string") return opts.split(",").map((s) => s.trim());
+  if (Array.isArray(opts))
+    return opts.map((o) => (typeof o === "string" ? o : o.value || o.name));
+};
+
+const fieldProperties = (field) => {
+  const props = {};
+  const typeName = field.type?.name || field.type || field.input_type;
+  if (field.isRepeat) {
+    props.type = "array";
+    const properties = {};
+    field.fields.map((f) => {
+      properties[f.name] = {
+        description: f.sublabel || f.label,
+        ...fieldProperties(f),
+      };
+    });
+    props.items = {
+      type: "object",
+      properties,
+    };
+  }
+  switch (typeName) {
+    case "String":
+      props.type = "string";
+      if (field.attributes?.options)
+        props.enum = toArrayOfStrings(field.attributes.options);
+      break;
+    case "Bool":
+      props.type = "boolean";
+      break;
+    case "Integer":
+      props.type = "integer";
+      break;
+    case "Float":
+      props.type = "number";
+      break;
+    case "select":
+      props.type = "string";
+      if (field.options) props.enum = toArrayOfStrings(field.options);
+      break;
+  }
+  if (!props.type) {
+    switch (field.input_type) {
+      case "code":
+        props.type = "string";
+        break;
+    }
+  }
+  return props;
+};
+
+module.exports = { fieldProperties };