@saltcorn/agents 0.6.2 → 0.6.4

package/agent-view.js

@@ -31,6 +31,7 @@ const {
   label,
   a,
   br,
+  img,
 } = require("@saltcorn/markup/tags");
 const { getState } = require("@saltcorn/data/db/state");
 const {
@@ -45,9 +46,12 @@ const {
   is_debug_mode,
   get_initial_interactions,
   get_skill_instances,
+  saveInteractions,
 } = require("./common");
 const MarkdownIt = require("markdown-it"),
   md = new MarkdownIt();
+const { isWeb } = require("@saltcorn/data/utils");
+const path = require("path");
 
 const configuration_workflow = (req) =>
   new Workflow({
@@ -169,6 +173,7 @@ const uploadForm = (viewname, req) =>
       type: "file",
       class: "d-none",
       accept: "image/*",
+      multiple: true,
       onchange: `agent_file_attach(event)`,
     }),
     span({ class: "ms-2 filename-label" }),
@@ -270,6 +275,7 @@ const run = async (
       interactMarkups.push(...run.context.html_interactions);
     } else
       for (const interact of run.context.interactions) {
+        //legacy
         switch (interact.role) {
           case "user":
             if (interact.content?.[0]?.type === "image_url") {
@@ -484,7 +490,7 @@ const run = async (
     ? div(
         div(
           {
-            class: "d-flex justify-content-between align-middle mb-2",
+            class: "d-flex flex-wrap justify-content-between align-middle mb-2",
           },
           div(
             { class: "d-flex" },
@@ -497,7 +503,7 @@ const run = async (
           button(
             {
               type: "button",
-              class: "btn btn-secondary btn-sm py-0",
+              class: "btn btn-secondary btn-sm pt-0 pb-1",
               style: "font-size: 0.9em;height:1.5em",
               onclick: "unset_state_field('run_id')",
               title: "New session",
@@ -513,7 +519,10 @@ const run = async (
             },
             div(
               { class: "d-flex justify-content-between" },
-              localeDateTime(run.started_at),
+              span(
+                { class: "text-truncate", style: "min-width:0" },
+                localeDateTime(run.started_at),
+              ),
               i({
                 class: "far fa-trash-alt",
                 onclick: `delprevrun(event, ${run.id})`,
@@ -549,7 +558,15 @@ const run = async (
       `div.interaction-segment:not(:first-child) {border-top: 1px solid #e7e7e7; }
               div.interaction-segment {padding-top: 5px;padding-bottom: 5px;}
               div.interaction-segment p {margin-bottom: 0px;}
-              div.interaction-segment div.card {margin-top: 0.5rem;}            
+              div.interaction-segment div.card {margin-top: 0.5rem;}
+              div.interaction-segment.to-right {
+              display: flex;             
+    flex-direction: row-reverse;
+}
+    div.interaction-segment.to-right div.badgewrap {
+              display: flex;             
+    flex-direction: row-reverse;
+}
             div.prevcopilotrun:hover {cursor: pointer; background-color: var(--tblr-secondary-bg-subtle, var(--bs-secondary-bg-subtle, gray));}
             div.prevcopilotrun i.fa-trash-alt {display: none;}
             div.prevcopilotrun:hover i.fa-trash-alt {display: block;}
@@ -584,9 +601,15 @@ const run = async (
               cursor: pointer;
               }
               .copilot-entry span.attach_agent_image_wrap {
-              position: relative; 
+              position: relative;
               top: -1.8rem;
-              left: 0.2rem;              
+              left: 0.2rem;
+            }
+              .copilot-entry.dragover {
+              outline: 2px dashed var(--tblr-primary, #0054a6);
+              outline-offset: -2px;
+              background: var(--tblr-primary-bg-subtle, rgba(0, 84, 166, 0.05));
+              border-radius: 0.25rem;
             }
               .copilot-entry .explainer {
               position: relative; 
@@ -618,19 +641,25 @@ const run = async (
     function get_run_id(elem) {
         return $("input[name=run_id").val()
     }
-    function processCopilotResponse(res) {        
+    function processCopilotResponse(res, not_final) {        
         console.log("processCopilotResponse", res)
-        const hadFile = $("input#attach_agent_image").val();
-        let fileBadge = hadFile ? '<span class="badge text-bg-info"><i class="fas fa-image me-1"></i>'+$("input#attach_agent_image")[0].files?.item?.(0)?.name||"File"+'</span>': ""
-        $("span.filename-label").text("");
+        const fileInput = $("input#attach_agent_image")[0];
+        let fileBadge = "";
+        if (fileInput?.files?.length) {
+          fileBadge = Array.from(fileInput.files).map(f =>
+            '<span class="badge text-bg-info"><i class="fas fa-image me-1"></i>'+f.name+'</span>'
+          ).join(" ");
+        }
+        $("span.filename-label").text("").removeClass("me-2");
+        _agentDT.items.clear();
         $("input#attach_agent_image").val(null);
-        $("#sendbuttonicon").attr("class","far fa-paper-plane");
+        if(!not_final || (!${JSON.stringify(dyn_updates)})) $("#sendbuttonicon").attr("class","far fa-paper-plane");
         const $runidin= $("input[name=run_id")
         if(res.run_id && (!$runidin.val() || $runidin.val()=="undefined"))
           $runidin.val(res.run_id);
         const wrapSegment = (html, who) => '<div class="interaction-segment"><span class="badge bg-secondary">'+who+'</span>'+html+'</div>'
         const user_input = $("textarea[name=userinput]").val()
-        if(user_input)
+        if(user_input && (!${JSON.stringify(dyn_updates)}))
           $("#copilotinteractions").append(wrapSegment('<p>'+user_input+'</p>'+fileBadge, "You"))
         $("textarea[name=userinput]").val("")
         $('form.agent-view div.next_response_scratch').html("")
@@ -639,8 +668,39 @@ const run = async (
             $("#copilotinteractions").append(res.response)
     }
     window.processCopilotResponse = processCopilotResponse;
+    window.final_agent_response = () => {
+      $("#sendbuttonicon").attr("class","far fa-paper-plane");
+    }
+    const _agentDT = new DataTransfer();
+    function setAgentFiles(files) {
+        for (const f of files) _agentDT.items.add(f);
+        document.getElementById('attach_agent_image').files = _agentDT.files;
+        updateFileLabel();
+    }
+    function updateFileLabel() {
+        const n = _agentDT.files.length;
+        const $label = $(".attach_agent_image_wrap span.filename-label");
+        if (n === 0) {
+          $label.html("").removeClass("me-2");
+        } else {
+          $label.addClass("me-2");
+          const text = n === 1 ? _agentDT.files[0].name : n + " files";
+          $label.html(${
+            isWeb(req)
+              ? `text + ' <span class="badge text-bg-secondary" style="cursor:pointer;font-size:.65em;vertical-align:middle" onclick="clearAgentFiles()" title="Remove files">&times;</span>'`
+              : `'<span style="max-width:8em;overflow:hidden;text-overflow:ellipsis;white-space:nowrap;display:inline-block;vertical-align:middle">' + text + '</span> <span class="badge text-bg-secondary" style="cursor:pointer;font-size:.65em;vertical-align:middle" onclick="clearAgentFiles()" title="Remove files">&times;</span>'`
+          });
+        }
+    }
+    function clearAgentFiles() {
+        _agentDT.items.clear();
+        $("input#attach_agent_image").val(null);
+        updateFileLabel();
+    }
+    window.clearAgentFiles = clearAgentFiles;
     function agent_file_attach(e) {
-        $(".attach_agent_image_wrap span.filename-label").text(e.target.files[0].name)
+        _agentDT.items.clear();
+        setAgentFiles(e.target.files);
     }
     function restore_old_button_elem(btn) {
         const oldText = $(btn).data("old-text");
@@ -692,6 +752,44 @@ const run = async (
         }        
     }
     document.getElementById("inputuserinput").addEventListener("keydown", submitOnEnter);
+    if (document.getElementById('attach_agent_image')) {
+      let _dragCtr = 0;
+      const _copilotEntry = document.querySelector('.copilot-entry');
+      _copilotEntry.addEventListener('dragover', function(e) {
+        e.preventDefault();
+      });
+      _copilotEntry.addEventListener('dragenter', function(e) {
+        e.preventDefault();
+        _dragCtr++;
+        this.classList.add('dragover');
+      });
+      _copilotEntry.addEventListener('dragleave', function(e) {
+        _dragCtr--;
+        if (_dragCtr === 0) this.classList.remove('dragover');
+      });
+      _copilotEntry.addEventListener('drop', function(e) {
+        e.preventDefault();
+        _dragCtr = 0;
+        this.classList.remove('dragover');
+        const imgs = Array.from(e.dataTransfer.files).filter(f => f.type.startsWith('image/'));
+        if (imgs.length) setAgentFiles(imgs);
+      });
+      document.getElementById('inputuserinput').addEventListener('paste', function(e) {
+        const items = e.clipboardData?.items;
+        if (!items) return;
+        const pastedFiles = [];
+        for (const item of items) {
+          if (item.type.startsWith('image/')) {
+            const file = item.getAsFile();
+            if (file) pastedFiles.push(file);
+          }
+        }
+        if (pastedFiles.length) {
+          e.preventDefault();
+          setAgentFiles(pastedFiles);
+        }
+      });
+    }
     function spin_send_button() {
       $("#sendbuttonicon").attr("class","fas fa-spinner fa-spin");
     };`,
@@ -753,44 +851,66 @@ const interact = async (table_id, viewname, config, body, { req, res }) => {
   }
   let fileBadges = "";
   if (config.image_upload && req.files?.file) {
-    const file = await File.from_req_files(
-      req.files.file,
-      req.user ? req.user.id : null,
-      100,
-      // file_field?.attributes?.folder
-    );
-    fileBadges = span(
-      { class: "badge text-bg-info" },
-      i({ class: "fas fa-image me-1" }),
-      file.filename,
-    );
-    const baseUrl = getState().getConfig("base_url").replace(/\/$/, "");
-    let imageurl;
-    if (
-      !config.image_base64 &&
-      baseUrl &&
-      !baseUrl.includes("http://localhost:")
-    ) {
-      imageurl = `${baseUrl}/files/serve/${file.path_to_serve}`;
-    } else {
-      const b64 = await file.get_contents("base64");
-      imageurl = `data:${file.mimetype};base64,${b64}`;
+    const rawFiles = Array.isArray(req.files.file)
+      ? req.files.file
+      : [req.files.file];
+    const badges = [];
+    for (const rawFile of rawFiles) {
+      const file = await File.from_req_files(
+        rawFile,
+        req.user ? req.user.id : null,
+        100,
+      );
+      badges.push(
+        div(
+          { class: "bg-secondary-subtle p-2 m-2 rounded-2" },
+          img({
+            src: `/files/resize/${50}/${50}/${file.path_to_serve}`,
+            class: "d-block",
+            onclick: `expand_thumbnail('${file.path_to_serve}', '${path.basename(file.path_to_serve)}')`,
+          }),
+          file.filename,
+        ),
+      );
+      const baseUrl = getState().getConfig("base_url").replace(/\/$/, "");
+      let imageurl;
+      if (
+        !config.image_base64 &&
+        baseUrl &&
+        !baseUrl.includes("http://localhost:")
+      ) {
+        imageurl = `${baseUrl}/files/serve/${file.path_to_serve}`;
+      } else {
+        const b64 = await file.get_contents("base64");
+        imageurl = `data:${file.mimetype};base64,${b64}`;
+      }
+      await getState().functions.llm_add_message.run("image", imageurl, {
+        chat: run.context.interactions || [],
+      });
     }
-    await getState().functions.llm_add_message.run("image", imageurl, {
-      chat: run.context.interactions || [],
-    });
-    await addToContext(run, {
-      interactions: run.context.interactions || [],
-    });
+    await saveInteractions(run);
+    fileBadges = div({ class: "d-flex" }, badges);
   }
+  const userInteractions = wrapSegment(p(userinput) + fileBadges, "You", true);
+
   await addToContext(run, {
     interactions: [
       ...(run.context.interactions || []),
       { role: "user", content: userinput },
     ],
-    html_interactions: [wrapSegment(p(userinput) + fileBadges, "You")],
+    html_interactions: [userInteractions],
   });
   const dyn_updates = getState().getConfig("enable_dynamic_updates", true);
+  if (dyn_updates) {
+    getState().emitDynamicUpdate(
+      db.getTenantSchema(),
+      {
+        eval_js: `processCopilotResponse({response: ${JSON.stringify(userInteractions)}, run_id: ${run.id}}, true)`,
+        page_load_tag: req?.headers?.["page-load-tag"],
+      },
+      [req.user.id],
+    );
+  }
   const process_promise = process_interaction(
     run,
     action.configuration,
@@ -945,4 +1065,5 @@ module.exports = {
   table_optional: true,
   run,
   routes: { interact, delprevrun, debug_info, skillroute, execute_user_action },
+  mobile_render_server_side: true,
 };

package/common.js

@@ -166,14 +166,20 @@ const addToContext = async (run, newCtx) => {
   if (changed && run.update) await run.update({ context: run.context });
 };
 
-const wrapSegment = (html, who) =>
+const saveInteractions = async (run) => {
+  await addToContext(run, {
+    interactions: run.context.interactions || [],
+  });
+};
+
+const wrapSegment = (html, who, to_right) =>
   who === null
     ? html
-    : '<div class="interaction-segment"><span class="badge bg-secondary">' +
+    : `<div class="interaction-segment ${to_right ? "to-right" : ""}"><div><div class="badgewrap"><span class="badge bg-secondary">` +
       who +
-      "</span>" +
+      "</span></div>" +
       html +
-      "</div>";
+      "</div></div>";
 
 const wrapCard = (title, ...inners) =>
   span({ class: "badge bg-info ms-1" }, title) +
@@ -236,12 +242,13 @@ const process_interaction = async (
     interactions: complArgs.chat,
   });
   const responses = [];
-  const add_response = async (resp) => {
+
+  const add_response = async (resp, not_final) => {
     if (dyn_updates)
       getState().emitDynamicUpdate(
         db.getTenantSchema(),
         {
-          eval_js: `processCopilotResponse({response: ${JSON.stringify(resp)}, run_id: ${run.id}})`,
+          eval_js: `processCopilotResponse({response: ${JSON.stringify(resp)}, run_id: ${run.id}}, true)`,
           page_load_tag: req?.headers?.["page-load-tag"],
         },
         [req.user.id],
@@ -479,6 +486,15 @@ const process_interaction = async (
         ? answer
         : wrapSegment(md.render(answer), agent_label),
     );
+  if (dyn_updates)
+    getState().emitDynamicUpdate(
+      db.getTenantSchema(),
+      {
+        eval_js: `final_agent_response()`,
+        page_load_tag: req?.headers?.["page-load-tag"],
+      },
+      [req.user.id],
+    );
 
   return {
     json: {
@@ -497,6 +513,7 @@ module.exports = {
   get_skill_instances,
   getCompletionArguments,
   addToContext,
+  saveInteractions,
   wrapCard,
   wrapSegment,
   process_interaction,

package/index.js

@@ -18,6 +18,7 @@ module.exports = {
   dependencies: ["@saltcorn/large-language-model"],
   viewtemplates: [require("./agent-view")],
   plugin_name: "agents",
+  ready_for_mobile: true,
   headers: [
     {
       script: `/plugins/public/agents@${

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@saltcorn/agents",
-  "version": "0.6.2",
+  "version": "0.6.4",
   "description": "AI agents for Saltcorn",
   "main": "index.js",
   "dependencies": {