@saltcorn/copilot 0.4.2 → 0.4.4

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@saltcorn/copilot",
-  "version": "0.4.2",
+  "version": "0.4.4",
   "description": "AI assistant for building Saltcorn applications",
   "main": "index.js",
   "dependencies": {

package/user-copilot.js

@@ -107,6 +107,13 @@ const configuration_workflow = (req) =>
                       calcOptions: ["table_name", list_view_opts],
                     },
                   },
+                  {
+                    name: "exclude_fields",
+                    label: "Exclude fields",
+                    sublabel:
+                      "Exclude fields from the chat context. Comma-separated list.",
+                    type: "String",
+                  },
                 ],
               }),
             ],
@@ -189,6 +196,17 @@ const run = async (table_id, viewname, config, state, { res, req }) => {
         case "assistant":
         case "system":
           if (interact.tool_calls) {
+            if (interact.content) {
+              interactMarkups.push(
+                div(
+                  { class: "interaction-segment" },
+                  span({ class: "badge bg-secondary" }, "Copilot"),
+                  typeof interact.content === "string"
+                    ? md.render(interact.content)
+                    : interact.content
+                )
+              );
+            }
             for (const tool_call of interact.tool_calls) {
               const action = config.actions.find(
                 (a) => a.trigger_name === tool_call.function.name
@@ -205,17 +223,14 @@ const run = async (table_id, viewname, config, state, { res, req }) => {
                       "Copilot"
                     )
                   );
-              } else if (tool_call.function.name.startsWith("Query")) {
+              } else if (tool_call.function.name === "TableQuery") {
                 const query = JSON.parse(tool_call.function.arguments);
                 const queryText = query.sql_id_query
                   ? query.sql_id_query
                   : JSON.stringify(query, null, 2);
                 interactMarkups.push(
                   wrapSegment(
-                    wrapCard(
-                      "Query " + tool_call.function.name.replace("Query", ""),
-                      pre(queryText)
-                    ),
+                    wrapCard("Query " + query.table_name, pre(queryText)),
                     "Copilot"
                   )
                 );
@@ -233,25 +248,40 @@ const run = async (table_id, viewname, config, state, { res, req }) => {
             );
           break;
         case "tool":
-          if (interact.name.startsWith("Query")) {
-            const table = Table.findOne({
-              name: interact.name.replace("Query", ""),
-            });
-            interactMarkups.push(
-              await renderQueryInteraction(
-                table,
-                JSON.parse(interact.content),
-                config,
-                req
+          if (interact.name === "TableQuery") {
+            const tool_call = run.context.interactions
+              .map(
+                (i) =>
+                  i.tool_calls &&
+                  i.tool_calls.find((tc) => tc.id === interact.tool_call_id)
               )
-            );
+              .filter(Boolean)[0];
+            if (tool_call) {
+              const args = JSON.parse(tool_call.function.arguments);
+              const table = Table.findOne(args.table_name);
+              interactMarkups.push(
+                await renderQueryInteraction(
+                  table,
+                  JSON.parse(interact.content),
+                  config,
+                  req
+                )
+              );
+            }
           } else if (interact.content !== "Action run") {
+            let markupContent;
+            try {
+              markupContent = JSON.stringify(
+                JSON.parse(interact.content),
+                null,
+                2
+              );
+            } catch {
+              markupContent = interact.content;
+            }
             interactMarkups.push(
               wrapSegment(
-                wrapCard(
-                  interact.name,
-                  pre(JSON.stringify(JSON.parse(interact.content), null, 2))
-                ),
+                wrapCard(interact.name, pre(markupContent)),
                 "Copilot"
               )
             );
@@ -436,84 +466,40 @@ const run = async (table_id, viewname, config, state, { res, req }) => {
 const getCompletionArguments = async (config) => {
   let tools = [];
   const sysPrompts = [];
-  for (const tableCfg of config?.tables || []) {
-    let properties = {};
+  let properties = {};
 
-    const table = Table.findOne({ name: tableCfg.table_name });
+  const tableNames = (config?.tables || []).map((t) => t.table_name);
+  properties.table_name = {
+    type: "string",
+    enum: tableNames,
+    description: `Which table is this query from. Every query has to select rows from one table, even if it is based on joins from different tables`,
+  };
+  properties.sql_id_query = {
+    type: "string",
+    description: `An SQL query for this table's primary keys. This must select only the primary keys (even if the user wants a count), for example SELECT ${
+      tableNames[0][0]
+    }."${Table.findOne(tableNames[0]).pk_name}" from "${tableNames[0]}" ${
+      tableNames[0][0]
+    } JOIN ... where... Use this to join other tables in the database.`,
+  };
+  properties.is_count = {
+    type: "boolean",
+    description: `Is the only desired output a count? Make this true if the user wants a count of rows`,
+  };
 
-    table.fields
-      .filter((f) => !f.primary_key)
-      .forEach((field) => {
-        properties[field.name] = {
-          description: field.label + " " + field.description || "",
-        };
-        if (field.type.name === "String") {
-          properties[field.name].anyOf = [
-            { type: "string", description: "Match this value exactly" },
-            {
-              type: "object",
-              properties: {
-                ilike: {
-                  type: "string",
-                  description: "Case insensitive substring match.",
-                },
-                in: {
-                  type: "array",
-                  description: "Match any one of these values",
-                  items: {
-                    type: "string",
-                  },
-                },
-              },
-            },
-          ];
-        }
-        if (field.type.name === "Integer" || field.type.name === "Float") {
-          const basetype = field.type.name === "Integer" ? "integer" : "number";
-          properties[field.name].anyOf = [
-            { type: basetype, description: "Match this value exactly" },
-            {
-              type: "object",
-              properties: {
-                gt: {
-                  type: basetype,
-                  description: "Greater than this value",
-                },
-                lt: {
-                  type: basetype,
-                  description: "Less than this value",
-                },
-                in: {
-                  type: "array",
-                  description: "Match any one of these values",
-                  items: {
-                    type: basetype,
-                  },
-                },
-              },
-            },
-          ];
-        } else Object.assign(fieldProperties(field), properties[field.name]);
-      });
-    properties.sql_id_query = {
-      type: "string",
-      description: `An SQL query for this table's primary keys (${table.pk_name}). This must select only the primary keys, for example SELECT ${table.name[0]}."${table.pk_name}" from "${table.name}" ${table.name[0]} JOIN ... where... Use this to join other tables in the database. If you use sql_id_query, use it alone, do not combine with other field queries. Any other fields you want to filter on must be accessed in the WHERE clause`,
-    };
-    tools.push({
-      type: "function",
-      function: {
-        name: "Query" + table.name,
-        description: `Query the ${table.name} table. ${
-          table.description || ""
-        }`,
-        parameters: {
-          type: "object",
-          //required: ["action_javascript_code", "action_name"],
-          properties,
-        },
+  tools.push({
+    type: "function",
+    function: {
+      name: "TableQuery",
+      description: `Query a table and show the results to the user in a grid format`,
+      parameters: {
+        type: "object",
+        required: ["table_name", "sql_id_query", "is_count"],
+        properties,
       },
-    });
-  }
+    },
+  });
+
   for (const action of config?.actions || []) {
     let properties = {};
 
@@ -574,7 +560,14 @@ ${t.fields
   .join(",\n")}
 )`
       )
-      .join(";\n\n");
+      .join(";\n\n") +
+    `
+      
+Use the TableQuery tool if the user asks to see, find or count or otherwise access rows from a table that matches what the user is looking for, or if
+the user is asking for a summary or inference from such rows. The TableQuery query is parametrised by a SQL SELECT query which 
+selects primary key values from the specified table. You can join other tables or use complex logic in the WHERE clause, but you must 
+always return porimary key values from the specified table.
+`;
   //console.log("sysprompt", systemPrompt);
 
   if (tools.length === 0) tools = undefined;
@@ -611,7 +604,24 @@ const interact = async (table_id, viewname, config, body, { req, res }) => {
 };
 
 const renderQueryInteraction = async (table, result, config, req) => {
-  if (result.length === 0) return "No rows found";
+  if (typeof result === "number")
+    return wrapSegment(
+      wrapCard(
+        "Query " + table?.name || "",
+        //div("Query: ", code(JSON.stringify(query))),
+        `${result}`
+      ),
+      "Copilot"
+    );
+  if (result.length === 0)
+    return wrapSegment(
+      wrapCard(
+        "Query " + table?.name || "",
+        //div("Query: ", code(JSON.stringify(query))),
+        "No rows found"
+      ),
+      "Copilot"
+    );
 
   const tableCfg = config.tables.find((t) => t.table_name === table.name);
   let viewRes = "";
@@ -652,6 +662,7 @@ const process_interaction = async (
 ) => {
   const complArgs = await getCompletionArguments(config);
   complArgs.chat = run.context.interactions;
+  //complArgs.debugResult = true;
   //console.log(complArgs);
   console.log("complArgs", JSON.stringify(complArgs, null, 2));
 
@@ -660,12 +671,20 @@ const process_interaction = async (
   await addToContext(run, {
     interactions:
       typeof answer === "object" && answer.tool_calls
-        ? [{ role: "assistant", tool_calls: answer.tool_calls }]
+        ? [
+            {
+              role: "assistant",
+              tool_calls: answer.tool_calls,
+              content: answer.content,
+            },
+          ]
         : [{ role: "assistant", content: answer }],
   });
   const responses = [];
 
   if (typeof answer === "object" && answer.tool_calls) {
+    if (answer.content)
+      responses.push(wrapSegment(md.render(answer.content), "Copilot"));
     //const actions = [];
     let hasResult = false;
     for (const tool_call of answer.tool_calls) {
@@ -724,51 +743,66 @@ const process_interaction = async (
             },
           ],
         });
-      } else if (tool_call.function.name.startsWith("Query")) {
+      } else if (tool_call.function.name == "TableQuery") {
+        const query = JSON.parse(tool_call.function.arguments);
         const table = Table.findOne({
-          name: tool_call.function.name.replace("Query", ""),
+          name: query.table_name,
         });
-        const query = JSON.parse(tool_call.function.arguments);
-        let result;
-        if (query.sql_id_query) {
-          const is_sqlite = db.isSQLite;
-
-          const client = is_sqlite ? db : await db.getClient();
-          await client.query(`BEGIN;`);
-          if (!is_sqlite) {
-            await client.query(
-              `SET LOCAL search_path TO "${db.getTenantSchema()}";`
-            );
-            await client.query(
-              `SET SESSION CHARACTERISTICS AS TRANSACTION READ ONLY;`
-            );
-          }
+        const tableCfg = config.tables.find((t) => t.table_name === table.name);
 
-          const { rows } = await client.query(query.sql_id_query);
-          await client.query(`ROLLBACK;`);
+        const is_sqlite = db.isSQLite;
 
-          if (!is_sqlite) client.release(true);
-          result = await table.getRows(
-            { [table.pk_name]: { in: rows.map((r) => r[table.pk_name]) } },
-            {
-              forUser: req.user,
-              forPublic: !req.user,
-            }
+        const client = is_sqlite ? db : await db.getClient();
+        await client.query(`BEGIN;`);
+        if (!is_sqlite) {
+          await client.query(
+            `SET LOCAL search_path TO "${db.getTenantSchema()}";`
           );
-          responses.push(
-            wrapSegment(
-              wrapCard(
-                "Query " + tool_call.function.name.replace("Query", ""),
-                pre(query.sql_id_query)
-              ),
-              "Copilot"
-            )
+          await client.query(
+            `SET SESSION CHARACTERISTICS AS TRANSACTION READ ONLY;`
           );
-        } else
-          result = await table.getRows(query, {
+        }
+
+        const { rows } = await client.query(query.sql_id_query);
+        await client.query(`ROLLBACK;`);
+
+        if (!is_sqlite) client.release(true);
+        let result;
+        const id_query = {
+          [table.pk_name]: { in: rows.map((r) => r[table.pk_name]) },
+        };
+
+        if (query.is_count) {
+          const role = req.user?.role_id || 100;
+          if (role <= table.min_role_read) {
+            result = await table.countRows(id_query);
+          } else result = "Not authorized";
+        } else {
+          result = await table.getRows(id_query, {
             forUser: req.user,
             forPublic: !req.user,
           });
+          if (tableCfg.exclude_fields) {
+            const fields = tableCfg.exclude_fields
+              .split(",")
+              .map((s) => s.trim());
+            fields.forEach((f) => {
+              result.forEach((r) => {
+                delete r[f];
+              });
+            });
+          }
+        }
+        responses.push(
+          wrapSegment(
+            wrapCard(
+              "Query " + tool_call.function.name.replace("Query", ""),
+              pre(query.sql_id_query)
+            ),
+            "Copilot"
+          )
+        );
+
         await addToContext(run, {
           interactions: [
             {