@saltcorn/copilot 0.7.2 → 0.7.4

package/actions/generate-tables.js

@@ -5,7 +5,7 @@ const Table = require("@saltcorn/data/models/table");
 const Field = require("@saltcorn/data/models/field");
 const { apply, removeAllWhiteSpace } = require("@saltcorn/data/utils");
 const { getActionConfigFields } = require("@saltcorn/data/plugin-helper");
-const { a, pre, script, div } = require("@saltcorn/markup/tags");
+const { a, pre, script, div, domReady } = require("@saltcorn/markup/tags");
 const { fieldProperties } = require("../common");
 
 class GenerateTables {
@@ -143,6 +143,10 @@ class GenerateTables {
     need to specify a primary key, a primary key called id with autoincrementing integers is
     autmatically generated. 
 
+    Only include brand-new tables in the generate_tables arguments. If the user references a table
+    that already exists, explain that generate_tables can only add new tables (not modify existing
+    ones) and omit those tables from the tool call.
+
     The database already contains the following tables: 
 
     ${tableLines.join("\n\n")}
@@ -154,7 +158,14 @@ class GenerateTables {
     const mmdia = buildMermaidMarkup(sctables);
     return (
       pre({ class: "mermaid" }, mmdia) +
-      script(`mermaid.run({querySelector: 'pre.mermaid'});`)
+      script(
+        domReady(`
+        ensure_script_loaded("/static_assets/"+_sc_version_tag+"/mermaid.min.js", () => {
+          mermaid.initialize({ startOnLoad: false });
+          mermaid.run({ querySelector: ".mermaid" });
+        });
+      `),
+      ) 
     );
   }
 

package/actions/generate-workflow.js

@@ -3,24 +3,89 @@ const WorkflowStep = require("@saltcorn/data/models/workflow_step");
 const Trigger = require("@saltcorn/data/models/trigger");
 const Table = require("@saltcorn/data/models/table");
 const { getActionConfigFields } = require("@saltcorn/data/plugin-helper");
-const { a, pre, script, div } = require("@saltcorn/markup/tags");
+const { a, pre, script, div, domReady } = require("@saltcorn/markup/tags");
 const { fieldProperties } = require("../common");
 
+const table_triggers = ["Insert", "Update", "Delete", "Validate"];
+const additional_triggers_with_onlyif = ["Login", "PageLoad"];
+
+const optionNames = (options = []) =>
+  options
+    .map((opt) =>
+      typeof opt === "string"
+        ? opt
+        : opt?.name || opt?.label || opt?.value || "",
+    )
+    .filter(Boolean);
+
+const joinOptionNames = (options = [], limit = 10) => {
+  const names = optionNames(options);
+  if (!names.length) return "none";
+  const shown = names.slice(0, limit).join(", ");
+  return names.length > limit ? `${shown}, ...` : shown;
+};
+
+const summarizeTriggerActionMatrix = () => {
+  try {
+    const allActions = Trigger.action_options({
+      notRequireRow: false,
+      workflow: true,
+    });
+    const noRowActions = Trigger.action_options({
+      notRequireRow: true,
+      workflow: true,
+    });
+    const lines = Trigger.when_options.map((when) => {
+      const pool = table_triggers.includes(when) ? allActions : noRowActions;
+      return `* ${when}: ${joinOptionNames(pool)}`;
+    });
+    const notes = `Triggers requiring a table context: ${table_triggers.join(", ")}.
+Additional triggers that commonly use contextual only_if checks: ${additional_triggers_with_onlyif.join(", ")}.`;
+    return `${notes}
+${lines.join("\n")}`;
+  } catch (e) {
+    console.error("GenerateWorkflow: action matrix failed", e);
+    return "";
+  }
+};
+
+const summarizeTables = async () => {
+  try {
+    const tables = await Table.find({});
+    if (!tables?.length) return "";
+    const fieldType = (f = {}) =>
+      f.pretty_type || f.type?.name || f.type || f.input_type || "unknown";
+    const rows = tables.map((table) => {
+      const description = table.description ? ` – ${table.description}` : "";
+      const fields = (table.fields || [])
+        .slice(0, 8)
+        .map((f) => `${f.name}:${fieldType(f)}`)
+        .join(", ");
+      const fieldText = fields ? ` fields: ${fields}` : "";
+      return `* ${table.name}${description}${fieldText}`;
+    });
+    return rows.join("\n");
+  } catch (e) {
+    console.error("GenerateWorkflow: table summary failed", e);
+    return "";
+  }
+};
+
 const steps = async () => {
   const actionExplainers = WorkflowStep.builtInActionExplainers();
   const actionFields = await WorkflowStep.builtInActionConfigFields();
 
   let stateActions = getState().actions;
   const stateActionList = Object.entries(stateActions).filter(
-    ([k, v]) => !v.disableInWorkflow
+    ([k, v]) => !v.disableInWorkflow,
   );
 
   const stepTypeAndCfg = Object.keys(actionExplainers).map((actionName) => {
-    const properties = { 
-      step_type: { type: "string", enum: [actionName] }
+    const properties = {
+      step_type: { type: "string", enum: [actionName] },
     };
     const myFields = actionFields.filter(
-      (f) => f.showIf?.wf_action_name === actionName
+      (f) => f.showIf?.wf_action_name === actionName,
     );
     const required = ["step_type"];
     myFields.forEach((f) => {
@@ -39,8 +104,8 @@ const steps = async () => {
   });
   for (const [actionName, action] of stateActionList) {
     try {
-      const properties = { 
-        step_type: { type: "string", enum: [actionName] }
+      const properties = {
+        step_type: { type: "string", enum: [actionName] },
       };
       const cfgFields = await getActionConfigFields(action, null, {
         mode: "workflow",
@@ -73,9 +138,9 @@ const steps = async () => {
   //TODO workflows
   for (const trigger of triggers) {
     const properties = {
-      step_type: { 
+      step_type: {
         type: "string",
-        enum: [trigger.name],  
+        enum: [trigger.name],
       },
     };
     if (trigger.table_id) {
@@ -88,7 +153,7 @@ const steps = async () => {
       properties.row_expr = {
         type: "string",
         description: `JavaScript expression for the input to the action. This should be an expression for an object, with the following field name and types: ${fieldSpecs.join(
-          "; "
+          "; ",
         )}.`,
       };
     }
@@ -150,11 +215,10 @@ class GenerateWorkflow {
           description:
             "When the workflow should trigger. Optional, leave blank if unspecified or workflow will be run on button click",
           type: "string",
-          enum: ["Insert", "Delete", "Update", "Daily", "Hourly", "Weekly"],
+          enum: Trigger.when_options,
         },
         trigger_table: {
-          description:
-            "If the workflow trigger is Insert, Delete or Update, the name of the table that triggers the workflow",
+          description: `If the workflow trigger is ${table_triggers.join(", ")}, the name of the table that triggers the workflow`,
           type: "string",
         },
       },
@@ -165,15 +229,29 @@ class GenerateWorkflow {
     const actionExplainers = WorkflowStep.builtInActionExplainers();
     let stateActions = getState().actions;
     const stateActionList = Object.entries(stateActions).filter(
-      ([k, v]) => !v.disableInWorkflow
+      ([k, v]) => !v.disableInWorkflow,
     );
+    const [tableSummary, triggerMatrix] = await Promise.all([
+      summarizeTables(),
+      Promise.resolve(summarizeTriggerActionMatrix()),
+    ]);
+    const contextBlocks = [
+      tableSummary && `Current tables and key fields:\n${tableSummary}`,
+      triggerMatrix && `Workflow trigger compatibility:\n${triggerMatrix}`,
+    ]
+      .filter(Boolean)
+      .join("\n\n");
 
     return `Use the generate_workflow tool to construct computational workflows according to specifications. You must create 
   the workflow by calling the generate_workflow tool, with the step required to implement the specification.
   
+  ${contextBlocks}
+  
   The steps are specified as JSON objects. Each step has a name, specified in the step_name key in the JSON object. 
   The step name should be a valid JavaScript identifier.
   
+  When the user explicitly names an action/step type (for example "ForLoop" or "Toast"), ensure the generated workflow contains at least one step whose step_configuration.step_type exactly matches every requested action. Only skip this if the action genuinely does not exist; in that case, clearly explain the omission in the workflow description.
+
   Each run of the workflow is executed in the presence of a context, which is a JavaScript object that individual
   steps can read values from and write values to. This context is a state that is persisted on disk for each workflow 
   run. 
@@ -239,7 +317,7 @@ class GenerateWorkflow {
 
   static async execute(
     { workflow_steps, workflow_name, when_trigger, trigger_table },
-    req
+    req,
   ) {
     const steps = this.process_all_steps(workflow_steps);
     let table_id;
@@ -268,7 +346,7 @@ class GenerateWorkflow {
         "Workflow created. " +
         a(
           { target: "_blank", href: `/actions/configure/${trigger.id}` },
-          "Configure workflow."
+          "Configure workflow.",
         ),
     };
   }
@@ -286,21 +364,29 @@ class GenerateWorkflow {
         step.id = ix + 1;
       });
       const mmdia = WorkflowStep.generate_diagram(
-        steps.map((s) => new WorkflowStep(s))
+        steps.map((s) => new WorkflowStep(s)),
       );
+      console.log({ mmdia });
       return (
         div(
           `${workflow_name}${when_trigger ? `: ${when_trigger}` : ""}${
             trigger_table ? ` on ${trigger_table}` : ""
-          }`
+          }`,
         ) +
         pre({ class: "mermaid" }, mmdia) +
-        script(`mermaid.run({querySelector: 'pre.mermaid'});`)
+        script(
+          domReady(`
+          ensure_script_loaded("/static_assets/"+_sc_version_tag+"/mermaid.min.js", () => {
+            mermaid.initialize({ startOnLoad: false });
+            mermaid.run({ querySelector: ".mermaid" });
+          });
+        `),
+        ) 
       );
     }
 
     return `A workflow! Step names: ${workflow_steps.map(
-      (s) => s.step_name
+      (s) => s.step_name,
     )}. Upgrade Saltcorn to see diagrams in copilot`;
   }
 

package/agent-skills/database-design.js

@@ -1,4 +1,5 @@
 const GenerateTables = require("../actions/generate-tables");
+const Table = require("@saltcorn/data/models/table");
 
 const normalizeTablesPayload = (rawPayload) => {
   if (!rawPayload) return { tables: [] };
@@ -68,18 +69,73 @@ const collectTableWarnings = (tables) => {
 
       if (!field?.type_and_configuration?.data_type)
         warnings.push(
-          `${tableLabel}.${fieldLabel} must include type_and_configuration.data_type.`
+          `${tableLabel}.${fieldLabel} must include type_and_configuration.data_type.`,
         );
 
       if ((field?.name || "").toLowerCase() === "id")
         warnings.push(
-          `${tableLabel}.${fieldLabel} should be omitted because every table already has an auto-increment id.`
+          `${tableLabel}.${fieldLabel} should be omitted because every table already has an auto-increment id.`,
         );
     });
   });
   return warnings;
 };
 
+const fetchExistingTableNameSet = async () => {
+  const existingTables = await Table.find({});
+  const names = new Set();
+  existingTables.forEach((table) => {
+    if (table?.name) names.add(table.name.toLowerCase());
+  });
+  return names;
+};
+
+const partitionTablesByExistence = async (tables = []) => {
+  const existingNames = await fetchExistingTableNameSet();
+  const seenNewNames = new Set();
+  const newTables = [];
+  const skippedExisting = [];
+  const skippedDuplicates = [];
+  tables.forEach((table) => {
+    const tableName =
+      typeof table?.table_name === "string" ? table.table_name.trim() : "";
+    const normalized = tableName.toLowerCase();
+    if (tableName && existingNames.has(normalized)) {
+      skippedExisting.push(tableName);
+      return;
+    }
+    if (tableName && seenNewNames.has(normalized)) {
+      skippedDuplicates.push(tableName);
+      return;
+    }
+    if (tableName) seenNewNames.add(normalized);
+    newTables.push(table);
+  });
+  return { newTables, skippedExisting, skippedDuplicates };
+};
+
+const partitionTablesByValidity = (tables = []) => {
+  const validTables = [];
+  const skippedMissingNames = [];
+  const skippedMissingFields = [];
+  tables.forEach((table, idx) => {
+    const rawName =
+      typeof table?.table_name === "string" ? table.table_name.trim() : "";
+    const fallbackLabel = rawName || `Table #${idx + 1}`;
+    if (!rawName) {
+      skippedMissingNames.push(fallbackLabel);
+      return;
+    }
+    const fields = Array.isArray(table?.fields) ? table.fields : [];
+    if (!fields.length) {
+      skippedMissingFields.push(rawName);
+      return;
+    }
+    validTables.push({ ...table, table_name: rawName, fields });
+  });
+  return { validTables, skippedMissingNames, skippedMissingFields };
+};
+
 const payloadFromToolCall = (tool_call) => {
   if (!tool_call) return { tables: [] };
   if (tool_call.input) return normalizeTablesPayload(tool_call.input);
@@ -96,79 +152,187 @@ class GenerateTablesSkill {
   }
 
   constructor(cfg) {
-    console.log("GenerateTablesSkill.constructor called", { cfg });
     Object.assign(this, cfg);
   }
 
   async systemPrompt() {
-    console.log("GenerateTablesSkill.systemPrompt called");
     return await GenerateTables.system_prompt();
   }
 
   get userActions() {
-    console.log("GenerateTablesSkill.userActions getter accessed");
     return {
       async apply_copilot_tables({ user, tables }) {
-        console.log("GenerateTablesSkill.userActions.apply_copilot_tables called", {
-          user_id: user?.id,
-          table_count: tables?.length,
-        });
         if (!tables?.length) return { notify: "Nothing to create." };
-        await GenerateTables.execute({ tables }, { user });
+        const { newTables, skippedExisting, skippedDuplicates } =
+          await partitionTablesByExistence(tables);
+        const { validTables, skippedMissingNames, skippedMissingFields } =
+          partitionTablesByValidity(newTables);
+        if (!validTables.length) {
+          const skippedMessages = [];
+          if (skippedExisting.length)
+            skippedMessages.push(
+              `Existing tables: ${skippedExisting.join(", ")}`,
+            );
+          if (skippedDuplicates.length)
+            skippedMessages.push(
+              `Duplicate definitions: ${skippedDuplicates.join(", ")}`,
+            );
+          if (skippedMissingNames.length)
+            skippedMessages.push(
+              `Missing table_name: ${skippedMissingNames.join(", ")}`,
+            );
+          if (skippedMissingFields.length)
+            skippedMessages.push(
+              `Tables without fields: ${skippedMissingFields.join(", ")}`,
+            );
+          return {
+            notify:
+              skippedMessages.length > 0
+                ? `Nothing to create. Skipped ${skippedMessages.join("; ")}.`
+                : "Nothing to create.",
+          };
+        }
+        await GenerateTables.execute({ tables: validTables }, { user });
+        const createdNames = validTables.map((t) => t.table_name).join(", ");
+        const skippedMessages = [];
+        if (skippedExisting.length)
+          skippedMessages.push(
+            `Skipped existing tables: ${skippedExisting.join(", ")}`,
+          );
+        if (skippedDuplicates.length)
+          skippedMessages.push(
+            `Ignored duplicate definitions: ${skippedDuplicates.join(", ")}`,
+          );
+        if (skippedMissingNames.length)
+          skippedMessages.push(
+            `Missing table_name: ${skippedMissingNames.join(", ")}`,
+          );
+        if (skippedMissingFields.length)
+          skippedMessages.push(
+            `Tables without fields: ${skippedMissingFields.join(", ")}`,
+          );
         return {
-          notify: `Created tables: ${tables.map((t) => t.table_name).join(", ")}`,
+          notify: [`Created tables: ${createdNames}`, ...skippedMessages].join(
+            ". ",
+          ),
         };
       },
     };
   }
 
   provideTools = () => {
-    console.log("GenerateTablesSkill.provideTools called");
     const parameters = GenerateTables.json_schema();
     return {
       type: "function",
       process: async (input) => {
         const payload = normalizeTablesPayload(input);
         const tables = payload.tables || [];
-        console.log("GenerateTablesSkill.provideTools.process called", {
-          table_count: tables.length,
-        });
-        if (!tables.length) return "No tables were provided for generate_tables.";
-        const summaryLines = summarizeTables(tables).map((line) => `- ${line}`);
+        if (!tables.length) {
+          return "No tables were provided for generate_tables.";
+        }
+        const { newTables, skippedExisting, skippedDuplicates } =
+          await partitionTablesByExistence(tables);
+        const { validTables, skippedMissingNames, skippedMissingFields } =
+          partitionTablesByValidity(newTables);
+        const summaryLines = validTables.length
+          ? summarizeTables(validTables).map((line) => `- ${line}`)
+          : [];
         const warnings = collectTableWarnings(tables);
+        if (skippedExisting.length)
+          skippedExisting.forEach((name) =>
+            warnings.push(
+              `Table "${name}" already exists and will not be recreated by generate_tables.`,
+            ),
+          );
+        if (skippedDuplicates.length)
+          skippedDuplicates.forEach((name) =>
+            warnings.push(
+              `Table "${name}" was defined multiple times in this request; only the first definition will be used.`,
+            ),
+          );
+        skippedMissingNames.forEach((label) =>
+          warnings.push(
+            `${label} is skipped because it does not include a table_name.`,
+          ),
+        );
+        skippedMissingFields.forEach((label) =>
+          warnings.push(
+            `Table "${label}" is skipped because it does not define any fields.`,
+          ),
+        );
         const warningLines = warnings.length
           ? ["Warnings:", ...warnings.map((w) => `- ${w}`)]
           : [];
+        const summarySection = summaryLines.length
+          ? [
+              `Ready to create ${validTables.length} new table${
+                validTables.length === 1 ? "" : "s"
+              }:`,
+              ...summaryLines,
+            ]
+          : [
+              "No new tables remain after removing existing, duplicate, or invalid table definitions.",
+            ];
         return [
-          `Received ${tables.length} table definition${tables.length === 1 ? "" : "s"}:`,
-          ...summaryLines,
+          `Received ${tables.length} table definition${tables.length === 1 ? "" : "s"}.`,
+          ...summarySection,
           ...warningLines,
         ].join("\n");
       },
       postProcess: async ({ tool_call }) => {
-        console.log("GenerateTablesSkill.provideTools.postProcess called", {
-          has_input: !!tool_call?.input,
-        });
         const payload = payloadFromToolCall(tool_call);
         const tables = payload.tables || [];
+        const { newTables, skippedExisting, skippedDuplicates } =
+          await partitionTablesByExistence(tables);
+        const { validTables, skippedMissingNames, skippedMissingFields } =
+          partitionTablesByValidity(newTables);
         let preview = "";
         try {
-          preview = GenerateTables.render_html({ tables });
+          if (validTables.length) {
+            preview = GenerateTables.render_html({ tables: validTables });
+          } else {
+            preview =
+              '<div class="alert alert-info">No new tables to preview because every provided table already exists or was invalid.</div>';
+          }
         } catch (e) {
+          console.log("We are in postProcess but rendering failed", {
+            e,
+            time: new Date(),
+          });
           preview = `<pre>${JSON.stringify(payload, null, 2)}</pre>`;
         }
+        const warningChunks = [];
+        if (skippedExisting.length)
+          warningChunks.push(
+            `Skipped existing tables: ${skippedExisting.join(", ")}`,
+          );
+        if (skippedDuplicates.length)
+          warningChunks.push(
+            `Ignored duplicate definitions: ${skippedDuplicates.join(", ")}`,
+          );
+        if (skippedMissingNames.length)
+          warningChunks.push(
+            `Missing table_name: ${skippedMissingNames.join(", ")}`,
+          );
+        if (skippedMissingFields.length)
+          warningChunks.push(
+            `Tables without fields: ${skippedMissingFields.join(", ")}`,
+          );
+        const warningHtml = warningChunks.length
+          ? `<div class="alert alert-warning">${warningChunks.join("<br/>")}</div>`
+          : "";
         return {
           stop: true,
-          add_response: preview,
+          add_response: `${warningHtml}${preview}`,
           add_user_action:
-            tables.length > 0
+            validTables.length > 0
               ? {
                   name: "apply_copilot_tables",
                   type: "button",
-                  label: `Create tables (${tables
+                  label: `Create tables (${validTables
                     .map((t) => t.table_name)
                     .join(", ")})`,
-                  input: { tables },
+                  input: { tables: validTables },
                 }
               : undefined,
         };

package/agent-skills/pagegen.js

@@ -33,7 +33,6 @@ class GeneratePageSkill {
   }
 
   constructor(cfg) {
-    console.log("GeneratePageSkill.constructor called", { cfg });
     Object.assign(this, cfg);
   }
 
@@ -77,16 +76,6 @@ class GeneratePageSkill {
     };
   }
   provideTools = () => {
-    let properties = {};
-    (this.toolargs || []).forEach((arg) => {
-      properties[arg.name] = {
-        description: arg.description,
-        type: arg.argtype,
-      };
-      if (arg.options && arg.argtype === "string")
-        properties[arg.name].enum = arg.options.split(",").map((s) => s.trim());
-    });
-
     return {
       type: "function",
       process: async ({ name, existing_page_name }) => {