@saltcorn/copilot 0.7.1 → 0.7.3

package/actions/generate-page.js

@@ -28,7 +28,7 @@ class GeneratePage {
     let namedescription = `The name of the page, this should be a short name which is part of the url. `;
     if (allPageNames.length) {
       namedescription += `These are the names of the exising pages: ${allPageNames.join(
-        ", "
+        ", ",
       )}. Do not pick a name that is identical but follow the same naming convention.`;
     }
     const roles = await User.get_roles();
@@ -65,7 +65,7 @@ class GeneratePage {
     };
   }
   static async system_prompt() {
-    return `Use the generate_page to generate a page.`;
+    return `If the user asks to generate a page, a web page or a landing page, use the generate_page to generate a page.`;
   }
   static async follow_on_generate({ name, page_type }) {
     if (page_type === "Marketing page") {
@@ -272,7 +272,7 @@ class GeneratePage {
     }
     if (segment.type === "container") {
       const { customStyle, style, display, overflow } = splitContainerStyle(
-        segment.style
+        segment.style,
       );
       return {
         ...segment,
@@ -310,17 +310,17 @@ class GeneratePage {
             JSON.stringify(
               GeneratePage.walk_response(JSON.parse(contents)),
               null,
-              2
-            )
-          )
-        )
+              2,
+            ),
+          ),
+        ),
       )
     );
   }
   static async execute(
     { name, title, description, min_role, page_type },
     req,
-    contents
+    contents,
   ) {
     console.log("execute", name, contents);
     const roles = await User.get_roles();
@@ -341,12 +341,12 @@ class GeneratePage {
         "Page created. " +
         a(
           { target: "_blank", href: `/page/${name}`, class: "me-1" },
-          "Go to page"
+          "Go to page",
         ) +
         " | " +
         a(
           { target: "_blank", href: `/pageedit/edit/${name}`, class: "ms-1" },
-          "Configure page"
+          "Configure page",
         ),
     };
   }

package/actions/generate-tables.js

@@ -5,7 +5,7 @@ const Table = require("@saltcorn/data/models/table");
 const Field = require("@saltcorn/data/models/field");
 const { apply, removeAllWhiteSpace } = require("@saltcorn/data/utils");
 const { getActionConfigFields } = require("@saltcorn/data/plugin-helper");
-const { a, pre, script, div } = require("@saltcorn/markup/tags");
+const { a, pre, script, div, domReady } = require("@saltcorn/markup/tags");
 const { fieldProperties } = require("../common");
 
 class GenerateTables {
@@ -13,7 +13,7 @@ class GenerateTables {
   static function_name = "generate_tables";
   static description = "Generate database tables";
 
-  static async json_schema() {
+  static json_schema() {
     const types = Object.values(getState().types);
     const fieldTypeCfg = types.map((ty) => {
       const properties = {
@@ -143,6 +143,10 @@ class GenerateTables {
     need to specify a primary key, a primary key called id with autoincrementing integers is
     autmatically generated. 
 
+    Only include brand-new tables in the generate_tables arguments. If the user references a table
+    that already exists, explain that generate_tables can only add new tables (not modify existing
+    ones) and omit those tables from the tool call.
+
     The database already contains the following tables: 
 
     ${tableLines.join("\n\n")}
@@ -154,7 +158,14 @@ class GenerateTables {
     const mmdia = buildMermaidMarkup(sctables);
     return (
       pre({ class: "mermaid" }, mmdia) +
-      script(`mermaid.run({querySelector: 'pre.mermaid'});`)
+      script(
+        domReady(`
+        ensure_script_loaded("/static_assets/"+_sc_version_tag+"/mermaid.min.js", () => {
+          mermaid.initialize({ startOnLoad: false });
+          mermaid.run({ querySelector: ".mermaid" });
+        });
+      `),
+      ) 
     );
   }
 
@@ -175,9 +186,14 @@ class GenerateTables {
 
   static process_tables(tables) {
     return tables.map((table) => {
+      const sanitizedFields = Array.isArray(table.fields)
+        ? table.fields.filter(
+            (f) => (f?.name || "").toLowerCase() !== "id"
+          )
+        : [];
       return new Table({
         name: table.table_name,
-        fields: table.fields.map((f) => {
+        fields: sanitizedFields.map((f) => {
           const { data_type, reference_table, ...attributes } =
             f.type_and_configuration;
           let type = data_type;

package/actions/generate-workflow.js

@@ -3,24 +3,89 @@ const WorkflowStep = require("@saltcorn/data/models/workflow_step");
 const Trigger = require("@saltcorn/data/models/trigger");
 const Table = require("@saltcorn/data/models/table");
 const { getActionConfigFields } = require("@saltcorn/data/plugin-helper");
-const { a, pre, script, div } = require("@saltcorn/markup/tags");
+const { a, pre, script, div, domReady } = require("@saltcorn/markup/tags");
 const { fieldProperties } = require("../common");
 
+const table_triggers = ["Insert", "Update", "Delete", "Validate"];
+const additional_triggers_with_onlyif = ["Login", "PageLoad"];
+
+const optionNames = (options = []) =>
+  options
+    .map((opt) =>
+      typeof opt === "string"
+        ? opt
+        : opt?.name || opt?.label || opt?.value || "",
+    )
+    .filter(Boolean);
+
+const joinOptionNames = (options = [], limit = 10) => {
+  const names = optionNames(options);
+  if (!names.length) return "none";
+  const shown = names.slice(0, limit).join(", ");
+  return names.length > limit ? `${shown}, ...` : shown;
+};
+
+const summarizeTriggerActionMatrix = () => {
+  try {
+    const allActions = Trigger.action_options({
+      notRequireRow: false,
+      workflow: true,
+    });
+    const noRowActions = Trigger.action_options({
+      notRequireRow: true,
+      workflow: true,
+    });
+    const lines = Trigger.when_options.map((when) => {
+      const pool = table_triggers.includes(when) ? allActions : noRowActions;
+      return `* ${when}: ${joinOptionNames(pool)}`;
+    });
+    const notes = `Triggers requiring a table context: ${table_triggers.join(", ")}.
+Additional triggers that commonly use contextual only_if checks: ${additional_triggers_with_onlyif.join(", ")}.`;
+    return `${notes}
+${lines.join("\n")}`;
+  } catch (e) {
+    console.error("GenerateWorkflow: action matrix failed", e);
+    return "";
+  }
+};
+
+const summarizeTables = async () => {
+  try {
+    const tables = await Table.find({});
+    if (!tables?.length) return "";
+    const fieldType = (f = {}) =>
+      f.pretty_type || f.type?.name || f.type || f.input_type || "unknown";
+    const rows = tables.map((table) => {
+      const description = table.description ? ` – ${table.description}` : "";
+      const fields = (table.fields || [])
+        .slice(0, 8)
+        .map((f) => `${f.name}:${fieldType(f)}`)
+        .join(", ");
+      const fieldText = fields ? ` fields: ${fields}` : "";
+      return `* ${table.name}${description}${fieldText}`;
+    });
+    return rows.join("\n");
+  } catch (e) {
+    console.error("GenerateWorkflow: table summary failed", e);
+    return "";
+  }
+};
+
 const steps = async () => {
   const actionExplainers = WorkflowStep.builtInActionExplainers();
   const actionFields = await WorkflowStep.builtInActionConfigFields();
 
   let stateActions = getState().actions;
   const stateActionList = Object.entries(stateActions).filter(
-    ([k, v]) => !v.disableInWorkflow
+    ([k, v]) => !v.disableInWorkflow,
   );
 
   const stepTypeAndCfg = Object.keys(actionExplainers).map((actionName) => {
-    const properties = { 
-      step_type: { type: "string", enum: [actionName] }
+    const properties = {
+      step_type: { type: "string", enum: [actionName] },
     };
     const myFields = actionFields.filter(
-      (f) => f.showIf?.wf_action_name === actionName
+      (f) => f.showIf?.wf_action_name === actionName,
     );
     const required = ["step_type"];
     myFields.forEach((f) => {
@@ -39,8 +104,8 @@ const steps = async () => {
   });
   for (const [actionName, action] of stateActionList) {
     try {
-      const properties = { 
-        step_type: { type: "string", enum: [actionName] }
+      const properties = {
+        step_type: { type: "string", enum: [actionName] },
       };
       const cfgFields = await getActionConfigFields(action, null, {
         mode: "workflow",
@@ -73,9 +138,9 @@ const steps = async () => {
   //TODO workflows
   for (const trigger of triggers) {
     const properties = {
-      step_type: { 
+      step_type: {
         type: "string",
-        enum: [trigger.name],  
+        enum: [trigger.name],
       },
     };
     if (trigger.table_id) {
@@ -88,7 +153,7 @@ const steps = async () => {
       properties.row_expr = {
         type: "string",
         description: `JavaScript expression for the input to the action. This should be an expression for an object, with the following field name and types: ${fieldSpecs.join(
-          "; "
+          "; ",
         )}.`,
       };
     }
@@ -150,11 +215,10 @@ class GenerateWorkflow {
           description:
             "When the workflow should trigger. Optional, leave blank if unspecified or workflow will be run on button click",
           type: "string",
-          enum: ["Insert", "Delete", "Update", "Daily", "Hourly", "Weekly"],
+          enum: Trigger.when_options,
         },
         trigger_table: {
-          description:
-            "If the workflow trigger is Insert, Delete or Update, the name of the table that triggers the workflow",
+          description: `If the workflow trigger is ${table_triggers.join(", ")}, the name of the table that triggers the workflow`,
           type: "string",
         },
       },
@@ -165,15 +229,29 @@ class GenerateWorkflow {
     const actionExplainers = WorkflowStep.builtInActionExplainers();
     let stateActions = getState().actions;
     const stateActionList = Object.entries(stateActions).filter(
-      ([k, v]) => !v.disableInWorkflow
+      ([k, v]) => !v.disableInWorkflow,
     );
+    const [tableSummary, triggerMatrix] = await Promise.all([
+      summarizeTables(),
+      Promise.resolve(summarizeTriggerActionMatrix()),
+    ]);
+    const contextBlocks = [
+      tableSummary && `Current tables and key fields:\n${tableSummary}`,
+      triggerMatrix && `Workflow trigger compatibility:\n${triggerMatrix}`,
+    ]
+      .filter(Boolean)
+      .join("\n\n");
 
     return `Use the generate_workflow tool to construct computational workflows according to specifications. You must create 
   the workflow by calling the generate_workflow tool, with the step required to implement the specification.
   
+  ${contextBlocks}
+  
   The steps are specified as JSON objects. Each step has a name, specified in the step_name key in the JSON object. 
   The step name should be a valid JavaScript identifier.
   
+  When the user explicitly names an action/step type (for example "ForLoop" or "Toast"), ensure the generated workflow contains at least one step whose step_configuration.step_type exactly matches every requested action. Only skip this if the action genuinely does not exist; in that case, clearly explain the omission in the workflow description.
+
   Each run of the workflow is executed in the presence of a context, which is a JavaScript object that individual
   steps can read values from and write values to. This context is a state that is persisted on disk for each workflow 
   run. 
@@ -239,7 +317,7 @@ class GenerateWorkflow {
 
   static async execute(
     { workflow_steps, workflow_name, when_trigger, trigger_table },
-    req
+    req,
   ) {
     const steps = this.process_all_steps(workflow_steps);
     let table_id;
@@ -268,7 +346,7 @@ class GenerateWorkflow {
         "Workflow created. " +
         a(
           { target: "_blank", href: `/actions/configure/${trigger.id}` },
-          "Configure workflow."
+          "Configure workflow.",
         ),
     };
   }
@@ -286,21 +364,29 @@ class GenerateWorkflow {
         step.id = ix + 1;
       });
       const mmdia = WorkflowStep.generate_diagram(
-        steps.map((s) => new WorkflowStep(s))
+        steps.map((s) => new WorkflowStep(s)),
       );
+      console.log({ mmdia });
       return (
         div(
           `${workflow_name}${when_trigger ? `: ${when_trigger}` : ""}${
             trigger_table ? ` on ${trigger_table}` : ""
-          }`
+          }`,
         ) +
         pre({ class: "mermaid" }, mmdia) +
-        script(`mermaid.run({querySelector: 'pre.mermaid'});`)
+        script(
+          domReady(`
+          ensure_script_loaded("/static_assets/"+_sc_version_tag+"/mermaid.min.js", () => {
+            mermaid.initialize({ startOnLoad: false });
+            mermaid.run({ querySelector: ".mermaid" });
+          });
+        `),
+        ) 
       );
     }
 
     return `A workflow! Step names: ${workflow_steps.map(
-      (s) => s.step_name
+      (s) => s.step_name,
     )}. Upgrade Saltcorn to see diagrams in copilot`;
   }
 

package/agent-skills/database-design.js

@@ -0,0 +1,349 @@
+const GenerateTables = require("../actions/generate-tables");
+const Table = require("@saltcorn/data/models/table");
+
+const normalizeTablesPayload = (rawPayload) => {
+  if (!rawPayload) return { tables: [] };
+  let payload = rawPayload;
+  if (typeof payload === "string") {
+    try {
+      payload = JSON.parse(payload);
+    } catch (e) {
+      console.error("Failed to parse generate_tables payload", e);
+      return { tables: [] };
+    }
+  }
+  if (typeof payload !== "object") return { tables: [] };
+  const normalized = { ...payload };
+  normalized.tables = Array.isArray(normalized.tables)
+    ? normalized.tables.filter(Boolean).map((table) => ({
+        ...table,
+        fields: Array.isArray(table?.fields)
+          ? table.fields.filter(Boolean).map((field) => ({
+              ...field,
+              type_and_configuration: field?.type_and_configuration || {},
+            }))
+          : [],
+      }))
+    : [];
+  return normalized;
+};
+
+const summarizeTables = (tables) =>
+  tables.map((table, idx) => {
+    const fields = (table.fields || []).slice(0, 5);
+    const fieldSummary = fields
+      .map((field) => {
+        const fname = field?.name || "(missing name)";
+        const ftype = field?.type_and_configuration?.data_type || "Unknown";
+        return `${fname}:${ftype}`;
+      })
+      .join(", ");
+    const ellipsis = table.fields.length > fields.length ? "..." : "";
+    return `${idx + 1}. ${table.table_name || "(missing name)"} – ${
+      table.fields.length
+    } field(s)${fieldSummary ? ` (${fieldSummary}${ellipsis})` : ""}`;
+  });
+
+const collectTableWarnings = (tables) => {
+  const warnings = [];
+  const seenTables = new Set();
+  tables.forEach((table, tableIdx) => {
+    const tableLabel = table.table_name || `Table #${tableIdx + 1}`;
+    if (!table.table_name)
+      warnings.push(`${tableLabel} is missing a table_name.`);
+    else if (seenTables.has(table.table_name))
+      warnings.push(`Duplicate table name "${table.table_name}".`);
+    else seenTables.add(table.table_name);
+
+    if (!Array.isArray(table.fields) || table.fields.length === 0)
+      warnings.push(`${tableLabel} does not define any fields.`);
+
+    const fieldNames = new Set();
+    (table.fields || []).forEach((field, fieldIdx) => {
+      const fieldLabel = field?.name || `Field #${fieldIdx + 1}`;
+      if (!field?.name)
+        warnings.push(`${tableLabel} has a field without a name.`);
+      else if (fieldNames.has(field.name))
+        warnings.push(`${tableLabel} repeats the field name "${field.name}".`);
+      else fieldNames.add(field.name);
+
+      if (!field?.type_and_configuration?.data_type)
+        warnings.push(
+          `${tableLabel}.${fieldLabel} must include type_and_configuration.data_type.`,
+        );
+
+      if ((field?.name || "").toLowerCase() === "id")
+        warnings.push(
+          `${tableLabel}.${fieldLabel} should be omitted because every table already has an auto-increment id.`,
+        );
+    });
+  });
+  return warnings;
+};
+
+const fetchExistingTableNameSet = async () => {
+  const existingTables = await Table.find({});
+  const names = new Set();
+  existingTables.forEach((table) => {
+    if (table?.name) names.add(table.name.toLowerCase());
+  });
+  return names;
+};
+
+const partitionTablesByExistence = async (tables = []) => {
+  const existingNames = await fetchExistingTableNameSet();
+  const seenNewNames = new Set();
+  const newTables = [];
+  const skippedExisting = [];
+  const skippedDuplicates = [];
+  tables.forEach((table) => {
+    const tableName =
+      typeof table?.table_name === "string" ? table.table_name.trim() : "";
+    const normalized = tableName.toLowerCase();
+    if (tableName && existingNames.has(normalized)) {
+      skippedExisting.push(tableName);
+      return;
+    }
+    if (tableName && seenNewNames.has(normalized)) {
+      skippedDuplicates.push(tableName);
+      return;
+    }
+    if (tableName) seenNewNames.add(normalized);
+    newTables.push(table);
+  });
+  return { newTables, skippedExisting, skippedDuplicates };
+};
+
+const partitionTablesByValidity = (tables = []) => {
+  const validTables = [];
+  const skippedMissingNames = [];
+  const skippedMissingFields = [];
+  tables.forEach((table, idx) => {
+    const rawName =
+      typeof table?.table_name === "string" ? table.table_name.trim() : "";
+    const fallbackLabel = rawName || `Table #${idx + 1}`;
+    if (!rawName) {
+      skippedMissingNames.push(fallbackLabel);
+      return;
+    }
+    const fields = Array.isArray(table?.fields) ? table.fields : [];
+    if (!fields.length) {
+      skippedMissingFields.push(rawName);
+      return;
+    }
+    validTables.push({ ...table, table_name: rawName, fields });
+  });
+  return { validTables, skippedMissingNames, skippedMissingFields };
+};
+
+const payloadFromToolCall = (tool_call) => {
+  if (!tool_call) return { tables: [] };
+  if (tool_call.input) return normalizeTablesPayload(tool_call.input);
+  if (tool_call.function?.arguments)
+    return normalizeTablesPayload(tool_call.function.arguments);
+  return { tables: [] };
+};
+
+class GenerateTablesSkill {
+  static skill_name = "Database design";
+
+  get skill_label() {
+    return "Database Design";
+  }
+
+  constructor(cfg) {
+    Object.assign(this, cfg);
+  }
+
+  async systemPrompt() {
+    return await GenerateTables.system_prompt();
+  }
+
+  get userActions() {
+    return {
+      async apply_copilot_tables({ user, tables }) {
+        if (!tables?.length) return { notify: "Nothing to create." };
+        const { newTables, skippedExisting, skippedDuplicates } =
+          await partitionTablesByExistence(tables);
+        const { validTables, skippedMissingNames, skippedMissingFields } =
+          partitionTablesByValidity(newTables);
+        if (!validTables.length) {
+          const skippedMessages = [];
+          if (skippedExisting.length)
+            skippedMessages.push(
+              `Existing tables: ${skippedExisting.join(", ")}`,
+            );
+          if (skippedDuplicates.length)
+            skippedMessages.push(
+              `Duplicate definitions: ${skippedDuplicates.join(", ")}`,
+            );
+          if (skippedMissingNames.length)
+            skippedMessages.push(
+              `Missing table_name: ${skippedMissingNames.join(", ")}`,
+            );
+          if (skippedMissingFields.length)
+            skippedMessages.push(
+              `Tables without fields: ${skippedMissingFields.join(", ")}`,
+            );
+          return {
+            notify:
+              skippedMessages.length > 0
+                ? `Nothing to create. Skipped ${skippedMessages.join("; ")}.`
+                : "Nothing to create.",
+          };
+        }
+        await GenerateTables.execute({ tables: validTables }, { user });
+        const createdNames = validTables.map((t) => t.table_name).join(", ");
+        const skippedMessages = [];
+        if (skippedExisting.length)
+          skippedMessages.push(
+            `Skipped existing tables: ${skippedExisting.join(", ")}`,
+          );
+        if (skippedDuplicates.length)
+          skippedMessages.push(
+            `Ignored duplicate definitions: ${skippedDuplicates.join(", ")}`,
+          );
+        if (skippedMissingNames.length)
+          skippedMessages.push(
+            `Missing table_name: ${skippedMissingNames.join(", ")}`,
+          );
+        if (skippedMissingFields.length)
+          skippedMessages.push(
+            `Tables without fields: ${skippedMissingFields.join(", ")}`,
+          );
+        return {
+          notify: [`Created tables: ${createdNames}`, ...skippedMessages].join(
+            ". ",
+          ),
+        };
+      },
+    };
+  }
+
+  provideTools = () => {
+    const parameters = GenerateTables.json_schema();
+    return {
+      type: "function",
+      process: async (input) => {
+        const payload = normalizeTablesPayload(input);
+        const tables = payload.tables || [];
+        if (!tables.length) {
+          return "No tables were provided for generate_tables.";
+        }
+        const { newTables, skippedExisting, skippedDuplicates } =
+          await partitionTablesByExistence(tables);
+        const { validTables, skippedMissingNames, skippedMissingFields } =
+          partitionTablesByValidity(newTables);
+        const summaryLines = validTables.length
+          ? summarizeTables(validTables).map((line) => `- ${line}`)
+          : [];
+        const warnings = collectTableWarnings(tables);
+        if (skippedExisting.length)
+          skippedExisting.forEach((name) =>
+            warnings.push(
+              `Table "${name}" already exists and will not be recreated by generate_tables.`,
+            ),
+          );
+        if (skippedDuplicates.length)
+          skippedDuplicates.forEach((name) =>
+            warnings.push(
+              `Table "${name}" was defined multiple times in this request; only the first definition will be used.`,
+            ),
+          );
+        skippedMissingNames.forEach((label) =>
+          warnings.push(
+            `${label} is skipped because it does not include a table_name.`,
+          ),
+        );
+        skippedMissingFields.forEach((label) =>
+          warnings.push(
+            `Table "${label}" is skipped because it does not define any fields.`,
+          ),
+        );
+        const warningLines = warnings.length
+          ? ["Warnings:", ...warnings.map((w) => `- ${w}`)]
+          : [];
+        const summarySection = summaryLines.length
+          ? [
+              `Ready to create ${validTables.length} new table${
+                validTables.length === 1 ? "" : "s"
+              }:`,
+              ...summaryLines,
+            ]
+          : [
+              "No new tables remain after removing existing, duplicate, or invalid table definitions.",
+            ];
+        return [
+          `Received ${tables.length} table definition${tables.length === 1 ? "" : "s"}.`,
+          ...summarySection,
+          ...warningLines,
+        ].join("\n");
+      },
+      postProcess: async ({ tool_call }) => {
+        const payload = payloadFromToolCall(tool_call);
+        const tables = payload.tables || [];
+        const { newTables, skippedExisting, skippedDuplicates } =
+          await partitionTablesByExistence(tables);
+        const { validTables, skippedMissingNames, skippedMissingFields } =
+          partitionTablesByValidity(newTables);
+        let preview = "";
+        try {
+          if (validTables.length) {
+            preview = GenerateTables.render_html({ tables: validTables });
+          } else {
+            preview =
+              '<div class="alert alert-info">No new tables to preview because every provided table already exists or was invalid.</div>';
+          }
+        } catch (e) {
+          console.log("We are in postProcess but rendering failed", {
+            e,
+            time: new Date(),
+          });
+          preview = `<pre>${JSON.stringify(payload, null, 2)}</pre>`;
+        }
+        const warningChunks = [];
+        if (skippedExisting.length)
+          warningChunks.push(
+            `Skipped existing tables: ${skippedExisting.join(", ")}`,
+          );
+        if (skippedDuplicates.length)
+          warningChunks.push(
+            `Ignored duplicate definitions: ${skippedDuplicates.join(", ")}`,
+          );
+        if (skippedMissingNames.length)
+          warningChunks.push(
+            `Missing table_name: ${skippedMissingNames.join(", ")}`,
+          );
+        if (skippedMissingFields.length)
+          warningChunks.push(
+            `Tables without fields: ${skippedMissingFields.join(", ")}`,
+          );
+        const warningHtml = warningChunks.length
+          ? `<div class="alert alert-warning">${warningChunks.join("<br/>")}</div>`
+          : "";
+        return {
+          stop: true,
+          add_response: `${warningHtml}${preview}`,
+          add_user_action:
+            validTables.length > 0
+              ? {
+                  name: "apply_copilot_tables",
+                  type: "button",
+                  label: `Create tables (${validTables
+                    .map((t) => t.table_name)
+                    .join(", ")})`,
+                  input: { tables: validTables },
+                }
+              : undefined,
+        };
+      },
+      function: {
+        name: GenerateTables.function_name,
+        description: GenerateTables.description,
+        parameters,
+      },
+    };
+  };
+}
+
+module.exports = GenerateTablesSkill;