@saltcorn/copilot 0.8.2 → 0.8.3

package/actions/generate-page.js

@@ -65,13 +65,17 @@ class GeneratePage {
     };
   }
   static async system_prompt() {
-    return `If the user asks to generate a page, a web page or a landing page, use the generate_page to generate a page.`;
+    return `If the user asks to generate a page, a web page or a landing page, use the generate_page to generate a page.
+
+Critical: When generating HTML, your response must be the complete, final HTML document and nothing else. Do not include reasoning, planning notes, explanatory text, TODO comments, markdown code fences, or placeholder content. Everything you write will be stored verbatim and rendered directly to users — there is no review step between your output and the live page.`;
   }
   static async follow_on_generate({ name, page_type }) {
     if (page_type === "Marketing page") {
       return {
         prompt:
-          "Generate the HTML for the web page using the Bootstrap 5 CSS framework.",
+          "Generate the HTML for the web page using the Bootstrap 5 CSS framework. " +
+          'Include a toast notification area just before the closing </body> tag: ' +
+          '<div id="toasts-area" class="toast-container position-fixed top-0 start-50 p-0" style="z-index:999;" aria-live="polite" aria-atomic="true"></div>',
       };
     }
     const prompt = `Now generate the contents of the ${name} page`;

package/actions/generate-tables.js

@@ -156,6 +156,11 @@ class GenerateTables {
                 type: "string",
                 description: "The name of the table",
               },
+              description: {
+                type: "string",
+                description:
+                  "A short human-readable description of what this table stores and its role in the application",
+              },
               fields: {
                 type: "array",
                 items: this.field_item_schema(),
@@ -263,6 +268,28 @@ class GenerateTables {
     The type_and_configuration.data_type for a calculated field should reflect the return type of
     the expression (e.g. Integer, Float, String, Bool).
 
+    ## Table descriptions
+
+    Every table you define MUST include a description — a short sentence explaining what
+    the table stores and its role in the application. This description is shown in
+    subsequent prompts to give the planner context about the schema, so make it
+    informative (e.g. "Stores billable time entries logged by lawyers against a project").
+
+    ## Bulk data import and export
+
+    Do NOT create tables whose purpose is to trigger a bulk import or export (e.g. a
+    table with a File field that a user fills in via an Edit view to start an import).
+    Bulk import and export is a UI concern — there are plugins that provide dedicated
+    viewtemplates operating directly on the target table, which is a much better
+    solution than a workaround table with a file field and an Edit view.
+
+    A tracking table that records the status and outcome of an automated import or
+    export process (e.g. import_jobs) is acceptable, but only if the table is populated
+    automatically by the process — not filled in manually by a user. Such tables must
+    have a description that clearly says they are auto-populated and must not be edited
+    by hand (e.g. "Auto-populated by the import process. Records status and errors for
+    each import run. Not editable by users.").
+
     ## Existing tables
 
     The database already contains the following tables:
@@ -292,8 +319,10 @@ class GenerateTables {
   }
 
   static async execute({ tables }, req) {
-    const sctables = this.process_tables(tables);
-    for (const table of sctables) await Table.create(table.name);
+    const existingDbTables = await Table.find({});
+    const sctables = this.process_tables(tables, existingDbTables);
+    for (const table of sctables)
+      await Table.create(table.name, { description: table.description || "" });
     for (const table of sctables) {
       for (const field of table.fields) {
         field.table = Table.findOne({ name: table.name });
@@ -322,10 +351,11 @@ class GenerateTables {
     const added = [];
     const updated = [];
 
+    const existingDbTables = await Table.find({});
     for (const f of sanitized) {
       const fname = (f?.name || "").toLowerCase();
       if (!fname) continue;
-      const processed = this.process_field(f, []);
+      const processed = this.process_field(f, [], existingDbTables);
       const existing = existingFieldMap.get(fname);
       if (!existing) {
         processed.table = table;
@@ -359,7 +389,7 @@ class GenerateTables {
     return { added, updated };
   }
 
-  static process_field(f, allTablesList = []) {
+  static process_field(f, allTablesList = [], dbTables = []) {
     if (f.aggregation) {
       const { data_type } = f.type_and_configuration || {
         data_type: "Integer",
@@ -387,6 +417,8 @@ class GenerateTables {
       f.type_and_configuration || { data_type: "String" };
     let type = data_type;
     const scattributes = { ...attributes, importance: f.importance };
+    if (!scattributes.min_length) delete scattributes.min_length;
+    if (!scattributes.max_length) delete scattributes.max_length;
     if (data_type === "ForeignKey") {
       type = `Key to ${reference_table}`;
       const refTableHere = allTablesList.find(
@@ -404,6 +436,21 @@ class GenerateTables {
         }
       } else if (reference_table === "users") {
         scattributes.summary_field = "email";
+      } else {
+        const dbTable = dbTables.find((t) => t.name === reference_table);
+        if (dbTable) {
+          const strFields = (dbTable.fields || []).filter(
+            (rf) => rf.type?.name === "String" || rf.type === "String"
+          );
+          if (strFields.length) {
+            const maxImp = strFields.reduce((prev, curr) => {
+              const pi = prev?.attributes?.importance ?? 0;
+              const ci = curr?.attributes?.importance ?? 0;
+              return pi >= ci ? prev : curr;
+            });
+            scattributes.summary_field = maxImp.name;
+          }
+        }
       }
     }
     return {
@@ -417,14 +464,17 @@ class GenerateTables {
     };
   }
 
-  static process_tables(tables) {
+  static process_tables(tables, dbTables = []) {
     return tables.map((table) => {
       const sanitizedFields = Array.isArray(table.fields)
         ? table.fields.filter((f) => (f?.name || "").toLowerCase() !== "id")
         : [];
       return new Table({
         name: table.table_name,
-        fields: sanitizedFields.map((f) => this.process_field(f, tables)),
+        description: table.description || "",
+        fields: sanitizedFields.map((f) =>
+          this.process_field(f, tables, dbTables)
+        ),
       });
     });
   }

package/actions/generate-workflow.js

@@ -272,6 +272,16 @@ class GenerateWorkflow {
   IMPORTANT — keep row expressions simple; use dedicated steps for data fetching:
   Row expressions (e.g. in modify_row) should be simple references to values already in the context — not inline queries or complex logic. If the values you need are not already in context, add a dedicated step before the row expression step to fetch or compute them and write the results into the context. Choose the step type that fits the job: TableQuery to query a table, run_js_code for custom computation, or any other appropriate step. Each step should do one clear thing; the row expression then just picks the relevant context values.
 
+  CRITICAL — modify_row without a triggering row requires where="Database" + select_table + query (all three):
+  A basic modify_row step updates the row that triggered the workflow. If the workflow is NOT triggered by a specific table row (e.g. triggered by a button with no table, or running inside a ForLoop), there is no triggering row and table is undefined — this crashes with "Cannot read properties of undefined (reading 'tryUpdateRow')".
+  To update arbitrary rows in a workflow, the step configuration MUST include all three of:
+    - where: "Database"
+    - select_table: the name of the table to update (string)
+    - query: a JS expression (evaluated against the workflow context) returning a where-clause object that identifies the row(s) to update — e.g. {id: bh_id}
+  If any of these three is missing, the action falls through to the triggering-row path and crashes when table is undefined.
+  Example modify_row step configuration for updating a billable_hours row inside a ForLoop where bh_id is in context:
+    { "step_type": "modify_row", "where": "Database", "select_table": "billable_hours", "query": "{id: bh_id}", "row_expr": "{invoiced: true}" }
+
   CRITICAL — every workflow must form a single connected chain from the first step to the last:
   - Every step except the very last one MUST have a next_step that names another step in the workflow.
   - The very last step must have next_step omitted or set to an empty string to terminate the workflow.
@@ -282,6 +292,20 @@ class GenerateWorkflow {
   steps can read values from and write values to. This context is a state that is persisted on disk for each workflow
   run.
 
+  Important: every {{}} interpolation in any workflow step (email body, email subject, filename, prompt template,
+  etc.) must reference a variable that already exists in the workflow context at the point the step runs.
+  Do NOT use fallback expressions such as {{invoice_date || new Date().toISOString()}} — if the variable is not
+  defined, the interpolation engine throws before the || fallback can execute. For example, a send_email step
+  using {{invoice_date}} will fail with "invoice_date is not defined" if no prior step put invoice_date in context.
+  If a value might not be in context at that point, retrieve or compute it in an earlier step and store it under
+  a known key.
+
+  Important: when a workflow is triggered from a Show view action button, the trigger must have its table set to
+  the view's table. Saltcorn then automatically passes the full row as the initial workflow context — all field
+  values are available by their field names (e.g. id, name, contact_email). Do NOT attempt to pass row data
+  through a state property on the actions array — it is silently ignored. If the trigger has no table set, the
+  workflow starts with an empty context and all field references will throw "is not defined".
+
   Each step can have a next_step key which is the name of the next step, or a JavaScript expression which evaluates
   to the name of the next step based on the context. In the evaluation of the next step, each value in the context is
   in scope and can be addressed directly. Identifiers for the step names are also in scope, the name of the next step
@@ -307,16 +331,43 @@ class GenerateWorkflow {
   Most of them are are explained by their parameter descriptions. Here are some additional information for some
   step types:
   
+  TableQuery: stores the query results in the context under the name given in the query_variable configuration field.
+  This name is chosen by you — pick a short descriptive name (e.g. "billable_hours", "lawyer_rows").
+  Every subsequent step that reads those results — whether a run_js_code step or another TableQuery's query_object
+  expression — must use this exact variable name. Mismatched names cause "X is not defined" errors at runtime.
+
+  Important: an insert step writes ONLY the new row's id into the context (e.g. new_invoice_id). It does NOT
+  write any other field values from the inserted row. If a later step needs fields from that row (e.g. invoice_date,
+  total_amount, contact_email), add a TableQuery step immediately after the insert to fetch the row by id and make
+  its fields available in context. For example, query {id: new_invoice_id} on the invoices table with
+  query_variable "invoice_row", then reference invoice_row.invoice_date, invoice_row.total_amount etc. in
+  subsequent steps. Never assume that insert fields are in context — only the id is guaranteed.
+
   run_js_code: if the step_type is "run_js_code" then the step object should include the JavaScript code to be executed in the "code"
   key. You can use await in the code if you need to run asynchronous code. The values in the context are directly in scope and can be accessed using their name. In addition, the variable
   "context" is also in scope and can be used to address the context as a whole. To write values to the context, return an
   object. The following Saltcorn models are already available in scope without any require: Table, Row, Field, User, View, Trigger, Page, File — use them directly.
   When you need to require other modules, always use a plain require call, e.g. const moment = require('moment');
-  NEVER use the pattern const X = X || require(...) — this causes a ReferenceError because const variables cannot be referenced before initialization. The values in this object will be written into the current context. If a value already exists in the context 
+  NEVER use the pattern const X = X || require(...) — this causes a ReferenceError because const variables cannot be referenced before initialization.
+  The values in this object will be written into the current context. If a value already exists in the context
   it will be overwritten. For example, If the context contains values x and y which are numbers and you would like to push
-  the value "sum" which is the sum of x and y, then use this as the code: return {sum: x+y}. You cannot set the next step in the 
+  the value "sum" which is the sum of x and y, then use this as the code: return {sum: x+y}. You cannot set the next step in the
   return object or by returning a string from a run_js_code step, this will not work. To set the next step from a code action, always use the next_step property of the step object.
-  This expression for the next step can depend on value pushed to the context (by the return object in the code) as these values are in scope.  
+  This expression for the next step can depend on value pushed to the context (by the return object in the code) as these values are in scope.
+
+  SetContext: sets one or more variables in the workflow context. The configuration must include a ctx_values key
+  whose value is a JavaScript object expression — it MUST be wrapped in curly braces.
+  Example: {running_total: 0} initialises a counter, {invoice_date: new Date().toISOString().slice(0,10)} sets a date.
+  Do NOT write bare key-value pairs such as running_total: 0 — without the braces the expression is invalid and
+  throws "running_total is not defined" when a later step tries to read it.
+  The object expression can reference existing context variables: {total: subtotal + tax}.
+
+  Early termination pattern: to stop a workflow when a condition is not met (e.g. no rows found),
+  use a run_js_code step that writes a boolean flag to the context (e.g. return {has_rows: billable_hours.length > 0}),
+  then set that step's next_step to a conditional expression (e.g. has_rows ? "next_real_step" : "abort_step"),
+  and add a TerminateWorkflow step named "abort_step".
+  Do NOT try to terminate from inside run_js_code by returning null, throwing, or omitting a return — those do not stop the workflow.
+  The conditional must be in the step's next_step field, not inside the code.  
   
   ForLoop: ForLoop steps loop over an array which is specified by the array_expression JavaScript expression. Execution of the workflow steps is temporarily diverted to another set
   of steps, starting from the step specified by the loop_body_inital_step value, and runs until it encounters a

package/agent-skills/pagegen.js

@@ -60,21 +60,51 @@ class GeneratePageSkill {
         html,
         min_role = 100,
       }) {
-        const file = await File.from_contents(
-          `${name}.html`,
-          "text/html",
-          html,
-          user?.id,
-          min_role
-        );
-
-        await Page.create({
-          name,
-          title,
-          description,
-          min_role,
-          layout: { html_file: file.path_to_serve },
-        });
+        const existingPage = Page.findOne({ name });
+        if (existingPage) {
+          // Overwrite existing file if the page already has one, otherwise create new
+          let filePath;
+          if (existingPage.layout?.html_file) {
+            const existingFile = await File.findOne(
+              existingPage.layout.html_file
+            );
+            if (existingFile) {
+              await existingFile.overwrite_contents(html);
+              filePath = existingFile.path_to_serve;
+            }
+          }
+          if (!filePath) {
+            const file = await File.from_contents(
+              `${name}.html`,
+              "text/html",
+              html,
+              user?.id,
+              min_role
+            );
+            filePath = file.path_to_serve;
+          }
+          await Page.update(existingPage.id, {
+            title,
+            description,
+            min_role,
+            layout: { html_file: filePath },
+          });
+        } else {
+          const file = await File.from_contents(
+            `${name}.html`,
+            "text/html",
+            html,
+            user?.id,
+            min_role
+          );
+          await Page.create({
+            name,
+            title,
+            description,
+            min_role,
+            layout: { html_file: file.path_to_serve },
+          });
+        }
         setTimeout(() => getState().refresh_pages(), 200);
         return {
           notify: `Page saved: <a target="_blank" href="/page/${name}">${name}</a>`,
@@ -104,62 +134,130 @@ class GeneratePageSkill {
         } else return "Metadata recieved";
       },
       postProcess: async ({ tool_call, generate, req }) => {
-        const str = await generate(
-          `Now generate the contents of the ${tool_call.input.name} HTML page. If I asked you to embed a view, 
- use the <embed-view> self-closing tag to do so, setting the view name in the viewname attribute. For example, 
+        const version_tag = db.connectObj.version_tag;
+        const asset = (name) => `/static_assets/${version_tag}/${name}`;
+        const { name, title, description, min_role, page_type } =
+          tool_call.input;
+
+        if (page_type === "Layout page") {
+          const str = await generate(
+            `Generate the Saltcorn layout JSON for the ${name} page. ` +
+              `Your response must be a single valid JSON object — no explanatory text, no markdown fences, no reasoning. ` +
+              `\n\nThe layout is a tree of segments. The top-level object is:\n` +
+              `  {"above": [...segments...]}  — stack items vertically\n` +
+              `or:\n` +
+              `  {"besides": [...segments...], "widths": [6,6]}  — place side by side (widths must sum to 12)\n` +
+              `\nAvailable segment types:\n` +
+              `- Embed a view (receives page URL state, e.g. id):  {"type":"view","view":"viewname","state":"shared"}\n` +
+              `  Use state "shared" whenever the embedded view needs variables from the page URL (such as id for a Show view).\n` +
+              `  To pass only specific variables, add extra_state_fml: {"type":"view","view":"viewname","state":"shared","extra_state_fml":"{id: id}"}\n` +
+              `- HTML block:         {"type":"blank","isHTML":true,"contents":"<p>html</p>"}\n` +
+              `- Text block:         {"type":"blank","contents":"plain text"}\n` +
+              `- Container/wrapper:  {"type":"container","customClass":"p-3","htmlElement":"div","contents":{segment}}\n` +
+              `- Vertical stack:     {"above":[{segment},{segment}]}\n` +
+              `- Horizontal split:   {"besides":[{segment},{segment}],"widths":[8,4]}\n`
+          );
+          const jsonStr = str.includes("```json")
+            ? str.split("```json")[1].split("```")[0]
+            : str.includes("```")
+            ? str.split("```")[1].split("```")[0]
+            : str;
+          let layout;
+          try {
+            layout = JSON.parse(jsonStr.trim());
+          } catch (e) {
+            return {
+              stop: true,
+              add_response: `Error parsing layout JSON for ${name}: ${e.message}`,
+            };
+          }
+          if (this.yoloMode) {
+            const existingPage = await Page.findOne({ name });
+            if (existingPage) {
+              await Page.update(existingPage.id, {
+                title,
+                description,
+                min_role: min_role ?? 100,
+                layout,
+              });
+            } else {
+              await Page.create({
+                name,
+                title,
+                description,
+                min_role: min_role ?? 100,
+                layout,
+              });
+            }
+            setTimeout(() => getState().refresh_pages(), 200);
+            return {
+              stop: true,
+              add_response: `Page ${name} created.`,
+            };
+          }
+          return {
+            stop: true,
+            add_response: `Page ${name} layout generated (preview not available for layout pages).`,
+          };
+        } else {
+          const str = await generate(
+            `Generate the complete HTML for the ${name} page. Your response must be the raw HTML document only — no explanatory text before or after, no markdown code fences, no reasoning, no placeholder comments. The output is saved and rendered directly to users without any review step, so anything you write outside valid HTML will be visible on the page.
+
+ If I asked you to embed a view, use the <embed-view> self-closing tag to do so, setting the view name in the viewname attribute. For example,
  to embed the view LeadForm inside a div, write: <div><embed-view viewname="LeadForm"></div>
- 
+
  If you need to include the standard bootstrap CSS and javascript files, they are available as:
 
    <link href="https://cdn.jsdelivr.net/npm/bootstrap@5.3.8/dist/css/bootstrap.min.css" rel="stylesheet" integrity="sha384-sRIl4kxILFvY47J16cr9ZwB07vP4J8+LH7qKQnuqkuIAvNWLzeN8tE5YBujZqJLB" crossorigin="anonymous">
 
- and 
-  <script src="/static_assets/js/jquery-3.6.0.min.js"></script>
+ and
+  <script src="${asset("jquery-3.6.0.min.js")}"></script>
   <script src="https://cdn.jsdelivr.net/npm/bootstrap@5.3.8/dist/js/bootstrap.bundle.min.js" integrity="sha384-FKyoEForCGlyvwx9Hj09JcYn3nv7wiPVlz7YYwJrWVcXK/BmnVDxM+D2scQbITxI" crossorigin="anonymous"></script>
 
  If you are embedding views with <embed-view>, you should also embed the following script sources at the end of the <body> tag to make sure the content inside those views works:
 
- <script src="/static_assets/js/saltcorn-common.js"></script>
- <script src="/static_assets/js/saltcorn.js">
+ <script src="${asset("saltcorn-common.js")}"></script>
+ <script src="${asset("saltcorn.js")}"></script>
 `
-        );
-        const html = str.includes("```html")
-          ? str.split("```html")[1].split("```")[0]
-          : str;
-
-        if (this.yoloMode) {
-          await this.userActions.build_copilot_page_gen({
-            user: req?.user,
-            name: tool_call.input.name,
-            title: tool_call.input.title,
-            description: tool_call.input.description,
-            min_role: tool_call.input.min_role ?? 100,
-            html,
-          });
+          );
+          const html = str.includes("```html")
+            ? str.split("```html")[1].split("```")[0]
+            : str;
+
+          if (this.yoloMode) {
+            await this.userActions.build_copilot_page_gen({
+              user: req?.user,
+              name,
+              title,
+              description,
+              min_role: min_role ?? 100,
+              html,
+            });
+            return {
+              stop: true,
+              add_response: `Page ${name} created.`,
+            };
+          }
           return {
             stop: true,
-            add_response: `Page ${tool_call.input.name} created.`,
-          };
-        }
-        return {
-          stop: true,
-          add_response: iframe({
-            srcdoc: text_attr(html),
-            width: 500,
-            height: 800,
-          }),
-          add_system_prompt: `If the user asks you to regenerate the page,
+            add_response: iframe({
+              srcdoc: text_attr(html),
+              width: 500,
+              height: 800,
+            }),
+            add_system_prompt: `If the user asks you to regenerate the page,
           you must run the generate_page tool again. After running this tool
           you will be prompted to generate the html again. You should repeat
           the html from the previous answer except for the changes the user
           is requesting.`,
-          add_user_action: {
-            name: "build_copilot_page_gen",
-            type: "button",
-            label: "Save page " + tool_call.input.name,
-            input: { html },
-          },
-        };
+            add_user_action: {
+              name: "build_copilot_page_gen",
+              type: "button",
+              label: "Save page " + name,
+              input: { html },
+            },
+          };
+        }
       },
 
       /*renderToolCall({ phrase }, { req }) {
@@ -210,9 +308,9 @@ class GeneratePageSkill {
             },
             page_type: {
               description:
-                "The type of page to generate: a Marketing page if for promotional purposes, such as a landing page or a brouchure, with an appealing design. An Application page is simpler and an integrated part of the application",
+                "The type of page to generate. Use 'Layout page' for any page that is part of the application (dashboards, print pages, data pages, pages that embed views) — this creates a proper Saltcorn layout page. Use 'Marketing page' for public-facing promotional pages such as landing pages or brochures. Use 'Application page' only for standalone HTML pages that are part of the app but do not embed Saltcorn views.",
               type: "string",
-              enum: ["Marketing page", "Application page"],
+              enum: ["Layout page", "Marketing page", "Application page"],
             },
           },
         },

package/agent-skills/registry-editor.js

@@ -376,6 +376,18 @@ with both the entity type and name, and the new JSON definition as a string as a
               const trigger = Trigger.findOne({ name: input.entity_name });
               if (!trigger) return `trigger not found`;
               const value = trigger.toJson;
+              // For workflow triggers, attach all steps so the agent can see their configurations
+              if (trigger.action === "Workflow") {
+                const steps = await WorkflowStep.find({ trigger_id: trigger.id });
+                value.steps = steps.map((s) => ({
+                  id: s.id,
+                  name: s.name,
+                  action_name: s.action_name,
+                  initial_step: s.initial_step,
+                  next_step: s.next_step,
+                  configuration: s.configuration,
+                }));
+              }
               const schema = {
                 name: { type: "string", description: "trigger name" },
                 action: {
@@ -726,8 +738,9 @@ with both the entity type and name, and the new JSON definition as a string as a
                     table || table_name,
                   )?.id;
 
-                // Pre-check action before saving
-                if (tsNoTableName.action) {
+                // Pre-check action before saving (skip built-ins handled outside actions registry)
+                const builtinActions = new Set(["Workflow", "Multi-step action"]);
+                if (tsNoTableName.action && !builtinActions.has(tsNoTableName.action)) {
                   const action = getState().actions[tsNoTableName.action];
                   if (!action)
                     return `Action '${tsNoTableName.action}' not found`;

package/app-constructor/common.js

@@ -1,5 +1,11 @@
 const viewname = "Saltcorn AppConstructor (experimental)";
 
+const TaskType = Object.freeze({
+  PLUGIN: "plugin",
+  DATA_MODEL: "data_model",
+  FEATURE: "feature",
+});
+
 const tool_choice = (tool_name) => ({
   tool_choice: {
     type: "function",
@@ -9,4 +15,4 @@ const tool_choice = (tool_name) => ({
   },
 });
 
-module.exports = { viewname, tool_choice };
+module.exports = { viewname, tool_choice, TaskType };