@saltcorn/copilot 0.8.1 → 0.8.2

package/actions/generate-tables.js

@@ -163,6 +163,13 @@ class GenerateTables {
             },
           },
         },
+        reused_table_names: {
+          type: "array",
+          items: { type: "string" },
+          description:
+            "Names of existing tables that are already complete and require no changes. " +
+            "List them here so the caller knows which tables were reused as-is. Do NOT repeat their field definitions in the tables array.",
+        },
       },
     };
   }
@@ -202,6 +209,12 @@ class GenerateTables {
     want to add or update. The system will automatically add new fields and update the settings of
     existing fields — it will not recreate or drop the table.
 
+    ## Reused tables
+
+    If an existing table is used by the application as-is (no new fields needed), do NOT repeat it
+    in the tables array. Instead, add its name to the reused_table_names array. This tells the
+    system to include it in the schema diagram without attempting to modify it.
+
     If a user requests creating a table with certain fields and the table already exists, automatically add any missing fields to that table. Do not ask the user for confirmation or prompt them again—just proceed with the table update.
 
     If a table has a ForeignKey field that references another table which does not yet exist in the
@@ -451,6 +464,7 @@ const buildMermaidMarkup = (tables) => {
 };
 
 module.exports = GenerateTables;
+module.exports.buildMermaidMarkup = buildMermaidMarkup;
 
 /* todo
 

package/agent-skills/pagegen.js

@@ -52,20 +52,27 @@ class GeneratePageSkill {
   }
   get userActions() {
     return {
-      async build_copilot_page_gen({ user, name, title, description, html }) {
+      async build_copilot_page_gen({
+        user,
+        name,
+        title,
+        description,
+        html,
+        min_role = 100,
+      }) {
         const file = await File.from_contents(
           `${name}.html`,
           "text/html",
           html,
           user?.id,
-          100,
+          min_role
         );
 
         await Page.create({
           name,
           title,
           description,
-          min_role: 100,
+          min_role,
           layout: { html_file: file.path_to_serve },
         });
         setTimeout(() => getState().refresh_pages(), 200);
@@ -114,7 +121,7 @@ class GeneratePageSkill {
 
  <script src="/static_assets/js/saltcorn-common.js"></script>
  <script src="/static_assets/js/saltcorn.js">
-`,
+`
         );
         const html = str.includes("```html")
           ? str.split("```html")[1].split("```")[0]
@@ -126,6 +133,7 @@ class GeneratePageSkill {
             name: tool_call.input.name,
             title: tool_call.input.title,
             description: tool_call.input.description,
+            min_role: tool_call.input.min_role ?? 100,
             html,
           });
           return {
@@ -148,7 +156,7 @@ class GeneratePageSkill {
           add_user_action: {
             name: "build_copilot_page_gen",
             type: "button",
-            label: "Save page "+tool_call.input.name,
+            label: "Save page " + tool_call.input.name,
             input: { html },
           },
         };
@@ -163,7 +171,7 @@ class GeneratePageSkill {
           response.includes("Unable to provide HTML for this page")
         )
           return response;
-           if (
+        if (
           typeof response === "string" &&
           response.includes("The HTML code for the ")
         )
@@ -194,6 +202,12 @@ class GeneratePageSkill {
                 "A longer description that is not visible but appears in the page header and is indexed by search engines",
               type: "string",
             },
+            min_role: {
+              description:
+                "Minimum role required to access this page. Use 1 for admin-only, 40 for staff and above, 80 for logged-in users and above, 100 for public. Set this based on the intended audience described in the task.",
+              type: "integer",
+              enum: [1, 40, 80, 100],
+            },
             page_type: {
               description:
                 "The type of page to generate: a Marketing page if for promotional purposes, such as a landing page or a brouchure, with an appealing design. An Application page is simpler and an integrated part of the application",

package/agent-skills/registry-editor.js

@@ -6,6 +6,7 @@ const Field = require("@saltcorn/data/models/field");
 const User = require("@saltcorn/data/models/user");
 const Plugin = require("@saltcorn/data/models/plugin");
 const Role = require("@saltcorn/data/models/role");
+const File = require("@saltcorn/data/models/file");
 const WorkflowStep = require("@saltcorn/data/models/workflow_step");
 const { getState } = require("@saltcorn/data/db/state");
 
@@ -167,6 +168,7 @@ with both the entity type and name, and the new JSON definition as a string as a
                   "trigger",
                   "plugin",
                   "system-configuration-value",
+                  "file",
                   "type",
                 ],
               },
@@ -317,6 +319,11 @@ with both the entity type and name, and the new JSON definition as a string as a
               }
               return v;
             }
+            case "file": {
+              const file = await File.findOne({ filename: input.entity_name });
+              if (!file) return `file not found`;
+              return { filename: file.filename, min_role_read: file.min_role_read };
+            }
             case "plugin": {
               const plugin = await Plugin.findOne({ name: input.entity_name });
               if (!plugin) return `plugin not found`;
@@ -473,6 +480,7 @@ with both the entity type and name, and the new JSON definition as a string as a
                   "system-configuration-value",
                   "module-configuration",
                   "role",
+                  "file",
                 ],
               },
               entity_name: {
@@ -897,6 +905,13 @@ with both the entity type and name, and the new JSON definition as a string as a
                 await getState().refresh_roles();
                 return "Role created";
               }
+
+              case "file": {
+                const file = await File.findOne({ filename: input.entity_name });
+                if (!file) return `file not found: ${input.entity_name}`;
+                await file.set_role(entityValue.min_role_read);
+                return "Done";
+              }
             }
             return "Done";
           } catch (e) {

package/agent-skills/triggergen.js

@@ -33,11 +33,7 @@ class AnyActionSkill {
       `If the task requires several independent single-step actions (e.g. "mark complete" and "mark incomplete"), call this tool once per action — do NOT bundle them into one workflow.\n\n` +
       `**Navigation is a view concern:** If a task description says "return the user to X" or "navigate back", ` +
       `do NOT add a navigation step inside the trigger. Triggers only handle data operations. ` +
-      `Navigation (GoBack) is configured on the button in the list view, not inside the trigger itself.\n\n` +
-      `**Verification/confirmation emails after user registration:** Create a trigger with event "Insert" on the "users" table, ` +
-      `action_type "run_js_code". Do NOT use the send_email action — it will not work for verification. ` +
-      `Set the action_config "code" field to exactly:\n` +
-      `const { send_verification_email } = require("@saltcorn/data/models/email");\nawait send_verification_email(row, req);`
+      `Navigation (GoBack) is configured on the button in the list view, not inside the trigger itself.`
     );
   }
 

package/agent-skills/viewgen.js

@@ -23,6 +23,21 @@ const {
   getRelationPathsForPairs,
 } = require("../relation-paths");
 
+const collectViewLinks = (segment, out = []) => {
+  if (!segment || typeof segment !== "object") return out;
+  if (Array.isArray(segment)) {
+    segment.forEach((s) => collectViewLinks(s, out));
+    return out;
+  }
+  if (segment.type === "view_link" && segment.view) out.push(segment);
+  if (segment.above) collectViewLinks(segment.above, out);
+  if (segment.besides) collectViewLinks(segment.besides, out);
+  if (segment.contents) collectViewLinks(segment.contents, out);
+  if (Array.isArray(segment.tabs))
+    segment.tabs.forEach((t) => collectViewLinks(t?.contents, out));
+  return out;
+};
+
 const collectLayoutFieldNames = (segment, out = new Set()) => {
   if (!segment || typeof segment !== "object") return out;
   if (Array.isArray(segment)) {
@@ -129,7 +144,8 @@ class GenerateViewSkill {
       `(3) Write out the complete updated configuration JSON in full — every key from the existing config must be present, with only your targeted changes merged in.\n` +
       `(4) Call apply_view_config with that complete object. NEVER call apply_view_config before step (3) is finished. NEVER call it with only the name or a partial object — the configuration field is mandatory and must be the full merged result from step (3). Calling apply_view_config without a complete configuration is an error.\n\n` +
       `**Generating a new view that contains view_links or embedded views:**\n` +
-      `Call get_relation_paths once with all source_table/target_view pairs you need before constructing the layout.\n\n` +
+      `If the task or prompt mentions a viewlink, a link to another view, or a button that opens another view from a list row, that view_link column is REQUIRED — do not omit it. ` +
+      `You MUST call get_relation_paths with all source_table/target_view pairs before constructing the layout. Never skip this step when view_links are needed.\n\n` +
       `**Embedded view segment format (for Show layouts):**\n` +
       `  { "type": "view", "view": "<viewName>", "name": "<viewName>", "relation": "<from get_relation_paths>" }\n` +
       `Do NOT use blank text segments as placeholders — always use a real view segment with a relation string from get_relation_paths.\n\n` +
@@ -161,11 +177,16 @@ class GenerateViewSkill {
             error: `View "${name}" already exists. Use get_view_config and apply_view_config to update it.`,
           };
         const tableRow = table ? Table.findOne({ name: table }) : null;
-        const roleName = typeof min_role === "number" ? null : (min_role || "public");
+        const roleName =
+          typeof min_role === "number" ? null : min_role || "public";
         const resolvedRole =
           typeof min_role === "number"
             ? min_role
-            : ((getState().roles || []).find((r) => r.role === roleName) || { id: 100 }).id;
+            : (
+                (getState().roles || []).find((r) => r.role === roleName) || {
+                  id: 100,
+                }
+              ).id;
         await View.create({
           name,
           viewtemplate: viewpattern,
@@ -308,6 +329,21 @@ class GenerateViewSkill {
             });
             if (baseCfg?.columns) wfctx.columns = baseCfg.columns;
           }
+          if (viewpattern === "List" && wfctx.layout) {
+            // initial_config_all_fields never generates ViewLink columns — inject them from layout
+            const viewLinks = collectViewLinks(wfctx.layout);
+            const viewLinkColumns = viewLinks.map((seg) => ({
+              type: "ViewLink",
+              view: seg.view,
+              block: seg.block || false,
+              label: seg.view_label || "",
+              minRole: seg.minRole || 100,
+              ...(seg.relation ? { relation: seg.relation } : {}),
+              isFormula: seg.isFormula || {},
+            }));
+            if (viewLinkColumns.length > 0)
+              wfctx.columns = [...(wfctx.columns || []), ...viewLinkColumns];
+          }
           if (viewpattern === "Edit" && table) {
             const layoutFieldNames = collectLayoutFieldNames(wfctx.layout);
             const fields = table.fields || [];
@@ -331,7 +367,10 @@ class GenerateViewSkill {
               }
             }
             if (usersFkColumnsToAdd.length > 0)
-              wfctx.columns = [...(wfctx.columns || []), ...usersFkColumnsToAdd];
+              wfctx.columns = [
+                ...(wfctx.columns || []),
+                ...usersFkColumnsToAdd,
+              ];
             if (Object.keys(fixed).length > 0) wfctx.fixed = fixed;
             wfctx.destination_type = "Back to referer";
           }
@@ -398,12 +437,15 @@ class GenerateViewSkill {
             for (const field of form.fields) {
               if (prefilledFields.has(field.name)) continue;
               //TODO showIf
+              const isShowif = field.name.endsWith("_showif");
               properties[field.name] = {
                 description:
                   field.copilot_description ||
-                  `${field.label}.${
-                    field.sublabel ? ` ${field.sublabel}` : ""
-                  }`,
+                  (isShowif
+                    ? `${field.label}. The correct default is an empty string — leave it blank to always show this element. Only provide a JavaScript expression if the task description explicitly states that this element should be conditionally hidden based on a URL state variable or the current user. Never invent field names or copy examples.`
+                    : `${field.label}.${
+                        field.sublabel ? ` ${field.sublabel}` : ""
+                      }`),
                 ...fieldProperties(field),
               };
               if (!properties[field.name].type) {

package/app-constructor/requirements.js

@@ -44,7 +44,7 @@ const requirementsList = async (req) => {
       type: "CopilotConstructMgr",
       name: "requirement",
     },
-    { orderBy: "written_at" },
+    { orderBy: "written_at" }
   );
   const starFieldview = getState().types.Integer.fieldviews.show_star_rating;
 
@@ -67,43 +67,85 @@ const requirementsList = async (req) => {
                   class: "btn btn-outline-danger btn-sm",
                   onclick: `view_post("${viewname}", "del_req", {id:${r.id}})`,
                 },
-                i({ class: "fas fa-trash-alt" }),
+                i({ class: "fas fa-trash-alt" })
               ),
           },
         ],
-        rs,
+        rs
       ),
       button(
         {
           class: "btn btn-outline-danger mb-4",
           onclick: `view_post("${viewname}", "del_all_reqs")`,
         },
-        "Delete all",
-      ),
+        "Delete all"
+      )
     );
-  } else {
+  }
+
+  const generating = await MetaData.findOne({
+    type: "CopilotConstructMgr",
+    name: "generating_requirements",
+  });
+  if (generating) {
     return div(
       { class: "mt-2" },
-      p("No requirements found"),
-      button(
-        {
-          class: "btn btn-primary",
-          onclick: `press_store_button(this);view_post("${viewname}", "gen_reqs")`,
-        },
-        "Generate requirements",
+      p(
+        i({ class: "fas fa-spinner fa-spin me-2" }),
+        "Generating requirements, please wait..."
       ),
+      script(
+        domReady(`
+(function() {
+  const poll = () => {
+    view_post(${JSON.stringify(viewname)}, 'req_status', {}, (resp) => {
+      if (resp && !resp.generating) location.reload();
+      else setTimeout(poll, 3000);
+    });
+  };
+  setTimeout(poll, 3000);
+})();
+`)
+      )
     );
   }
+
+  return div(
+    { class: "mt-2", id: "req-gen-area" },
+    p("No requirements found"),
+    button(
+      { class: "btn btn-primary", onclick: `copilotGenReqs()` },
+      "Generate requirements"
+    ),
+    script(
+      domReady(`
+window.copilotGenReqs = () => {
+  document.getElementById('req-gen-area').innerHTML =
+    '<p><i class="fas fa-spinner fa-spin me-2"></i>Generating requirements, please wait...</p>';
+  view_post(${JSON.stringify(viewname)}, 'gen_reqs', {}, () => {});
+  const poll = () => {
+    view_post(${JSON.stringify(viewname)}, 'req_status', {}, (resp) => {
+      if (resp && !resp.generating) location.reload();
+      else setTimeout(poll, 3000);
+    });
+  };
+  setTimeout(poll, 3000);
+};
+`)
+    )
+  );
 };
 
-const gen_reqs = async (table_id, viewname, config, body, { req, res }) => {
-  const spec = await MetaData.findOne({
+const doGenReqs = async (spec, userId) => {
+  const generatingMd = await MetaData.create({
     type: "CopilotConstructMgr",
-    name: "spec",
+    name: "generating_requirements",
+    body: {},
+    user_id: userId,
   });
-  if (!spec) throw new Error("Specification not found");
-  const answer = await getState().functions.llm_generate.run(
-    `Generate the requirements for this application:
+  try {
+    const answer = await getState().functions.llm_generate.run(
+      `Generate the requirements for this application:
 
 Description: ${spec.body.description}
 Audience: ${spec.body.audience}
@@ -113,24 +155,44 @@ Visual style: ${spec.body.visual_style}
 
 Now use the make_requirements tool to list the requirements for this software application
 `,
-    {
-      tools: [requirements_tool],
-      ...tool_choice("make_requirements"),
-      systemPrompt:
-        "You are a project manager. The user wants to build an application, and you must analyse their application description",
-    },
-  );
+      {
+        tools: [requirements_tool],
+        ...tool_choice("make_requirements"),
+        systemPrompt:
+          "You are a project manager. The user wants to build an application, and you must analyse their application description",
+      }
+    );
+    const tc = answer.getToolCalls()[0];
+    for (const reqm of tc.input.requirements)
+      await MetaData.create({
+        type: "CopilotConstructMgr",
+        name: "requirement",
+        body: reqm,
+        user_id: userId,
+      });
+  } finally {
+    await generatingMd.delete();
+  }
+};
 
-  const tc = answer.getToolCalls()[0];
+const gen_reqs = async (table_id, viewname, config, body, { req, res }) => {
+  const spec = await MetaData.findOne({
+    type: "CopilotConstructMgr",
+    name: "spec",
+  });
+  if (!spec) throw new Error("Specification not found");
+  doGenReqs(spec, req.user?.id).catch((e) =>
+    console.error("gen_reqs error", e)
+  );
+  return { json: { success: true } };
+};
 
-  for (const reqm of tc.input.requirements)
-    await MetaData.create({
-      type: "CopilotConstructMgr",
-      name: "requirement",
-      body: reqm,
-      user_id: req.user?.id,
-    });
-  return { json: { reload_page: true } };
+const req_status = async (table_id, viewname, config, body, { req, res }) => {
+  const generating = await MetaData.findOne({
+    type: "CopilotConstructMgr",
+    name: "generating_requirements",
+  });
+  return { json: { generating: !!generating } };
 };
 
 const del_req = async (table_id, viewname, config, body, { req, res }) => {
@@ -151,6 +213,6 @@ const del_all_reqs = async (table_id, viewname, config, body, { req, res }) => {
   return { json: { reload_page: true } };
 };
 
-const req_routes = { gen_reqs, del_req, del_all_reqs };
+const req_routes = { gen_reqs, req_status, del_req, del_all_reqs };
 
 module.exports = { requirementsList, req_routes };

package/app-constructor/run_task.js

@@ -43,6 +43,7 @@ const runTask = async (md_id, req) => {
         { skill_type: "Generate trigger", yoloMode: true },
         { skill_type: "Generate View", yoloMode: true },
         { skill_type: "Install Plugin", yoloMode: true },
+        { skill_type: "Registry editor", yoloMode: true },
       ],
     },
   });
@@ -58,49 +59,68 @@ Important: The database schema is already fully implemented. Do NOT use generate
 
 Important: Some fields are non-stored (virtual) calculated fields — they have no database column and are computed on-the-fly by Saltcorn. Never include such fields in modify_row, SQL UPDATE statements, or recalculate_stored_fields calls. Only fields that exist as actual database columns (regular fields and stored calculated fields) can be written. If a calculated field needs updating, it will refresh automatically when the fields it depends on change.
 
-Important: The "users" table is built-in. Passwords are platform-managed — never add a password field to a view. Signup uses a built-in form, not an Edit view.
+Important: The "users" table is built-in. Passwords are platform-managed — never add a password field to a view. Signup uses the built-in page at /auth/signup, login at /auth/login. Do NOT create triggers for registration or email verification — the platform handles this natively.
+
+Important: On landing pages, place Log in / Create account buttons in no more than two locations (e.g. navbar and one hero call-to-action). Do not repeat them in a third "Get started" section or anywhere else. For links that take an already-authenticated user to their dashboard, use href="/" — not /auth/login.
+
+Important: Do not name any page or view "Admin dashboard" — that name is reserved by the Saltcorn platform. For pages intended for role 1 (admin), use a name like "App admin dashboard" or prefix it with the application name (e.g. "Law Firm admin dashboard").
+
+Important: When creating a page or view, always set min_role based on the intended audience: 1 for admin-only, 40 for staff and above, 80 for logged-in users and above, 100 for public. Never default to public (100) unless the page or view is explicitly intended for unauthenticated users (e.g. a landing page). A dashboard or view for clients/users is role 80, a staff page or view is role 40, an admin page or view is role 1.
+
+Important: Two-factor authentication (2FA/TOTP) is fully built into the platform. To configure it, call set_entity directly with entity_type "system-configuration-value" and entity_name "twofa_policy_by_role". The entity_definition must be the plain JSON object itself — for example: {"1": "Mandatory", "100": "Disabled"}. Do NOT wrap it in {"type": "json", "value": ...} or any other envelope. Read the current value first with get_entity and merge rather than overwrite. Do NOT create a workflow or trigger to do this.
+
+Important: To set a page as the home page for a role, call set_entity directly with entity_type "system-configuration-value" and entity_name "home_page_by_role". The value is a JSON object mapping role IDs to page names — Role IDs: public=100, user=80, staff=40, admin=1. The entity_definition must be the plain JSON object itself — for example: {"100": "landing", "80": "client_dashboard"}. Do NOT wrap it in {"type": "json", "value": ...} or any other envelope. Read the current value first with get_entity so you can merge rather than overwrite. Do NOT create a workflow or trigger to do this — use set_entity directly.
+
+Important: If the task description mentions adding a viewlink, linking rows to another view, or a button that opens another view from a list — that viewlink column MUST be present in the finished view. Do not skip it. Viewlinks require calling get_relation_paths first to obtain the relation string before generating the layout.
+
+Important: Before creating or updating any view or page that embeds, links to, or opens another view (including viewlinks, action buttons, and ajax_modal calls), call list_entities (entity_type "view") to get all existing view names. Only reference views that appear in that list — never invent a name or assume a view exists. If a view is not in the list, omit the reference entirely. Do the same for pages: call list_entities (entity_type "page") before linking to any page by name.
 
 Your task now is:
 ${md.body.description}`;
-  const safeReq = req?.__
-    ? req
-    : { ...req, __: (s) => s, user: req?.user };
+  const safeReq = req?.__ ? req : { ...req, __: (s) => s, user: req?.user };
 
   await md.update({ body: { ...md.body, status: "Running" } });
-  const actionres = await agent_action.runWithoutRow({
-    row: { prompt },
-    req: safeReq,
-    user: safeReq.user,
-  });
-  const run_id = actionres.json.run_id;
-  const run = await WorkflowRun.findOne({ id: run_id });
-  await agent_action.runWithoutRow({
-    row: {
-      prompt:
-        "Write a description of what you did, for the purposes of a progress report. Write 1-4 sentences. Do not use any tools or write any code",
-    },
-    req: safeReq,
-    run,
-    user: safeReq.user,
-  });
-  const lastInteraction =
-    run.context.interactions[run.context.interactions.length - 1];
-  const lastText =
-    typeof lastInteraction.content === "string"
-      ? lastInteraction.content
+  try {
+    const actionres = await agent_action.runWithoutRow({
+      row: { prompt },
+      req: safeReq,
+      user: safeReq.user,
+    });
+    const run_id = actionres.json.run_id;
+    const run = await WorkflowRun.findOne({ id: run_id });
+    await agent_action.runWithoutRow({
+      row: {
+        prompt:
+          "Write a description of what you did, for the purposes of a progress report. Write 1-4 sentences. Do not use any tools or write any code",
+      },
+      req: safeReq,
+      run,
+      user: safeReq.user,
+    });
+    const updatedRun = await WorkflowRun.findOne({ id: run_id });
+    const lastInteraction =
+      updatedRun.context.interactions[
+        updatedRun.context.interactions.length - 1
+      ];
+    const lastText =
+      typeof lastInteraction.content === "string"
+        ? lastInteraction.content
         : lastInteraction.content.text
         ? lastInteraction.content.text
-          : Array.isArray(lastInteraction.content)
-          ? lastInteraction.content[0].text
-          : lastInteraction.content;
-  await MetaData.create({
-    type: "CopilotConstructMgr",
-    name: "progress",
-    body: { text: lastText, run_id, task_id: md.id },
-    user_id: req?.user?.id,
-  });
-
-  await md.update({ body: { ...md.body, status: "Done", run_id } });
+        : Array.isArray(lastInteraction.content)
+        ? lastInteraction.content[0].text
+        : lastInteraction.content;
+    await MetaData.create({
+      type: "CopilotConstructMgr",
+      name: "progress",
+      body: { text: lastText, run_id, task_id: md.id },
+      user_id: req?.user?.id,
+    });
+    await md.update({ body: { ...md.body, status: "Done", run_id } });
+  } catch (e) {
+    await md.update({ body: { ...md.body, status: "To do" } });
+    throw e;
+  }
 };
 
 /**
@@ -128,9 +148,12 @@ const runNextTask = async (once = false) => {
   );
   const done = tasks.filter((t) => t.body.status === "Done");
   const done_names = new Set(done.map((t) => t.body.name));
+  const all_task_names = new Set(tasks.map((t) => t.body.name).filter(Boolean));
 
   const startable = todos.filter((t) =>
-    t.body.depends_on.every((nm) => done_names.has(nm))
+    (t.body.depends_on || []).every(
+      (nm) => done_names.has(nm) || !all_task_names.has(nm)
+    )
   );
 
   if (startable[0]) {
@@ -140,6 +163,13 @@ const runNextTask = async (once = false) => {
       : null;
     await runTask(startable[0].id, { user: taskUser, __: (s) => s });
     if (!once) await runNextTask();
+  } else if (!once) {
+    const settings = await MetaData.findOne({
+      type: "CopilotConstructMgr",
+      name: "settings",
+    });
+    if (settings?.body?.running)
+      await settings.update({ body: { ...settings.body, running: false } });
   }
 };