@saltcorn/copilot 0.8.0 → 0.8.2

package/app-constructor/run_task.js

@@ -8,41 +8,13 @@ const { findType } = require("@saltcorn/data/models/discovery");
 const { save_menu_items } = require("@saltcorn/data/models/config");
 const db = require("@saltcorn/data/db");
 const WorkflowRun = require("@saltcorn/data/models/workflow_run");
+const User = require("@saltcorn/data/models/user");
 const { viewname } = require("./common");
 
-const runNextTask = async (alwaysRun) => {
-  if (!alwaysRun) {
-    const settings = await MetaData.findOne({
-      type: "CopilotConstructMgr",
-      name: "settings",
-    });
-    if (!settings?.body?.running) return;
-  }
-  const tasks = await MetaData.find(
-    {
-      type: "CopilotConstructMgr",
-      name: "task",
-    },
-    { orderBy: "id" }
-  );
-  const todos = tasks.filter(
-    (t) => !t.body.status || t.body.status === "To do"
-  );
-  const done = tasks.filter((t) => t.body.status === "Done");
-  const done_names = new Set(done.map((t) => t.body.name));
-
-  const startable = todos.filter((t) =>
-    t.body.depends_on.every((nm) => done_names.has(nm))
-  );
-
-  if (startable[0]) {
-    console.log("running task", startable[0]);
-
-    return await runTask(startable[0].id, {});
-  }
-  //not done
-};
-
+/**
+ * @param {number} md_id - MetaData id of the task to run
+ * @param {object} req - Express request (may be empty `{}` from scheduler)
+ */
 const runTask = async (md_id, req) => {
   const md = await MetaData.findOne({
     id: md_id,
@@ -59,15 +31,19 @@ const runTask = async (md_id, req) => {
     when_trigger: "Never",
     configuration: {
       viewname: viewname,
-      sys_prompt: "",
+      sys_prompt:
+        "Each task creates exactly one view or one page. " +
+        "Never create more than one view or page per task, even if the description mentions multiple. " +
+        "Call the view or page tool exactly once and then stop.",
       prompt: "{{prompt}}",
       skills: [
         { skill_type: "Generate Page", yoloMode: true },
         { skill_type: "Database design", yoloMode: true },
         { skill_type: "Generate Workflow", yoloMode: true },
+        { skill_type: "Generate trigger", yoloMode: true },
         { skill_type: "Generate View", yoloMode: true },
         { skill_type: "Install Plugin", yoloMode: true },
-        { skill_type: "AppConstructor Context" },
+        { skill_type: "Registry editor", yoloMode: true },
       ],
     },
   });
@@ -83,46 +59,118 @@ Important: The database schema is already fully implemented. Do NOT use generate
 
 Important: Some fields are non-stored (virtual) calculated fields — they have no database column and are computed on-the-fly by Saltcorn. Never include such fields in modify_row, SQL UPDATE statements, or recalculate_stored_fields calls. Only fields that exist as actual database columns (regular fields and stored calculated fields) can be written. If a calculated field needs updating, it will refresh automatically when the fields it depends on change.
 
+Important: The "users" table is built-in. Passwords are platform-managed — never add a password field to a view. Signup uses the built-in page at /auth/signup, login at /auth/login. Do NOT create triggers for registration or email verification — the platform handles this natively.
+
+Important: On landing pages, place Log in / Create account buttons in no more than two locations (e.g. navbar and one hero call-to-action). Do not repeat them in a third "Get started" section or anywhere else. For links that take an already-authenticated user to their dashboard, use href="/" — not /auth/login.
+
+Important: Do not name any page or view "Admin dashboard" — that name is reserved by the Saltcorn platform. For pages intended for role 1 (admin), use a name like "App admin dashboard" or prefix it with the application name (e.g. "Law Firm admin dashboard").
+
+Important: When creating a page or view, always set min_role based on the intended audience: 1 for admin-only, 40 for staff and above, 80 for logged-in users and above, 100 for public. Never default to public (100) unless the page or view is explicitly intended for unauthenticated users (e.g. a landing page). A dashboard or view for clients/users is role 80, a staff page or view is role 40, an admin page or view is role 1.
+
+Important: Two-factor authentication (2FA/TOTP) is fully built into the platform. To configure it, call set_entity directly with entity_type "system-configuration-value" and entity_name "twofa_policy_by_role". The entity_definition must be the plain JSON object itself — for example: {"1": "Mandatory", "100": "Disabled"}. Do NOT wrap it in {"type": "json", "value": ...} or any other envelope. Read the current value first with get_entity and merge rather than overwrite. Do NOT create a workflow or trigger to do this.
+
+Important: To set a page as the home page for a role, call set_entity directly with entity_type "system-configuration-value" and entity_name "home_page_by_role". The value is a JSON object mapping role IDs to page names — Role IDs: public=100, user=80, staff=40, admin=1. The entity_definition must be the plain JSON object itself — for example: {"100": "landing", "80": "client_dashboard"}. Do NOT wrap it in {"type": "json", "value": ...} or any other envelope. Read the current value first with get_entity so you can merge rather than overwrite. Do NOT create a workflow or trigger to do this — use set_entity directly.
+
+Important: If the task description mentions adding a viewlink, linking rows to another view, or a button that opens another view from a list — that viewlink column MUST be present in the finished view. Do not skip it. Viewlinks require calling get_relation_paths first to obtain the relation string before generating the layout.
+
+Important: Before creating or updating any view or page that embeds, links to, or opens another view (including viewlinks, action buttons, and ajax_modal calls), call list_entities (entity_type "view") to get all existing view names. Only reference views that appear in that list — never invent a name or assume a view exists. If a view is not in the list, omit the reference entirely. Do the same for pages: call list_entities (entity_type "page") before linking to any page by name.
+
 Your task now is:
 ${md.body.description}`;
+  const safeReq = req?.__ ? req : { ...req, __: (s) => s, user: req?.user };
 
   await md.update({ body: { ...md.body, status: "Running" } });
-  const actionres = await agent_action.runWithoutRow({
-    row: { prompt },
-    req,
-    user: req?.user,
-  });
-  //console.log("actionres", actionres);
-  const run_id = actionres.json.run_id;
-  const run = await WorkflowRun.findOne({ id: run_id });
-  await agent_action.runWithoutRow({
-    row: {
-      prompt:
-        "Write a description of what you did, for the purposes of a progress report. Write 1-4 sentences. Do not use any tools or write any code",
-    },
-    req,
-    run,
-    user: req?.user,
-  });
-  const lastInteraction =
-    run.context.interactions[run.context.interactions.length - 1];
-  const lastText =
-    typeof lastInteraction.content === "string"
-      ? lastInteraction.content
-      : lastInteraction.content.text
+  try {
+    const actionres = await agent_action.runWithoutRow({
+      row: { prompt },
+      req: safeReq,
+      user: safeReq.user,
+    });
+    const run_id = actionres.json.run_id;
+    const run = await WorkflowRun.findOne({ id: run_id });
+    await agent_action.runWithoutRow({
+      row: {
+        prompt:
+          "Write a description of what you did, for the purposes of a progress report. Write 1-4 sentences. Do not use any tools or write any code",
+      },
+      req: safeReq,
+      run,
+      user: safeReq.user,
+    });
+    const updatedRun = await WorkflowRun.findOne({ id: run_id });
+    const lastInteraction =
+      updatedRun.context.interactions[
+        updatedRun.context.interactions.length - 1
+      ];
+    const lastText =
+      typeof lastInteraction.content === "string"
+        ? lastInteraction.content
+        : lastInteraction.content.text
         ? lastInteraction.content.text
         : Array.isArray(lastInteraction.content)
-          ? lastInteraction.content[0].text
-          : lastInteraction.content;
-  await MetaData.create({
-    type: "CopilotConstructMgr",
-    name: "progress",
-    body: { text: lastText, run_id, task_id: md.id },
-    user_id: req?.user?.id,
-  });
+        ? lastInteraction.content[0].text
+        : lastInteraction.content;
+    await MetaData.create({
+      type: "CopilotConstructMgr",
+      name: "progress",
+      body: { text: lastText, run_id, task_id: md.id },
+      user_id: req?.user?.id,
+    });
+    await md.update({ body: { ...md.body, status: "Done", run_id } });
+  } catch (e) {
+    await md.update({ body: { ...md.body, status: "To do" } });
+    throw e;
+  }
+};
+
+/**
+ * Run the next startable task
+ * @param {boolean} [once=false] - true: run one task and stop, false: iterate all tasks
+ */
+const runNextTask = async (once = false) => {
+  if (!once) {
+    const settings = await MetaData.findOne({
+      type: "CopilotConstructMgr",
+      name: "settings",
+    });
+    if (!settings?.body?.running) return;
+  }
+  const tasks = await MetaData.find(
+    {
+      type: "CopilotConstructMgr",
+      name: "task",
+    },
+    { orderBy: "id" }
+  );
+  if (tasks.some((t) => t.body.status === "Running")) return;
+  const todos = tasks.filter(
+    (t) => !t.body.status || t.body.status === "To do"
+  );
+  const done = tasks.filter((t) => t.body.status === "Done");
+  const done_names = new Set(done.map((t) => t.body.name));
+  const all_task_names = new Set(tasks.map((t) => t.body.name).filter(Boolean));
 
-  //console.log("run", run);
-  await md.update({ body: { ...md.body, status: "Done", run_id } });
+  const startable = todos.filter((t) =>
+    (t.body.depends_on || []).every(
+      (nm) => done_names.has(nm) || !all_task_names.has(nm)
+    )
+  );
+
+  if (startable[0]) {
+    console.log("running task", startable[0]);
+    const taskUser = startable[0].user_id
+      ? await User.findOne({ id: startable[0].user_id })
+      : null;
+    await runTask(startable[0].id, { user: taskUser, __: (s) => s });
+    if (!once) await runNextTask();
+  } else if (!once) {
+    const settings = await MetaData.findOne({
+      type: "CopilotConstructMgr",
+      name: "settings",
+    });
+    if (settings?.body?.running)
+      await settings.update({ body: { ...settings.body, running: false } });
+  }
 };
 
 module.exports = { runTask, runNextTask };

package/app-constructor/schema.js

@@ -39,6 +39,7 @@ const { viewname, tool_choice } = require("./common");
 const { requirements_tool } = require("./tools");
 const { saltcorn_description, existing_tables_list } = require("./prompts");
 const GenerateTables = require("../actions/generate-tables");
+const { buildMermaidMarkup } = GenerateTables;
 const GenerateTablesSkill = require("../agent-skills/database-design");
 
 const showSchema = async (req) => {
@@ -48,15 +49,94 @@ const showSchema = async (req) => {
   });
 
   if (schema) {
-    const preview = GenerateTables.render_html(
-      { tables: schema.body.tables },
-      true
+    const newTableDefs = schema.body.tables || [];
+    const newTableInstances = GenerateTables.process_tables(newTableDefs);
+    const reusedMd = await MetaData.findOne({
+      type: "CopilotConstructMgr",
+      name: "reused_schema",
+    });
+    const reusedNames = reusedMd?.body?.table_names || [];
+    const reusedInstances = reusedNames
+      .map((n) => Table.findOne({ name: n }))
+      .filter(Boolean);
+    const allTables = [...newTableInstances, ...reusedInstances];
+    const mmdia = buildMermaidMarkup(allTables);
+    const implemented = !!schema.body.implemented;
+
+    const newNames = newTableDefs.map((t) => t.table_name).filter(Boolean);
+
+    const colorMap = Object.fromEntries([
+      ...newNames.map((n) => [n, "#198754"]),
+      ...reusedNames.map((n) => [n, "#6c757d"]),
+    ]);
+    const colorScript = script(
+      domReady(`
+  const colors = ${JSON.stringify(colorMap)};
+  const pre = document.querySelector('.schema-mermaid');
+  if (!pre) return;
+
+  const doRender = () => {
+    mermaid.run({ nodes: [pre], suppressErrors: true, postRenderCallback: () => {
+      for (const g of pre.querySelectorAll('g[id^="entity-"]')) {
+        const name = g.id.replace(/^entity-/, '').replace(/-\\d+$/, '');
+        const color = colors[name];
+        if (color) {
+          const p = g.querySelector('path');
+          if (p) p.setAttribute('fill', color);
+        }
+      }
+      for (const el of pre.querySelectorAll('g.label.name .nodeLabel')) {
+        el.style.color = 'white';
+        el.style.fontWeight = 'bold';
+      }
+    }});
+  };
+
+  // Defer render until tab is visible, then colorize nodes.
+  const pane = pre.closest('.tab-pane');
+  if (pane && !pane.classList.contains('active')) {
+    const link = document.querySelector('[href="#' + pane.id + '"]');
+    if (link) link.addEventListener('shown.bs.tab', doRender, { once: true });
+    else {
+      const o = new MutationObserver(() => {
+        if (pane.classList.contains('active')) { o.disconnect(); doRender(); }
+      });
+      o.observe(pane, { attributes: true, attributeFilter: ['class'] });
+    }
+  } else doRender();
+`)
+    );
+
+    const legend = div(
+      { class: "mt-3 d-flex flex-wrap gap-3 align-items-start" },
+      newNames.length
+        ? div(
+            { class: "d-flex flex-wrap align-items-center gap-1" },
+            span(
+              { class: "me-1 text-muted small" },
+              implemented ? "Was created:" : "Will be created:"
+            ),
+            ...newNames.map((n) => span({ class: "badge bg-success" }, n))
+          )
+        : "",
+      reusedNames.length
+        ? div(
+            { class: "d-flex flex-wrap align-items-center gap-1" },
+            span(
+              { class: "me-1 text-muted small" },
+              implemented ? "Already existed:" : "Already exists:"
+            ),
+            ...reusedNames.map((n) => span({ class: "badge bg-secondary" }, n))
+          )
+        : ""
     );
 
     return div(
       { class: "mt-2" },
-      preview,
-      !schema.body.implemented &&
+      pre({ class: "schema-mermaid" }, mmdia),
+      colorScript,
+      legend,
+      !implemented &&
         div(
           { class: "mb-4 d-block mt-3" },
           button(
@@ -75,37 +155,71 @@ const showSchema = async (req) => {
           )
         )
     );
-  } else {
+  }
+
+  const generating = await MetaData.findOne({
+    type: "CopilotConstructMgr",
+    name: "generating_schema",
+  });
+  if (generating) {
     return div(
       { class: "mt-2" },
-      p("Schema not found"),
-      button(
-        {
-          class: "btn btn-primary",
-          onclick: `press_store_button(this);view_post("${viewname}", "gen_schema")`,
-        },
-        "Generate schema"
+      p(
+        i({ class: "fas fa-spinner fa-spin me-2" }),
+        "Generating schema, please wait..."
+      ),
+      script(
+        domReady(`
+const poll = () => {
+  view_post(${JSON.stringify(viewname)}, 'schema_status', {}, (resp) => {
+    if (resp && !resp.generating) location.reload();
+    else setTimeout(poll, 3000);
+  });
+};
+setTimeout(poll, 3000);
+`)
       )
     );
   }
+
+  return div(
+    { class: "mt-2", id: "schema-gen-area" },
+    p("Schema not found"),
+    button(
+      { class: "btn btn-primary", onclick: `copilotGenSchema()` },
+      "Generate schema"
+    ),
+    script(
+      domReady(`
+window.copilotGenSchema = () => {
+  document.getElementById('schema-gen-area').innerHTML =
+    '<p><i class="fas fa-spinner fa-spin me-2"></i>Generating schema, please wait...</p>';
+  view_post(${JSON.stringify(viewname)}, 'gen_schema', {}, () => {});
+  const poll = () => {
+    view_post(${JSON.stringify(viewname)}, 'schema_status', {}, (resp) => {
+      if (resp && !resp.generating) location.reload();
+      else setTimeout(poll, 3000);
+    });
+  };
+  setTimeout(poll, 3000);
+};
+`)
+    )
+  );
 };
 
-const gen_schema = async (table_id, viewname, config, body, { req, res }) => {
-  const spec = await MetaData.findOne({
-    type: "CopilotConstructMgr",
-    name: "spec",
-  });
-  if (!spec) throw new Error("Specification not found");
-  const rs = await MetaData.find({
+const doGenSchema = async (spec, rs, userId) => {
+  const generatingMd = await MetaData.create({
     type: "CopilotConstructMgr",
-    name: "requirement",
+    name: "generating_schema",
+    body: {},
+    user_id: userId,
   });
-  if (!rs.length) throw new Error("No requirements found");
-
-  const databaseDesignTool = new GenerateTablesSkill({}).provideTools();
-  const existing_tables = await Table.find({});
-  const answer = await getState().functions.llm_generate.run(
-    `Generate the database schema for this application:
+  try {
+    const databaseDesignTool = new GenerateTablesSkill({}).provideTools();
+    const existing_tables = await Table.find({});
+    const answer = await getState().functions.llm_generate.run(
+      `Generate the database schema for this application:
 
 Description: ${spec.body.description}
 Audience: ${spec.body.audience}
@@ -113,7 +227,7 @@ Core features: ${spec.body.core_features}
 Out of scope: ${spec.body.out_of_scope}
 Visual style: ${spec.body.visual_style}
 
-These are the requirements of the application: 
+These are the requirements of the application:
 
 ${rs.map((r) => `* ${r.body.requirement}`).join("\n")}
 
@@ -123,6 +237,10 @@ ${existing_tables_list(existing_tables)}
 
 Design a complete database schema that covers ALL requirements listed above. Every distinct entity in the application must have its own table. Do not produce a minimal or partial schema — all tables needed to implement every requirement must be included in this single call. Do not leave any tables for a later step.
 
+The tables listed above already exist in the database. Do NOT modify or extend them — treat them as fixed. Handle them as follows:
+- If an existing table is already complete and used as-is: add its name to reused_table_names. Do NOT define its fields again in the tables array.
+- New tables not yet in the database: include them in the tables array with all their fields as usual.
+
 For every field that must be unique (e.g. unique email, unique slug, unique combination keys expressed as individual unique fields), set unique=true on that field.
 For every field that must not be empty, set not_null=true.
 Do NOT leave uniqueness or required constraints for a later step — express them fully in this schema.
@@ -130,34 +248,78 @@ Do NOT leave uniqueness or required constraints for a later step — express the
 Note: ownership configuration (automatically populating a FK-to-users field from the logged-in user) is a VIEW-level concern and cannot be expressed in the schema. Do not attempt to annotate fields as "ownership fields" here — simply define the foreign key field normally. Ownership will be configured when the Edit views are generated.
 
 Now use the ${
-      databaseDesignTool.function.name
-    } tool to generate the complete database schema for this software application
+        databaseDesignTool.function.name
+      } tool to generate the complete database schema for this software application
 `,
-    {
-      tools: [databaseDesignTool],
-      ...tool_choice(databaseDesignTool.function.name),
-      systemPrompt:
-        "You are a database designer. The user wants to build an application, and you must analyse their application description and requirements and design a complete schema. Every entity needed by any requirement must have its own table. Never produce a partial schema.",
-    }
-  );
+      {
+        tools: [databaseDesignTool],
+        ...tool_choice(databaseDesignTool.function.name),
+        systemPrompt:
+          "You are a database designer. The user wants to build an application, and you must analyse their application description and requirements and design a complete schema. Every entity needed by any requirement must have its own table. Never produce a partial schema.",
+      }
+    );
+
+    const tc = answer.getToolCalls()[0];
 
-  const tc = answer.getToolCalls()[0];
+    const noNewTables = !tc.input.tables || tc.input.tables.length === 0;
+    await MetaData.create({
+      type: "CopilotConstructMgr",
+      name: "schema",
+      body: { tables: tc.input.tables || [], implemented: noNewTables },
+      user_id: userId,
+    });
 
-  await MetaData.create({
+    const reusedNames = tc.input.reused_table_names || [];
+    if (reusedNames.length) {
+      await MetaData.create({
+        type: "CopilotConstructMgr",
+        name: "reused_schema",
+        body: { table_names: reusedNames },
+        user_id: userId,
+      });
+    }
+  } finally {
+    await generatingMd.delete();
+  }
+};
+
+const gen_schema = async (table_id, viewname, config, body, { req, res }) => {
+  const spec = await MetaData.findOne({
     type: "CopilotConstructMgr",
-    name: "schema",
-    body: { tables: tc.input.tables, implemented: false },
-    user_id: req.user?.id,
+    name: "spec",
   });
-  return { json: { reload_page: true } };
+  if (!spec) throw new Error("Specification not found");
+  const rs = await MetaData.find({
+    type: "CopilotConstructMgr",
+    name: "requirement",
+  });
+  if (!rs.length) throw new Error("No requirements found");
+
+  doGenSchema(spec, rs, req.user?.id).catch((e) =>
+    console.error("gen_schema error", e)
+  );
+  return { json: { success: true } };
 };
 
-const del_schema = async (table_id, viewname, config, body, { req, res }) => {
-  const rs = await MetaData.find({
+const schema_status = async (
+  table_id,
+  viewname,
+  config,
+  body,
+  { req, res }
+) => {
+  const generating = await MetaData.findOne({
     type: "CopilotConstructMgr",
-    name: "schema",
+    name: "generating_schema",
   });
-  for (const r of rs) await r.delete();
+  return { json: { generating: !!generating } };
+};
+
+const del_schema = async (table_id, viewname, config, body, { req, res }) => {
+  for (const name of ["schema", "reused_schema", "generating_schema"]) {
+    const rs = await MetaData.find({ type: "CopilotConstructMgr", name });
+    for (const r of rs) await r.delete();
+  }
   return { json: { reload_page: true } };
 };
 
@@ -174,13 +336,29 @@ const implement_schema = async (
   });
 
   const { apply_copilot_tables } = new GenerateTablesSkill({}).userActions;
-  await apply_copilot_tables({ tables: md.body.tables, user: req.user });
+  const existingNames = new Set((await Table.find({})).map((t) => t.name));
+  const newTables = md.body.tables.filter((t) => {
+    if (existingNames.has(t.name)) {
+      getState().log(
+        2,
+        `AppConstructor: skipping table "${t.name}" — already exists in database`
+      );
+      return false;
+    }
+    return true;
+  });
+  await apply_copilot_tables({ tables: newTables, user: req.user });
   md.body.implemented = true;
   await md.update({ body: md.body });
 
   return { json: { reload_page: true } };
 };
 
-const schema_routes = { gen_schema, del_schema, implement_schema };
+const schema_routes = {
+  gen_schema,
+  schema_status,
+  del_schema,
+  implement_schema,
+};
 
 module.exports = { showSchema, schema_routes };