@saltcorn/copilot 0.8.0 → 0.8.1

package/agent-skills/viewgen.js

@@ -1,7 +1,10 @@
 const Table = require("@saltcorn/data/models/table");
 const View = require("@saltcorn/data/models/view");
 const { fieldProperties } = require("../common");
-const { initial_config_all_fields } = require("@saltcorn/data/plugin-helper");
+const {
+  initial_config_all_fields,
+  build_schema_data,
+} = require("@saltcorn/data/plugin-helper");
 const { getState } = require("@saltcorn/data/db/state");
 const {
   div,
@@ -14,6 +17,11 @@ const {
   text_attr,
 } = require("@saltcorn/markup/tags");
 const builderGen = require("../builder-gen");
+const {
+  RELATION_PATH_DOC,
+  GET_RELATION_PATHS_FUNCTION,
+  getRelationPathsForPairs,
+} = require("../relation-paths");
 
 const collectLayoutFieldNames = (segment, out = new Set()) => {
   if (!segment || typeof segment !== "object") return out;
@@ -112,14 +120,34 @@ class GenerateViewSkill {
 
   async systemPrompt() {
     return (
-      `If the user asks to generate a view, use the generate_view tool to enter ` +
-      `a view generation mode. The tool call only requires high-level details to start this sequence.\n` +
-      `The Edit viewtemplate serves both create (no id in state) and edit (id in state) — one view covers both modes.`
+      `If the user asks to generate a view, use the generate_view tool — but ONLY if the view does not already exist. ` +
+      `If a view with that name already exists, do NOT call generate_view — doing so will create a duplicate. Instead follow the modification sequence below.\n` +
+      `The Edit viewtemplate serves both create (no id in state) and edit (id in state) — one view covers both.\n\n` +
+      `**Modifying an existing view — required sequence:**\n` +
+      `(1) Call get_view_config to fetch the current configuration.\n` +
+      `(2) Only if you are adding view_link columns or embedded view (type "view") segments: call get_relation_paths once with all the source_table/target_view pairs you need. For changes that don't involve linking or embedding views (e.g. adding a field, changing a label), skip this step.\n` +
+      `(3) Write out the complete updated configuration JSON in full — every key from the existing config must be present, with only your targeted changes merged in.\n` +
+      `(4) Call apply_view_config with that complete object. NEVER call apply_view_config before step (3) is finished. NEVER call it with only the name or a partial object — the configuration field is mandatory and must be the full merged result from step (3). Calling apply_view_config without a complete configuration is an error.\n\n` +
+      `**Generating a new view that contains view_links or embedded views:**\n` +
+      `Call get_relation_paths once with all source_table/target_view pairs you need before constructing the layout.\n\n` +
+      `**Embedded view segment format (for Show layouts):**\n` +
+      `  { "type": "view", "view": "<viewName>", "name": "<viewName>", "relation": "<from get_relation_paths>" }\n` +
+      `Do NOT use blank text segments as placeholders — always use a real view segment with a relation string from get_relation_paths.\n\n` +
+      RELATION_PATH_DOC
     );
   }
 
   get userActions() {
     return {
+      async build_copilot_view_update({ name, configuration }) {
+        const existingView = View.findOne({ name });
+        if (!existingView) return { error: `View "${name}" not found` };
+        await View.update({ configuration }, existingView.id);
+        setTimeout(() => getState().refresh_views(), 200);
+        return {
+          notify: `View updated: <a target="_blank" href="/view/${name}">${name}</a>`,
+        };
+      },
       async build_copilot_view_gen({
         wfctx,
         name,
@@ -127,14 +155,23 @@ class GenerateViewSkill {
         table,
         min_role,
       }) {
-        const normalizedRole = min_role || "public";
+        const existing = View.findOne({ name });
+        if (existing)
+          return {
+            error: `View "${name}" already exists. Use get_view_config and apply_view_config to update it.`,
+          };
         const tableRow = table ? Table.findOne({ name: table }) : null;
+        const roleName = typeof min_role === "number" ? null : (min_role || "public");
+        const resolvedRole =
+          typeof min_role === "number"
+            ? min_role
+            : ((getState().roles || []).find((r) => r.role === roleName) || { id: 100 }).id;
         await View.create({
           name,
           viewtemplate: viewpattern,
           table_id: tableRow?.id,
           table: tableRow,
-          min_role: { admin: 1, public: 100, user: 80 }[normalizedRole],
+          min_role: resolvedRole,
           configuration: wfctx,
         });
         const vt = getState().viewtemplates[viewpattern];
@@ -199,12 +236,12 @@ class GenerateViewSkill {
       },
     };
 
-    return {
+    const generateViewTool = {
       type: "function",
       function: {
         name: "generate_view",
         description:
-          "Generate a view by supplying high-level details. This will trigger a view generation sequence",
+          "Generate a NEW view by supplying high-level details. Only call this for views that do not yet exist — if the view already exists, use get_view_config + apply_view_config instead.",
         parameters,
       },
       process: async (input) => {
@@ -264,7 +301,9 @@ class GenerateViewSkill {
             chat
           );
           if (table && viewpattern !== "Filter") {
-            const baseCfg = await initial_config_all_fields(false)({
+            // isEdit=true: FK fields get Field+select columns; false gives JoinField (display-only)
+            const isEditView = viewpattern === "Edit";
+            const baseCfg = await initial_config_all_fields(isEditView)({
               table_id: table.id,
             });
             if (baseCfg?.columns) wfctx.columns = baseCfg.columns;
@@ -273,14 +312,26 @@ class GenerateViewSkill {
             const layoutFieldNames = collectLayoutFieldNames(wfctx.layout);
             const fields = table.fields || [];
             const fixed = {};
+            const usersFkColumnsToAdd = [];
             for (const f of fields) {
               if (f.primary_key || f.calculated) continue;
-              if (layoutFieldNames.has(f.name)) continue;
               if (f.type === "Key" && f.reftable_name === "users") {
-                fixed[`preset_${f.name}`] = "LoggedIn";
-                fixed[`_block_${f.name}`] = true;
+                if (layoutFieldNames.has(f.name)) {
+                  // Explicitly placed in layout — add a select column so getForm renders it
+                  usersFkColumnsToAdd.push({
+                    field_name: f.name,
+                    type: "Field",
+                    fieldview: "select",
+                    state_field: true,
+                  });
+                } else {
+                  fixed[`preset_${f.name}`] = "LoggedIn";
+                  fixed[`_block_${f.name}`] = true;
+                }
               }
             }
+            if (usersFkColumnsToAdd.length > 0)
+              wfctx.columns = [...(wfctx.columns || []), ...usersFkColumnsToAdd];
             if (Object.keys(fixed).length > 0) wfctx.fixed = fixed;
             wfctx.destination_type = "Back to referer";
           }
@@ -388,13 +439,18 @@ class GenerateViewSkill {
                 },
               }
             );
-            const tc = answer.getToolCalls()[0];
-            await getState().functions.llm_add_message.run(
-              "tool_response",
-              { type: "text", value: "Details provided" },
-              { chat, tool_call: tc }
-            );
-            Object.assign(wfctx, tc.input);
+            const tc =
+              typeof answer?.getToolCalls === "function"
+                ? answer.getToolCalls()[0]
+                : null;
+            if (tc) {
+              await getState().functions.llm_add_message.run(
+                "tool_response",
+                { type: "text", value: "Details provided" },
+                { chat, tool_call: tc }
+              );
+              Object.assign(wfctx, tc.input);
+            }
           }
         }
         const roleName = tool_call.input.min_role || "public";
@@ -402,6 +458,13 @@ class GenerateViewSkill {
         const min_role = rolesState
           ? (rolesState.find((r) => r.role === roleName) || { id: 100 }).id
           : { admin: 1, public: 100, user: 80 }[roleName] ?? 100;
+        const existingView = View.findOne({ name: tool_call.input.name });
+        if (existingView) {
+          return {
+            stop: true,
+            add_response: `Error: view "${tool_call.input.name}" already exists. Do NOT call generate_view again — use get_view_config to inspect the current configuration and apply_view_config to update it.`,
+          };
+        }
         const view = new View({
           name: tool_call.input.name,
           viewtemplate: tool_call.input.viewpattern,
@@ -441,6 +504,127 @@ class GenerateViewSkill {
         };
       },
     };
+
+    const getViewConfigTool = {
+      type: "function",
+      function: {
+        name: "get_view_config",
+        description:
+          "Retrieve the current configuration of an existing view. " +
+          "Call this first to inspect the layout before calling apply_view_config to save changes. " +
+          "Returns the full configuration JSON and the viewtemplate name.",
+        parameters: {
+          type: "object",
+          required: ["name"],
+          properties: {
+            name: {
+              description: "The name of the existing view to inspect.",
+              type: "string",
+            },
+          },
+        },
+      },
+      process: async ({ name }) => {
+        const existingView = View.findOne({ name });
+        if (!existingView)
+          return `View "${name}" not found. Use generate_view to create a new view instead.`;
+        return (
+          `Current configuration of view "${name}" (viewtemplate: ${existingView.viewtemplate}):\n` +
+          JSON.stringify(existingView.configuration, null, 2)
+        );
+      },
+    };
+
+    const applyViewConfigTool = {
+      type: "function",
+      function: {
+        name: "apply_view_config",
+        description:
+          "Save an updated configuration to an existing view. " +
+          "STRICT PRECONDITION: you must have already called get_view_config AND written out the complete merged configuration JSON before calling this tool. " +
+          "Do NOT call this tool as a placeholder or before the configuration is fully constructed. " +
+          "Calling this tool without a complete configuration object is always wrong and will fail.",
+        parameters: {
+          type: "object",
+          required: ["name", "configuration"],
+          properties: {
+            name: {
+              description: "The name of the existing view to update.",
+              type: "string",
+            },
+            configuration: {
+              type: "object",
+              description:
+                "REQUIRED. The complete updated configuration object — every key from the existing config preserved, with only your changes merged in. " +
+                "You MUST have the full object written out before calling this tool. " +
+                "Passing null, an empty object, or a partial object (e.g. only the name) is always wrong and will return an error.",
+            },
+          },
+        },
+      },
+      process: async ({ name, configuration }) => {
+        const existingView = View.findOne({ name });
+        if (!existingView) return `View "${name}" not found.`;
+        if (!configuration || typeof configuration !== "object")
+          return (
+            `ERROR: configuration is missing. ` +
+            `You must call get_view_config first, merge your changes into the full existing configuration, then call apply_view_config again with the complete configuration object.`
+          );
+        return { name, configuration, view_id: existingView.id };
+      },
+      postProcess: async ({ tool_call, req }) => {
+        const { name, configuration } = tool_call.input;
+        const existingView = View.findOne({ name });
+        if (!existingView)
+          return { stop: true, add_response: `View "${name}" not found.` };
+        if (!configuration || typeof configuration !== "object")
+          return {
+            stop: true,
+            add_response:
+              `apply_view_config called for "${name}" without a configuration object. ` +
+              `Call get_view_config first, merge your changes into the full existing configuration, then call apply_view_config again with the complete configuration.`,
+          };
+        const cfg = configuration;
+
+        if (this.yoloMode) {
+          await View.update({ configuration: cfg }, existingView.id);
+          setTimeout(() => getState().refresh_views(), 200);
+          return { stop: true, add_response: `View ${name} updated.` };
+        }
+        return {
+          stop: true,
+          add_response: pre(JSON.stringify(cfg, null, 2)),
+          add_user_action: {
+            name: "build_copilot_view_update",
+            type: "button",
+            label: "Save updated view " + name,
+            input: { name, configuration: cfg },
+          },
+        };
+      },
+    };
+
+    const getRelationPathsTool = {
+      type: "function",
+      function: GET_RELATION_PATHS_FUNCTION,
+      process: async ({ pairs }) => {
+        const schemaData = await build_schema_data();
+        const sections = getRelationPathsForPairs(pairs || [], schemaData);
+        return (
+          sections.join("\n\n") +
+          `\n\nFor each pair, set the "relation" property to one of the strings listed above.\n` +
+          `Pick by type: ChildList = multiple child rows, ParentShow = single parent, OneToOneShow = unique child. ` +
+          `If multiple paths of the same type exist, choose the one whose FK field name best matches the task. Prefer shorter paths.`
+        );
+      },
+    };
+
+    return [
+      generateViewTool,
+      getViewConfigTool,
+      applyViewConfigTool,
+      getRelationPathsTool,
+    ];
   };
 }
 

package/app-constructor/prompts.js

@@ -42,12 +42,14 @@ const existing_tables_list = (tables) => {
   tables.forEach((table) => {
     const fieldLines = table.fields.map(
       (f) =>
-        `  * ${f.name} with type: ${f.pretty_type}.${f.description ? ` ${f.description}` : ""}`,
+        `  * ${f.name} with type: ${f.pretty_type}.${
+          f.description ? ` ${f.description}` : ""
+        }`
     );
     tableLines.push(
       `${table.name}${
         table.description ? `: ${table.description}.` : "."
-      } Contains the following fields:\n${fieldLines.join("\n")}`,
+      } Contains the following fields:\n${fieldLines.join("\n")}`
     );
   });
   return `The database already contains the following tables: 
@@ -55,4 +57,64 @@ const existing_tables_list = (tables) => {
 ${tableLines.join("\n\n")}`;
 };
 
-module.exports = { saltcorn_description, existing_tables_list };
+const existing_entities_list = ({ views, triggers, pages, tableById = {} }) => {
+  const sections = [];
+  if (views.length)
+    sections.push(
+      `The following views are already implemented — do NOT plan tasks to create them:\n` +
+        views
+          .map((v) => {
+            const tablePart =
+              v.table?.name ||
+              (v.table_id && tableById[v.table_id]) ||
+              v.exttable_name;
+            return `- ${v.name} (${v.viewtemplate}${
+              tablePart ? ` on ${tablePart}` : ""
+            })`;
+          })
+          .join("\n")
+    );
+  if (triggers.length)
+    sections.push(
+      `The following triggers are already implemented — do NOT plan tasks to create them:\n` +
+        triggers
+          .map(
+            (t) =>
+              `- ${t.name} (${t.action}${
+                t.when_trigger ? `, ${t.when_trigger}` : ""
+              })`
+          )
+          .join("\n")
+    );
+  if (pages.length)
+    sections.push(
+      `The following pages are already implemented — do NOT plan tasks to create them:\n` +
+        pages.map((p) => `- ${p.name}`).join("\n")
+    );
+  return sections.join("\n\n");
+};
+
+const available_plugins_list = (storePlugins, installedNames) => {
+  const uninstalled = storePlugins.filter((p) => !installedNames.has(p.name));
+  if (!uninstalled.length) return "";
+  const lines = uninstalled.map((p) => {
+    let line = `### ${p.name}`;
+    if (p.description) line += `\n${p.description}`;
+    if (p.contents) line += `\n${p.contents}`;
+    return line;
+  });
+  return (
+    `The following plugins are available in the Saltcorn store but not yet installed. ` +
+    `If a task requires functionality provided by one of these plugins (e.g. a specific view template, field type, or action), ` +
+    `include an explicit "Install plugin <name>" task before it with the exact plugin name as listed here. ` +
+    `The executor will use that name directly without needing to look it up.\n\n` +
+    lines.join("\n\n")
+  );
+};
+
+module.exports = {
+  saltcorn_description,
+  existing_tables_list,
+  existing_entities_list,
+  available_plugins_list,
+};

package/app-constructor/run_task.js

@@ -8,41 +8,13 @@ const { findType } = require("@saltcorn/data/models/discovery");
 const { save_menu_items } = require("@saltcorn/data/models/config");
 const db = require("@saltcorn/data/db");
 const WorkflowRun = require("@saltcorn/data/models/workflow_run");
+const User = require("@saltcorn/data/models/user");
 const { viewname } = require("./common");
 
-const runNextTask = async (alwaysRun) => {
-  if (!alwaysRun) {
-    const settings = await MetaData.findOne({
-      type: "CopilotConstructMgr",
-      name: "settings",
-    });
-    if (!settings?.body?.running) return;
-  }
-  const tasks = await MetaData.find(
-    {
-      type: "CopilotConstructMgr",
-      name: "task",
-    },
-    { orderBy: "id" }
-  );
-  const todos = tasks.filter(
-    (t) => !t.body.status || t.body.status === "To do"
-  );
-  const done = tasks.filter((t) => t.body.status === "Done");
-  const done_names = new Set(done.map((t) => t.body.name));
-
-  const startable = todos.filter((t) =>
-    t.body.depends_on.every((nm) => done_names.has(nm))
-  );
-
-  if (startable[0]) {
-    console.log("running task", startable[0]);
-
-    return await runTask(startable[0].id, {});
-  }
-  //not done
-};
-
+/**
+ * @param {number} md_id - MetaData id of the task to run
+ * @param {object} req - Express request (may be empty `{}` from scheduler)
+ */
 const runTask = async (md_id, req) => {
   const md = await MetaData.findOne({
     id: md_id,
@@ -59,15 +31,18 @@ const runTask = async (md_id, req) => {
     when_trigger: "Never",
     configuration: {
       viewname: viewname,
-      sys_prompt: "",
+      sys_prompt:
+        "Each task creates exactly one view or one page. " +
+        "Never create more than one view or page per task, even if the description mentions multiple. " +
+        "Call the view or page tool exactly once and then stop.",
       prompt: "{{prompt}}",
       skills: [
         { skill_type: "Generate Page", yoloMode: true },
         { skill_type: "Database design", yoloMode: true },
         { skill_type: "Generate Workflow", yoloMode: true },
+        { skill_type: "Generate trigger", yoloMode: true },
         { skill_type: "Generate View", yoloMode: true },
         { skill_type: "Install Plugin", yoloMode: true },
-        { skill_type: "AppConstructor Context" },
       ],
     },
   });
@@ -83,16 +58,20 @@ Important: The database schema is already fully implemented. Do NOT use generate
 
 Important: Some fields are non-stored (virtual) calculated fields — they have no database column and are computed on-the-fly by Saltcorn. Never include such fields in modify_row, SQL UPDATE statements, or recalculate_stored_fields calls. Only fields that exist as actual database columns (regular fields and stored calculated fields) can be written. If a calculated field needs updating, it will refresh automatically when the fields it depends on change.
 
+Important: The "users" table is built-in. Passwords are platform-managed — never add a password field to a view. Signup uses a built-in form, not an Edit view.
+
 Your task now is:
 ${md.body.description}`;
+  const safeReq = req?.__
+    ? req
+    : { ...req, __: (s) => s, user: req?.user };
 
   await md.update({ body: { ...md.body, status: "Running" } });
   const actionres = await agent_action.runWithoutRow({
     row: { prompt },
-    req,
-    user: req?.user,
+    req: safeReq,
+    user: safeReq.user,
   });
-  //console.log("actionres", actionres);
   const run_id = actionres.json.run_id;
   const run = await WorkflowRun.findOne({ id: run_id });
   await agent_action.runWithoutRow({
@@ -100,18 +79,18 @@ ${md.body.description}`;
       prompt:
         "Write a description of what you did, for the purposes of a progress report. Write 1-4 sentences. Do not use any tools or write any code",
     },
-    req,
+    req: safeReq,
     run,
-    user: req?.user,
+    user: safeReq.user,
   });
   const lastInteraction =
     run.context.interactions[run.context.interactions.length - 1];
   const lastText =
     typeof lastInteraction.content === "string"
       ? lastInteraction.content
-      : lastInteraction.content.text
+        : lastInteraction.content.text
         ? lastInteraction.content.text
-        : Array.isArray(lastInteraction.content)
+          : Array.isArray(lastInteraction.content)
           ? lastInteraction.content[0].text
           : lastInteraction.content;
   await MetaData.create({
@@ -121,8 +100,47 @@ ${md.body.description}`;
     user_id: req?.user?.id,
   });
 
-  //console.log("run", run);
   await md.update({ body: { ...md.body, status: "Done", run_id } });
 };
 
+/**
+ * Run the next startable task
+ * @param {boolean} [once=false] - true: run one task and stop, false: iterate all tasks
+ */
+const runNextTask = async (once = false) => {
+  if (!once) {
+    const settings = await MetaData.findOne({
+      type: "CopilotConstructMgr",
+      name: "settings",
+    });
+    if (!settings?.body?.running) return;
+  }
+  const tasks = await MetaData.find(
+    {
+      type: "CopilotConstructMgr",
+      name: "task",
+    },
+    { orderBy: "id" }
+  );
+  if (tasks.some((t) => t.body.status === "Running")) return;
+  const todos = tasks.filter(
+    (t) => !t.body.status || t.body.status === "To do"
+  );
+  const done = tasks.filter((t) => t.body.status === "Done");
+  const done_names = new Set(done.map((t) => t.body.name));
+
+  const startable = todos.filter((t) =>
+    t.body.depends_on.every((nm) => done_names.has(nm))
+  );
+
+  if (startable[0]) {
+    console.log("running task", startable[0]);
+    const taskUser = startable[0].user_id
+      ? await User.findOne({ id: startable[0].user_id })
+      : null;
+    await runTask(startable[0].id, { user: taskUser, __: (s) => s });
+    if (!once) await runNextTask();
+  }
+};
+
 module.exports = { runTask, runNextTask };

package/app-constructor/schema.js

@@ -123,6 +123,8 @@ ${existing_tables_list(existing_tables)}
 
 Design a complete database schema that covers ALL requirements listed above. Every distinct entity in the application must have its own table. Do not produce a minimal or partial schema — all tables needed to implement every requirement must be included in this single call. Do not leave any tables for a later step.
 
+The tables listed above are already implemented in the database — include them in the schema as-is so the full data model is visible, but do not change their fields. Only add new tables for entities not yet covered. The implementation step will skip any table whose name already exists.
+
 For every field that must be unique (e.g. unique email, unique slug, unique combination keys expressed as individual unique fields), set unique=true on that field.
 For every field that must not be empty, set not_null=true.
 Do NOT leave uniqueness or required constraints for a later step — express them fully in this schema.
@@ -174,7 +176,18 @@ const implement_schema = async (
   });
 
   const { apply_copilot_tables } = new GenerateTablesSkill({}).userActions;
-  await apply_copilot_tables({ tables: md.body.tables, user: req.user });
+  const existingNames = new Set((await Table.find({})).map((t) => t.name));
+  const newTables = md.body.tables.filter((t) => {
+    if (existingNames.has(t.name)) {
+      getState().log(
+        2,
+        `AppConstructor: skipping table "${t.name}" — already exists in database`
+      );
+      return false;
+    }
+    return true;
+  });
+  await apply_copilot_tables({ tables: newTables, user: req.user });
   md.body.implemented = true;
   await md.update({ body: md.body });