@saltcorn/copilot 0.8.0 → 0.8.2

package/agent-skills/viewgen.js

@@ -1,7 +1,10 @@
 const Table = require("@saltcorn/data/models/table");
 const View = require("@saltcorn/data/models/view");
 const { fieldProperties } = require("../common");
-const { initial_config_all_fields } = require("@saltcorn/data/plugin-helper");
+const {
+  initial_config_all_fields,
+  build_schema_data,
+} = require("@saltcorn/data/plugin-helper");
 const { getState } = require("@saltcorn/data/db/state");
 const {
   div,
@@ -14,6 +17,26 @@ const {
   text_attr,
 } = require("@saltcorn/markup/tags");
 const builderGen = require("../builder-gen");
+const {
+  RELATION_PATH_DOC,
+  GET_RELATION_PATHS_FUNCTION,
+  getRelationPathsForPairs,
+} = require("../relation-paths");
+
+const collectViewLinks = (segment, out = []) => {
+  if (!segment || typeof segment !== "object") return out;
+  if (Array.isArray(segment)) {
+    segment.forEach((s) => collectViewLinks(s, out));
+    return out;
+  }
+  if (segment.type === "view_link" && segment.view) out.push(segment);
+  if (segment.above) collectViewLinks(segment.above, out);
+  if (segment.besides) collectViewLinks(segment.besides, out);
+  if (segment.contents) collectViewLinks(segment.contents, out);
+  if (Array.isArray(segment.tabs))
+    segment.tabs.forEach((t) => collectViewLinks(t?.contents, out));
+  return out;
+};
 
 const collectLayoutFieldNames = (segment, out = new Set()) => {
   if (!segment || typeof segment !== "object") return out;
@@ -112,14 +135,35 @@ class GenerateViewSkill {
 
   async systemPrompt() {
     return (
-      `If the user asks to generate a view, use the generate_view tool to enter ` +
-      `a view generation mode. The tool call only requires high-level details to start this sequence.\n` +
-      `The Edit viewtemplate serves both create (no id in state) and edit (id in state) — one view covers both modes.`
+      `If the user asks to generate a view, use the generate_view tool — but ONLY if the view does not already exist. ` +
+      `If a view with that name already exists, do NOT call generate_view — doing so will create a duplicate. Instead follow the modification sequence below.\n` +
+      `The Edit viewtemplate serves both create (no id in state) and edit (id in state) — one view covers both.\n\n` +
+      `**Modifying an existing view — required sequence:**\n` +
+      `(1) Call get_view_config to fetch the current configuration.\n` +
+      `(2) Only if you are adding view_link columns or embedded view (type "view") segments: call get_relation_paths once with all the source_table/target_view pairs you need. For changes that don't involve linking or embedding views (e.g. adding a field, changing a label), skip this step.\n` +
+      `(3) Write out the complete updated configuration JSON in full — every key from the existing config must be present, with only your targeted changes merged in.\n` +
+      `(4) Call apply_view_config with that complete object. NEVER call apply_view_config before step (3) is finished. NEVER call it with only the name or a partial object — the configuration field is mandatory and must be the full merged result from step (3). Calling apply_view_config without a complete configuration is an error.\n\n` +
+      `**Generating a new view that contains view_links or embedded views:**\n` +
+      `If the task or prompt mentions a viewlink, a link to another view, or a button that opens another view from a list row, that view_link column is REQUIRED — do not omit it. ` +
+      `You MUST call get_relation_paths with all source_table/target_view pairs before constructing the layout. Never skip this step when view_links are needed.\n\n` +
+      `**Embedded view segment format (for Show layouts):**\n` +
+      `  { "type": "view", "view": "<viewName>", "name": "<viewName>", "relation": "<from get_relation_paths>" }\n` +
+      `Do NOT use blank text segments as placeholders — always use a real view segment with a relation string from get_relation_paths.\n\n` +
+      RELATION_PATH_DOC
     );
   }
 
   get userActions() {
     return {
+      async build_copilot_view_update({ name, configuration }) {
+        const existingView = View.findOne({ name });
+        if (!existingView) return { error: `View "${name}" not found` };
+        await View.update({ configuration }, existingView.id);
+        setTimeout(() => getState().refresh_views(), 200);
+        return {
+          notify: `View updated: <a target="_blank" href="/view/${name}">${name}</a>`,
+        };
+      },
       async build_copilot_view_gen({
         wfctx,
         name,
@@ -127,14 +171,28 @@ class GenerateViewSkill {
         table,
         min_role,
       }) {
-        const normalizedRole = min_role || "public";
+        const existing = View.findOne({ name });
+        if (existing)
+          return {
+            error: `View "${name}" already exists. Use get_view_config and apply_view_config to update it.`,
+          };
         const tableRow = table ? Table.findOne({ name: table }) : null;
+        const roleName =
+          typeof min_role === "number" ? null : min_role || "public";
+        const resolvedRole =
+          typeof min_role === "number"
+            ? min_role
+            : (
+                (getState().roles || []).find((r) => r.role === roleName) || {
+                  id: 100,
+                }
+              ).id;
         await View.create({
           name,
           viewtemplate: viewpattern,
           table_id: tableRow?.id,
           table: tableRow,
-          min_role: { admin: 1, public: 100, user: 80 }[normalizedRole],
+          min_role: resolvedRole,
           configuration: wfctx,
         });
         const vt = getState().viewtemplates[viewpattern];
@@ -199,12 +257,12 @@ class GenerateViewSkill {
       },
     };
 
-    return {
+    const generateViewTool = {
       type: "function",
       function: {
         name: "generate_view",
         description:
-          "Generate a view by supplying high-level details. This will trigger a view generation sequence",
+          "Generate a NEW view by supplying high-level details. Only call this for views that do not yet exist — if the view already exists, use get_view_config + apply_view_config instead.",
         parameters,
       },
       process: async (input) => {
@@ -264,23 +322,55 @@ class GenerateViewSkill {
             chat
           );
           if (table && viewpattern !== "Filter") {
-            const baseCfg = await initial_config_all_fields(false)({
+            // isEdit=true: FK fields get Field+select columns; false gives JoinField (display-only)
+            const isEditView = viewpattern === "Edit";
+            const baseCfg = await initial_config_all_fields(isEditView)({
               table_id: table.id,
             });
             if (baseCfg?.columns) wfctx.columns = baseCfg.columns;
           }
+          if (viewpattern === "List" && wfctx.layout) {
+            // initial_config_all_fields never generates ViewLink columns — inject them from layout
+            const viewLinks = collectViewLinks(wfctx.layout);
+            const viewLinkColumns = viewLinks.map((seg) => ({
+              type: "ViewLink",
+              view: seg.view,
+              block: seg.block || false,
+              label: seg.view_label || "",
+              minRole: seg.minRole || 100,
+              ...(seg.relation ? { relation: seg.relation } : {}),
+              isFormula: seg.isFormula || {},
+            }));
+            if (viewLinkColumns.length > 0)
+              wfctx.columns = [...(wfctx.columns || []), ...viewLinkColumns];
+          }
           if (viewpattern === "Edit" && table) {
             const layoutFieldNames = collectLayoutFieldNames(wfctx.layout);
             const fields = table.fields || [];
             const fixed = {};
+            const usersFkColumnsToAdd = [];
             for (const f of fields) {
               if (f.primary_key || f.calculated) continue;
-              if (layoutFieldNames.has(f.name)) continue;
               if (f.type === "Key" && f.reftable_name === "users") {
-                fixed[`preset_${f.name}`] = "LoggedIn";
-                fixed[`_block_${f.name}`] = true;
+                if (layoutFieldNames.has(f.name)) {
+                  // Explicitly placed in layout — add a select column so getForm renders it
+                  usersFkColumnsToAdd.push({
+                    field_name: f.name,
+                    type: "Field",
+                    fieldview: "select",
+                    state_field: true,
+                  });
+                } else {
+                  fixed[`preset_${f.name}`] = "LoggedIn";
+                  fixed[`_block_${f.name}`] = true;
+                }
               }
             }
+            if (usersFkColumnsToAdd.length > 0)
+              wfctx.columns = [
+                ...(wfctx.columns || []),
+                ...usersFkColumnsToAdd,
+              ];
             if (Object.keys(fixed).length > 0) wfctx.fixed = fixed;
             wfctx.destination_type = "Back to referer";
           }
@@ -347,12 +437,15 @@ class GenerateViewSkill {
             for (const field of form.fields) {
               if (prefilledFields.has(field.name)) continue;
               //TODO showIf
+              const isShowif = field.name.endsWith("_showif");
               properties[field.name] = {
                 description:
                   field.copilot_description ||
-                  `${field.label}.${
-                    field.sublabel ? ` ${field.sublabel}` : ""
-                  }`,
+                  (isShowif
+                    ? `${field.label}. The correct default is an empty string — leave it blank to always show this element. Only provide a JavaScript expression if the task description explicitly states that this element should be conditionally hidden based on a URL state variable or the current user. Never invent field names or copy examples.`
+                    : `${field.label}.${
+                        field.sublabel ? ` ${field.sublabel}` : ""
+                      }`),
                 ...fieldProperties(field),
               };
               if (!properties[field.name].type) {
@@ -388,13 +481,18 @@ class GenerateViewSkill {
                 },
               }
             );
-            const tc = answer.getToolCalls()[0];
-            await getState().functions.llm_add_message.run(
-              "tool_response",
-              { type: "text", value: "Details provided" },
-              { chat, tool_call: tc }
-            );
-            Object.assign(wfctx, tc.input);
+            const tc =
+              typeof answer?.getToolCalls === "function"
+                ? answer.getToolCalls()[0]
+                : null;
+            if (tc) {
+              await getState().functions.llm_add_message.run(
+                "tool_response",
+                { type: "text", value: "Details provided" },
+                { chat, tool_call: tc }
+              );
+              Object.assign(wfctx, tc.input);
+            }
           }
         }
         const roleName = tool_call.input.min_role || "public";
@@ -402,6 +500,13 @@ class GenerateViewSkill {
         const min_role = rolesState
           ? (rolesState.find((r) => r.role === roleName) || { id: 100 }).id
           : { admin: 1, public: 100, user: 80 }[roleName] ?? 100;
+        const existingView = View.findOne({ name: tool_call.input.name });
+        if (existingView) {
+          return {
+            stop: true,
+            add_response: `Error: view "${tool_call.input.name}" already exists. Do NOT call generate_view again — use get_view_config to inspect the current configuration and apply_view_config to update it.`,
+          };
+        }
         const view = new View({
           name: tool_call.input.name,
           viewtemplate: tool_call.input.viewpattern,
@@ -441,6 +546,127 @@ class GenerateViewSkill {
         };
       },
     };
+
+    const getViewConfigTool = {
+      type: "function",
+      function: {
+        name: "get_view_config",
+        description:
+          "Retrieve the current configuration of an existing view. " +
+          "Call this first to inspect the layout before calling apply_view_config to save changes. " +
+          "Returns the full configuration JSON and the viewtemplate name.",
+        parameters: {
+          type: "object",
+          required: ["name"],
+          properties: {
+            name: {
+              description: "The name of the existing view to inspect.",
+              type: "string",
+            },
+          },
+        },
+      },
+      process: async ({ name }) => {
+        const existingView = View.findOne({ name });
+        if (!existingView)
+          return `View "${name}" not found. Use generate_view to create a new view instead.`;
+        return (
+          `Current configuration of view "${name}" (viewtemplate: ${existingView.viewtemplate}):\n` +
+          JSON.stringify(existingView.configuration, null, 2)
+        );
+      },
+    };
+
+    const applyViewConfigTool = {
+      type: "function",
+      function: {
+        name: "apply_view_config",
+        description:
+          "Save an updated configuration to an existing view. " +
+          "STRICT PRECONDITION: you must have already called get_view_config AND written out the complete merged configuration JSON before calling this tool. " +
+          "Do NOT call this tool as a placeholder or before the configuration is fully constructed. " +
+          "Calling this tool without a complete configuration object is always wrong and will fail.",
+        parameters: {
+          type: "object",
+          required: ["name", "configuration"],
+          properties: {
+            name: {
+              description: "The name of the existing view to update.",
+              type: "string",
+            },
+            configuration: {
+              type: "object",
+              description:
+                "REQUIRED. The complete updated configuration object — every key from the existing config preserved, with only your changes merged in. " +
+                "You MUST have the full object written out before calling this tool. " +
+                "Passing null, an empty object, or a partial object (e.g. only the name) is always wrong and will return an error.",
+            },
+          },
+        },
+      },
+      process: async ({ name, configuration }) => {
+        const existingView = View.findOne({ name });
+        if (!existingView) return `View "${name}" not found.`;
+        if (!configuration || typeof configuration !== "object")
+          return (
+            `ERROR: configuration is missing. ` +
+            `You must call get_view_config first, merge your changes into the full existing configuration, then call apply_view_config again with the complete configuration object.`
+          );
+        return { name, configuration, view_id: existingView.id };
+      },
+      postProcess: async ({ tool_call, req }) => {
+        const { name, configuration } = tool_call.input;
+        const existingView = View.findOne({ name });
+        if (!existingView)
+          return { stop: true, add_response: `View "${name}" not found.` };
+        if (!configuration || typeof configuration !== "object")
+          return {
+            stop: true,
+            add_response:
+              `apply_view_config called for "${name}" without a configuration object. ` +
+              `Call get_view_config first, merge your changes into the full existing configuration, then call apply_view_config again with the complete configuration.`,
+          };
+        const cfg = configuration;
+
+        if (this.yoloMode) {
+          await View.update({ configuration: cfg }, existingView.id);
+          setTimeout(() => getState().refresh_views(), 200);
+          return { stop: true, add_response: `View ${name} updated.` };
+        }
+        return {
+          stop: true,
+          add_response: pre(JSON.stringify(cfg, null, 2)),
+          add_user_action: {
+            name: "build_copilot_view_update",
+            type: "button",
+            label: "Save updated view " + name,
+            input: { name, configuration: cfg },
+          },
+        };
+      },
+    };
+
+    const getRelationPathsTool = {
+      type: "function",
+      function: GET_RELATION_PATHS_FUNCTION,
+      process: async ({ pairs }) => {
+        const schemaData = await build_schema_data();
+        const sections = getRelationPathsForPairs(pairs || [], schemaData);
+        return (
+          sections.join("\n\n") +
+          `\n\nFor each pair, set the "relation" property to one of the strings listed above.\n` +
+          `Pick by type: ChildList = multiple child rows, ParentShow = single parent, OneToOneShow = unique child. ` +
+          `If multiple paths of the same type exist, choose the one whose FK field name best matches the task. Prefer shorter paths.`
+        );
+      },
+    };
+
+    return [
+      generateViewTool,
+      getViewConfigTool,
+      applyViewConfigTool,
+      getRelationPathsTool,
+    ];
   };
 }
 

package/app-constructor/prompts.js

@@ -42,12 +42,14 @@ const existing_tables_list = (tables) => {
   tables.forEach((table) => {
     const fieldLines = table.fields.map(
       (f) =>
-        `  * ${f.name} with type: ${f.pretty_type}.${f.description ? ` ${f.description}` : ""}`,
+        `  * ${f.name} with type: ${f.pretty_type}.${
+          f.description ? ` ${f.description}` : ""
+        }`
     );
     tableLines.push(
       `${table.name}${
         table.description ? `: ${table.description}.` : "."
-      } Contains the following fields:\n${fieldLines.join("\n")}`,
+      } Contains the following fields:\n${fieldLines.join("\n")}`
     );
   });
   return `The database already contains the following tables: 
@@ -55,4 +57,64 @@ const existing_tables_list = (tables) => {
 ${tableLines.join("\n\n")}`;
 };
 
-module.exports = { saltcorn_description, existing_tables_list };
+const existing_entities_list = ({ views, triggers, pages, tableById = {} }) => {
+  const sections = [];
+  if (views.length)
+    sections.push(
+      `The following views are already implemented — do NOT plan tasks to create them:\n` +
+        views
+          .map((v) => {
+            const tablePart =
+              v.table?.name ||
+              (v.table_id && tableById[v.table_id]) ||
+              v.exttable_name;
+            return `- ${v.name} (${v.viewtemplate}${
+              tablePart ? ` on ${tablePart}` : ""
+            })`;
+          })
+          .join("\n")
+    );
+  if (triggers.length)
+    sections.push(
+      `The following triggers are already implemented — do NOT plan tasks to create them:\n` +
+        triggers
+          .map(
+            (t) =>
+              `- ${t.name} (${t.action}${
+                t.when_trigger ? `, ${t.when_trigger}` : ""
+              })`
+          )
+          .join("\n")
+    );
+  if (pages.length)
+    sections.push(
+      `The following pages are already implemented — do NOT plan tasks to create them:\n` +
+        pages.map((p) => `- ${p.name}`).join("\n")
+    );
+  return sections.join("\n\n");
+};
+
+const available_plugins_list = (storePlugins, installedNames) => {
+  const uninstalled = storePlugins.filter((p) => !installedNames.has(p.name));
+  if (!uninstalled.length) return "";
+  const lines = uninstalled.map((p) => {
+    let line = `### ${p.name}`;
+    if (p.description) line += `\n${p.description}`;
+    if (p.contents) line += `\n${p.contents}`;
+    return line;
+  });
+  return (
+    `The following plugins are available in the Saltcorn store but not yet installed. ` +
+    `If a task requires functionality provided by one of these plugins (e.g. a specific view template, field type, or action), ` +
+    `include an explicit "Install plugin <name>" task before it with the exact plugin name as listed here. ` +
+    `The executor will use that name directly without needing to look it up.\n\n` +
+    lines.join("\n\n")
+  );
+};
+
+module.exports = {
+  saltcorn_description,
+  existing_tables_list,
+  existing_entities_list,
+  available_plugins_list,
+};

package/app-constructor/requirements.js

@@ -44,7 +44,7 @@ const requirementsList = async (req) => {
       type: "CopilotConstructMgr",
       name: "requirement",
     },
-    { orderBy: "written_at" },
+    { orderBy: "written_at" }
   );
   const starFieldview = getState().types.Integer.fieldviews.show_star_rating;
 
@@ -67,43 +67,85 @@ const requirementsList = async (req) => {
                   class: "btn btn-outline-danger btn-sm",
                   onclick: `view_post("${viewname}", "del_req", {id:${r.id}})`,
                 },
-                i({ class: "fas fa-trash-alt" }),
+                i({ class: "fas fa-trash-alt" })
               ),
           },
         ],
-        rs,
+        rs
       ),
       button(
         {
           class: "btn btn-outline-danger mb-4",
           onclick: `view_post("${viewname}", "del_all_reqs")`,
         },
-        "Delete all",
-      ),
+        "Delete all"
+      )
     );
-  } else {
+  }
+
+  const generating = await MetaData.findOne({
+    type: "CopilotConstructMgr",
+    name: "generating_requirements",
+  });
+  if (generating) {
     return div(
       { class: "mt-2" },
-      p("No requirements found"),
-      button(
-        {
-          class: "btn btn-primary",
-          onclick: `press_store_button(this);view_post("${viewname}", "gen_reqs")`,
-        },
-        "Generate requirements",
+      p(
+        i({ class: "fas fa-spinner fa-spin me-2" }),
+        "Generating requirements, please wait..."
       ),
+      script(
+        domReady(`
+(function() {
+  const poll = () => {
+    view_post(${JSON.stringify(viewname)}, 'req_status', {}, (resp) => {
+      if (resp && !resp.generating) location.reload();
+      else setTimeout(poll, 3000);
+    });
+  };
+  setTimeout(poll, 3000);
+})();
+`)
+      )
     );
   }
+
+  return div(
+    { class: "mt-2", id: "req-gen-area" },
+    p("No requirements found"),
+    button(
+      { class: "btn btn-primary", onclick: `copilotGenReqs()` },
+      "Generate requirements"
+    ),
+    script(
+      domReady(`
+window.copilotGenReqs = () => {
+  document.getElementById('req-gen-area').innerHTML =
+    '<p><i class="fas fa-spinner fa-spin me-2"></i>Generating requirements, please wait...</p>';
+  view_post(${JSON.stringify(viewname)}, 'gen_reqs', {}, () => {});
+  const poll = () => {
+    view_post(${JSON.stringify(viewname)}, 'req_status', {}, (resp) => {
+      if (resp && !resp.generating) location.reload();
+      else setTimeout(poll, 3000);
+    });
+  };
+  setTimeout(poll, 3000);
+};
+`)
+    )
+  );
 };
 
-const gen_reqs = async (table_id, viewname, config, body, { req, res }) => {
-  const spec = await MetaData.findOne({
+const doGenReqs = async (spec, userId) => {
+  const generatingMd = await MetaData.create({
     type: "CopilotConstructMgr",
-    name: "spec",
+    name: "generating_requirements",
+    body: {},
+    user_id: userId,
   });
-  if (!spec) throw new Error("Specification not found");
-  const answer = await getState().functions.llm_generate.run(
-    `Generate the requirements for this application:
+  try {
+    const answer = await getState().functions.llm_generate.run(
+      `Generate the requirements for this application:
 
 Description: ${spec.body.description}
 Audience: ${spec.body.audience}
@@ -113,24 +155,44 @@ Visual style: ${spec.body.visual_style}
 
 Now use the make_requirements tool to list the requirements for this software application
 `,
-    {
-      tools: [requirements_tool],
-      ...tool_choice("make_requirements"),
-      systemPrompt:
-        "You are a project manager. The user wants to build an application, and you must analyse their application description",
-    },
-  );
+      {
+        tools: [requirements_tool],
+        ...tool_choice("make_requirements"),
+        systemPrompt:
+          "You are a project manager. The user wants to build an application, and you must analyse their application description",
+      }
+    );
+    const tc = answer.getToolCalls()[0];
+    for (const reqm of tc.input.requirements)
+      await MetaData.create({
+        type: "CopilotConstructMgr",
+        name: "requirement",
+        body: reqm,
+        user_id: userId,
+      });
+  } finally {
+    await generatingMd.delete();
+  }
+};
 
-  const tc = answer.getToolCalls()[0];
+const gen_reqs = async (table_id, viewname, config, body, { req, res }) => {
+  const spec = await MetaData.findOne({
+    type: "CopilotConstructMgr",
+    name: "spec",
+  });
+  if (!spec) throw new Error("Specification not found");
+  doGenReqs(spec, req.user?.id).catch((e) =>
+    console.error("gen_reqs error", e)
+  );
+  return { json: { success: true } };
+};
 
-  for (const reqm of tc.input.requirements)
-    await MetaData.create({
-      type: "CopilotConstructMgr",
-      name: "requirement",
-      body: reqm,
-      user_id: req.user?.id,
-    });
-  return { json: { reload_page: true } };
+const req_status = async (table_id, viewname, config, body, { req, res }) => {
+  const generating = await MetaData.findOne({
+    type: "CopilotConstructMgr",
+    name: "generating_requirements",
+  });
+  return { json: { generating: !!generating } };
 };
 
 const del_req = async (table_id, viewname, config, body, { req, res }) => {
@@ -151,6 +213,6 @@ const del_all_reqs = async (table_id, viewname, config, body, { req, res }) => {
   return { json: { reload_page: true } };
 };
 
-const req_routes = { gen_reqs, del_req, del_all_reqs };
+const req_routes = { gen_reqs, req_status, del_req, del_all_reqs };
 
 module.exports = { requirementsList, req_routes };