@saltcorn/copilot 0.8.1 → 0.8.2

package/app-constructor/schema.js

@@ -39,6 +39,7 @@ const { viewname, tool_choice } = require("./common");
 const { requirements_tool } = require("./tools");
 const { saltcorn_description, existing_tables_list } = require("./prompts");
 const GenerateTables = require("../actions/generate-tables");
+const { buildMermaidMarkup } = GenerateTables;
 const GenerateTablesSkill = require("../agent-skills/database-design");
 
 const showSchema = async (req) => {
@@ -48,15 +49,94 @@ const showSchema = async (req) => {
   });
 
   if (schema) {
-    const preview = GenerateTables.render_html(
-      { tables: schema.body.tables },
-      true
+    const newTableDefs = schema.body.tables || [];
+    const newTableInstances = GenerateTables.process_tables(newTableDefs);
+    const reusedMd = await MetaData.findOne({
+      type: "CopilotConstructMgr",
+      name: "reused_schema",
+    });
+    const reusedNames = reusedMd?.body?.table_names || [];
+    const reusedInstances = reusedNames
+      .map((n) => Table.findOne({ name: n }))
+      .filter(Boolean);
+    const allTables = [...newTableInstances, ...reusedInstances];
+    const mmdia = buildMermaidMarkup(allTables);
+    const implemented = !!schema.body.implemented;
+
+    const newNames = newTableDefs.map((t) => t.table_name).filter(Boolean);
+
+    const colorMap = Object.fromEntries([
+      ...newNames.map((n) => [n, "#198754"]),
+      ...reusedNames.map((n) => [n, "#6c757d"]),
+    ]);
+    const colorScript = script(
+      domReady(`
+  const colors = ${JSON.stringify(colorMap)};
+  const pre = document.querySelector('.schema-mermaid');
+  if (!pre) return;
+
+  const doRender = () => {
+    mermaid.run({ nodes: [pre], suppressErrors: true, postRenderCallback: () => {
+      for (const g of pre.querySelectorAll('g[id^="entity-"]')) {
+        const name = g.id.replace(/^entity-/, '').replace(/-\\d+$/, '');
+        const color = colors[name];
+        if (color) {
+          const p = g.querySelector('path');
+          if (p) p.setAttribute('fill', color);
+        }
+      }
+      for (const el of pre.querySelectorAll('g.label.name .nodeLabel')) {
+        el.style.color = 'white';
+        el.style.fontWeight = 'bold';
+      }
+    }});
+  };
+
+  // Defer render until tab is visible, then colorize nodes.
+  const pane = pre.closest('.tab-pane');
+  if (pane && !pane.classList.contains('active')) {
+    const link = document.querySelector('[href="#' + pane.id + '"]');
+    if (link) link.addEventListener('shown.bs.tab', doRender, { once: true });
+    else {
+      const o = new MutationObserver(() => {
+        if (pane.classList.contains('active')) { o.disconnect(); doRender(); }
+      });
+      o.observe(pane, { attributes: true, attributeFilter: ['class'] });
+    }
+  } else doRender();
+`)
+    );
+
+    const legend = div(
+      { class: "mt-3 d-flex flex-wrap gap-3 align-items-start" },
+      newNames.length
+        ? div(
+            { class: "d-flex flex-wrap align-items-center gap-1" },
+            span(
+              { class: "me-1 text-muted small" },
+              implemented ? "Was created:" : "Will be created:"
+            ),
+            ...newNames.map((n) => span({ class: "badge bg-success" }, n))
+          )
+        : "",
+      reusedNames.length
+        ? div(
+            { class: "d-flex flex-wrap align-items-center gap-1" },
+            span(
+              { class: "me-1 text-muted small" },
+              implemented ? "Already existed:" : "Already exists:"
+            ),
+            ...reusedNames.map((n) => span({ class: "badge bg-secondary" }, n))
+          )
+        : ""
     );
 
     return div(
       { class: "mt-2" },
-      preview,
-      !schema.body.implemented &&
+      pre({ class: "schema-mermaid" }, mmdia),
+      colorScript,
+      legend,
+      !implemented &&
         div(
           { class: "mb-4 d-block mt-3" },
           button(
@@ -75,37 +155,71 @@ const showSchema = async (req) => {
           )
         )
     );
-  } else {
+  }
+
+  const generating = await MetaData.findOne({
+    type: "CopilotConstructMgr",
+    name: "generating_schema",
+  });
+  if (generating) {
     return div(
       { class: "mt-2" },
-      p("Schema not found"),
-      button(
-        {
-          class: "btn btn-primary",
-          onclick: `press_store_button(this);view_post("${viewname}", "gen_schema")`,
-        },
-        "Generate schema"
+      p(
+        i({ class: "fas fa-spinner fa-spin me-2" }),
+        "Generating schema, please wait..."
+      ),
+      script(
+        domReady(`
+const poll = () => {
+  view_post(${JSON.stringify(viewname)}, 'schema_status', {}, (resp) => {
+    if (resp && !resp.generating) location.reload();
+    else setTimeout(poll, 3000);
+  });
+};
+setTimeout(poll, 3000);
+`)
       )
     );
   }
+
+  return div(
+    { class: "mt-2", id: "schema-gen-area" },
+    p("Schema not found"),
+    button(
+      { class: "btn btn-primary", onclick: `copilotGenSchema()` },
+      "Generate schema"
+    ),
+    script(
+      domReady(`
+window.copilotGenSchema = () => {
+  document.getElementById('schema-gen-area').innerHTML =
+    '<p><i class="fas fa-spinner fa-spin me-2"></i>Generating schema, please wait...</p>';
+  view_post(${JSON.stringify(viewname)}, 'gen_schema', {}, () => {});
+  const poll = () => {
+    view_post(${JSON.stringify(viewname)}, 'schema_status', {}, (resp) => {
+      if (resp && !resp.generating) location.reload();
+      else setTimeout(poll, 3000);
+    });
+  };
+  setTimeout(poll, 3000);
+};
+`)
+    )
+  );
 };
 
-const gen_schema = async (table_id, viewname, config, body, { req, res }) => {
-  const spec = await MetaData.findOne({
-    type: "CopilotConstructMgr",
-    name: "spec",
-  });
-  if (!spec) throw new Error("Specification not found");
-  const rs = await MetaData.find({
+const doGenSchema = async (spec, rs, userId) => {
+  const generatingMd = await MetaData.create({
     type: "CopilotConstructMgr",
-    name: "requirement",
+    name: "generating_schema",
+    body: {},
+    user_id: userId,
   });
-  if (!rs.length) throw new Error("No requirements found");
-
-  const databaseDesignTool = new GenerateTablesSkill({}).provideTools();
-  const existing_tables = await Table.find({});
-  const answer = await getState().functions.llm_generate.run(
-    `Generate the database schema for this application:
+  try {
+    const databaseDesignTool = new GenerateTablesSkill({}).provideTools();
+    const existing_tables = await Table.find({});
+    const answer = await getState().functions.llm_generate.run(
+      `Generate the database schema for this application:
 
 Description: ${spec.body.description}
 Audience: ${spec.body.audience}
@@ -113,7 +227,7 @@ Core features: ${spec.body.core_features}
 Out of scope: ${spec.body.out_of_scope}
 Visual style: ${spec.body.visual_style}
 
-These are the requirements of the application: 
+These are the requirements of the application:
 
 ${rs.map((r) => `* ${r.body.requirement}`).join("\n")}
 
@@ -123,7 +237,9 @@ ${existing_tables_list(existing_tables)}
 
 Design a complete database schema that covers ALL requirements listed above. Every distinct entity in the application must have its own table. Do not produce a minimal or partial schema — all tables needed to implement every requirement must be included in this single call. Do not leave any tables for a later step.
 
-The tables listed above are already implemented in the database — include them in the schema as-is so the full data model is visible, but do not change their fields. Only add new tables for entities not yet covered. The implementation step will skip any table whose name already exists.
+The tables listed above already exist in the database. Do NOT modify or extend them — treat them as fixed. Handle them as follows:
+- If an existing table is already complete and used as-is: add its name to reused_table_names. Do NOT define its fields again in the tables array.
+- New tables not yet in the database: include them in the tables array with all their fields as usual.
 
 For every field that must be unique (e.g. unique email, unique slug, unique combination keys expressed as individual unique fields), set unique=true on that field.
 For every field that must not be empty, set not_null=true.
@@ -132,34 +248,78 @@ Do NOT leave uniqueness or required constraints for a later step — express the
 Note: ownership configuration (automatically populating a FK-to-users field from the logged-in user) is a VIEW-level concern and cannot be expressed in the schema. Do not attempt to annotate fields as "ownership fields" here — simply define the foreign key field normally. Ownership will be configured when the Edit views are generated.
 
 Now use the ${
-      databaseDesignTool.function.name
-    } tool to generate the complete database schema for this software application
+        databaseDesignTool.function.name
+      } tool to generate the complete database schema for this software application
 `,
-    {
-      tools: [databaseDesignTool],
-      ...tool_choice(databaseDesignTool.function.name),
-      systemPrompt:
-        "You are a database designer. The user wants to build an application, and you must analyse their application description and requirements and design a complete schema. Every entity needed by any requirement must have its own table. Never produce a partial schema.",
-    }
-  );
+      {
+        tools: [databaseDesignTool],
+        ...tool_choice(databaseDesignTool.function.name),
+        systemPrompt:
+          "You are a database designer. The user wants to build an application, and you must analyse their application description and requirements and design a complete schema. Every entity needed by any requirement must have its own table. Never produce a partial schema.",
+      }
+    );
+
+    const tc = answer.getToolCalls()[0];
 
-  const tc = answer.getToolCalls()[0];
+    const noNewTables = !tc.input.tables || tc.input.tables.length === 0;
+    await MetaData.create({
+      type: "CopilotConstructMgr",
+      name: "schema",
+      body: { tables: tc.input.tables || [], implemented: noNewTables },
+      user_id: userId,
+    });
 
-  await MetaData.create({
+    const reusedNames = tc.input.reused_table_names || [];
+    if (reusedNames.length) {
+      await MetaData.create({
+        type: "CopilotConstructMgr",
+        name: "reused_schema",
+        body: { table_names: reusedNames },
+        user_id: userId,
+      });
+    }
+  } finally {
+    await generatingMd.delete();
+  }
+};
+
+const gen_schema = async (table_id, viewname, config, body, { req, res }) => {
+  const spec = await MetaData.findOne({
     type: "CopilotConstructMgr",
-    name: "schema",
-    body: { tables: tc.input.tables, implemented: false },
-    user_id: req.user?.id,
+    name: "spec",
   });
-  return { json: { reload_page: true } };
+  if (!spec) throw new Error("Specification not found");
+  const rs = await MetaData.find({
+    type: "CopilotConstructMgr",
+    name: "requirement",
+  });
+  if (!rs.length) throw new Error("No requirements found");
+
+  doGenSchema(spec, rs, req.user?.id).catch((e) =>
+    console.error("gen_schema error", e)
+  );
+  return { json: { success: true } };
 };
 
-const del_schema = async (table_id, viewname, config, body, { req, res }) => {
-  const rs = await MetaData.find({
+const schema_status = async (
+  table_id,
+  viewname,
+  config,
+  body,
+  { req, res }
+) => {
+  const generating = await MetaData.findOne({
     type: "CopilotConstructMgr",
-    name: "schema",
+    name: "generating_schema",
   });
-  for (const r of rs) await r.delete();
+  return { json: { generating: !!generating } };
+};
+
+const del_schema = async (table_id, viewname, config, body, { req, res }) => {
+  for (const name of ["schema", "reused_schema", "generating_schema"]) {
+    const rs = await MetaData.find({ type: "CopilotConstructMgr", name });
+    for (const r of rs) await r.delete();
+  }
   return { json: { reload_page: true } };
 };
 
@@ -194,6 +354,11 @@ const implement_schema = async (
   return { json: { reload_page: true } };
 };
 
-const schema_routes = { gen_schema, del_schema, implement_schema };
+const schema_routes = {
+  gen_schema,
+  schema_status,
+  del_schema,
+  implement_schema,
+};
 
 module.exports = { showSchema, schema_routes };