@saltcorn/copilot 0.8.2 → 0.8.3

package/app-constructor/tools.js

@@ -46,7 +46,13 @@ const task_tool = {
           type: "array",
           items: {
             type: "object",
-            required: ["name", "description", "priority", "depends_on"],
+            required: [
+              "name",
+              "description",
+              "priority",
+              "depends_on",
+              "task_type",
+            ],
             additionalProperties: false,
             properties: {
               name: {
@@ -71,6 +77,12 @@ const task_tool = {
                   type: "string",
                 },
               },
+              task_type: {
+                type: "string",
+                enum: ["plugin", "data_model", "feature"],
+                description:
+                  "plugin: specialized — installs a plugin from the Saltcorn plugin store. data_model: specialized — creates or modifies database tables/fields only. feature: broad catch-all — creates views, pages, triggers, workflows, or anything else not covered by the specialized types. Order: plugin tasks first, then data_model, then feature.",
+              },
             },
           },
         },

package/app-constructor/view.js

@@ -36,14 +36,20 @@ const {
 const { getState } = require("@saltcorn/data/db/state");
 const renderLayout = require("@saltcorn/markup/layout");
 const { viewname } = require("./common");
-const { requirementsList, req_routes } = require("./requirements");
-const { showSchema, schema_routes } = require("./schema");
-const { makeTaskList, task_routes } = require("./tasks");
-const { errorList, error_routes } = require("./errors");
+const { showSchema, schema_routes, schemaStaticScript } = require("./schema");
+const { task_routes } = require("./tasks");
+const {
+  errorList,
+  doCreateErrorFixTask,
+  error_routes,
+  errTableStaticHtml,
+} = require("./errors");
+const { req_routes, requirementsStaticScript } = require("./requirements");
 const { feedbackList, feedback_routes } = require("./feedback");
-const { progressList, progress_routes } = require("./progress");
+const { progress_routes } = require("./progress");
 const { runNextTask } = require("./run_task");
-const { makeTaskChart } = require("./taskchart");
+const { researchPanel, research_routes } = require("./research");
+const { phasesPanel, phasesStaticScript, phase_routes } = require("./phases");
 
 const get_state_fields = () => [];
 
@@ -56,37 +62,14 @@ const makeSpecForm = async (req) => {
   });
 
   return new Form({
-    blurb: "Provide a high-level description of the application",
+    blurb: "Describe the application you want to build",
     fields: [
       {
-        name: "description",
-        label: "Description",
-        type: "String",
-        fieldview: "textarea",
-      },
-      {
-        name: "audience",
-        label: "Audience",
-        type: "String",
-        fieldview: "textarea",
-      },
-      {
-        name: "core_features",
-        label: "Core features",
-        type: "String",
-        fieldview: "textarea",
-      },
-      {
-        name: "out_of_scope",
-        label: "Out of scope",
-        type: "String",
-        fieldview: "textarea",
-      },
-      {
-        name: "visual_style",
-        label: "Visual style",
+        name: "specification",
+        label: "Specification",
         type: "String",
         fieldview: "textarea",
+        attributes: { rows: 10 },
       },
     ],
     xhrSubmit: true,
@@ -95,14 +78,95 @@ const makeSpecForm = async (req) => {
   });
 };
 
+const specDepsModal = `
+<div class="modal fade" id="specDepsModal" tabindex="-1" aria-hidden="true">
+  <div class="modal-dialog">
+    <div class="modal-content">
+      <div class="modal-header">
+        <h5 class="modal-title">Specification changed</h5>
+        <button type="button" class="btn-close" data-bs-dismiss="modal"></button>
+      </div>
+      <div class="modal-body">
+        <p>The following items were generated from the previous specification
+        and may now be outdated. Select any you want to clear:</p>
+        <div id="spec-deps-checks"></div>
+      </div>
+      <div class="modal-footer">
+        <button type="button" class="btn btn-secondary" id="specDepsKeepBtn">Save (keep all)</button>
+        <button type="button" class="btn btn-primary" id="specDepsSaveBtn">Clear selected &amp; save</button>
+      </div>
+    </div>
+  </div>
+</div>`;
+
+const specDepsScript = (vn) =>
+  script(
+    domReady(`
+const _specVn = ${JSON.stringify(vn)};
+const form = document.querySelector('form[action*="submit_specs"]');
+if (!form) return;
+const btn = form.querySelector('button[onclick*="ajaxSubmitForm"]');
+if (!btn) return;
+
+const doSaveSpec = () => {
+  const data = {};
+  for (const [k, v] of new FormData(form)) data[k] = v;
+  view_post(_specVn, 'submit_specs', data);
+};
+
+// Remove Saltcorn's inline onclick and replace with our own handler
+btn.removeAttribute('onclick');
+btn.addEventListener('click', (e) => {
+  e.preventDefault();
+  e.stopPropagation();
+  view_post(_specVn, 'check_spec_dependencies', {}, (deps) => {
+    const items = [];
+    if (deps.hasResearch) items.push({ key: 'clearResearch', label: 'Research questions & answers' });
+    if (deps.hasRequirements) items.push({ key: 'clearRequirements', label: 'Requirements' });
+    if (deps.hasSchema) items.push({ key: 'clearSchema', label: 'Schema' });
+    if (deps.hasPhases) items.push({ key: 'clearPhases', label: 'Phases' });
+    if (deps.hasTasks) items.push({ key: 'clearTasks', label: 'Tasks' });
+    if (!items.length) { doSaveSpec(); return; }
+    document.getElementById('spec-deps-checks').innerHTML = items.map((item) =>
+      '<div class="form-check">' +
+        '<input class="form-check-input" type="checkbox" id="dep_' + item.key +
+        '" value="' + item.key + '" checked>' +
+        '<label class="form-check-label" for="dep_' + item.key + '">' + item.label + '</label>' +
+        '</div>'
+    ).join('');
+    new bootstrap.Modal(document.getElementById('specDepsModal')).show();
+  });
+});
+
+document.getElementById('specDepsSaveBtn').addEventListener('click', () => {
+  const modal = bootstrap.Modal.getInstance(document.getElementById('specDepsModal'));
+  const clearData = {};
+  document.querySelectorAll('#spec-deps-checks input:checked').forEach((cb) => {
+    clearData[cb.value] = '1';
+  });
+  modal.hide();
+  if (Object.keys(clearData).length) {
+    const formData = {};
+    for (const [k, v] of new FormData(form)) formData[k] = v;
+    view_post(_specVn, 'clear_and_save_spec', Object.assign(formData, clearData));
+  } else {
+    doSaveSpec();
+  }
+});
+
+document.getElementById('specDepsKeepBtn').addEventListener('click', () => {
+  bootstrap.Modal.getInstance(document.getElementById('specDepsModal')).hide();
+  doSaveSpec();
+});
+`)
+  );
+
 const run = async (table_id, viewname, cfg, state, { req, res }) => {
   const specForm = await makeSpecForm(req);
-  const reqList = await requirementsList(req);
-  const taskList = await makeTaskList(req);
+  const research = await researchPanel(req);
+  const phases = await phasesPanel(req);
   const errList = await errorList(req);
   const feedbacks = await feedbackList(req);
-  const progress = await progressList(req);
-  const taskChart = await makeTaskChart(req);
   const schema = await showSchema(req);
   const layout = {
     type: "tabs",
@@ -111,26 +175,40 @@ const run = async (table_id, viewname, cfg, state, { req, res }) => {
     lazyLoadViews: true,
     titles: [
       "Specification",
-      "Requirements",
+      "Research",
+      "Phases",
       "Schema",
-      "Tasks",
-      "Task chart",
-      "Progress",
       "Feedback",
       "Errors",
     ],
     contents: [
       {
         type: "blank",
-        contents: div({ class: "mt-2" }, renderForm(specForm, req.csrfToken())),
+        contents: div(
+          { class: "mt-2" },
+          renderForm(specForm, req.csrfToken()),
+          specDepsModal,
+          specDepsScript(viewname)
+        ),
+      },
+      { type: "blank", contents: research },
+      { type: "blank", contents: div(phasesStaticScript, phases) },
+      {
+        type: "blank",
+        contents: div(
+          schemaStaticScript,
+          div({ id: "schema-list-area" }, schema)
+        ),
       },
-      { type: "blank", contents: reqList },
-      { type: "blank", contents: schema },
-      { type: "blank", contents: taskList },
-      { type: "blank", contents: taskChart },
-      { type: "blank", contents: progress },
       { type: "blank", contents: feedbacks },
-      { type: "blank", contents: errList },
+      {
+        type: "blank",
+        contents: div(
+          requirementsStaticScript,
+          errTableStaticHtml,
+          div({ id: "err-list-area" }, errList)
+        ),
+      },
     ],
     deeplink: true,
     tabsStyle: "Tabs",
@@ -144,6 +222,119 @@ const run = async (table_id, viewname, cfg, state, { req, res }) => {
   });
 };
 
+const check_spec_dependencies = async (
+  table_id,
+  viewname,
+  config,
+  body,
+  { req, res }
+) => {
+  const hasResearch =
+    !!(await MetaData.findOne({
+      type: "CopilotConstructMgr",
+      name: "research_questions",
+    })) ||
+    !!(await MetaData.findOne({
+      type: "CopilotConstructMgr",
+      name: "research_answers",
+    }));
+  const hasRequirements =
+    (
+      await MetaData.find({
+        type: "CopilotConstructMgr",
+        name: "requirement",
+      })
+    ).length > 0;
+  const hasSchema = !!(await MetaData.findOne({
+    type: "CopilotConstructMgr",
+    name: "schema",
+  }));
+  const hasPhases =
+    (
+      await MetaData.find({
+        type: "CopilotConstructMgr",
+        name: "phase",
+      })
+    ).length > 0;
+  const hasTasks =
+    (
+      await MetaData.find({
+        type: "CopilotConstructMgr",
+        name: "task",
+      })
+    ).length > 0;
+  return {
+    json: { hasResearch, hasRequirements, hasPhases, hasSchema, hasTasks },
+  };
+};
+
+// Clears selected dependencies, saves the spec, and reloads — all in one round trip
+const clear_and_save_spec = async (
+  table_id,
+  viewname,
+  config,
+  body,
+  { req, res }
+) => {
+  const {
+    _csrf,
+    clearResearch,
+    clearRequirements,
+    clearSchema,
+    clearPhases,
+    clearTasks,
+    ...specBody
+  } = body;
+  if (clearResearch) {
+    for (const name of ["research_questions", "research_answers"]) {
+      const md = await MetaData.findOne({ type: "CopilotConstructMgr", name });
+      if (md) await md.delete();
+    }
+  }
+  if (clearRequirements) {
+    const rs = await MetaData.find({
+      type: "CopilotConstructMgr",
+      name: "requirement",
+    });
+    for (const r of rs) await r.delete();
+  }
+  if (clearSchema) {
+    const md = await MetaData.findOne({
+      type: "CopilotConstructMgr",
+      name: "schema",
+    });
+    if (md) await md.delete();
+  }
+  if (clearTasks) {
+    const ts = await MetaData.find({
+      type: "CopilotConstructMgr",
+      name: "task",
+    });
+    for (const t of ts) await t.delete();
+  }
+  if (clearPhases) {
+    const ps = await MetaData.find({
+      type: "CopilotConstructMgr",
+      name: "phase",
+    });
+    for (const ph of ps) await ph.delete();
+  }
+  const existing = await MetaData.findOne({
+    type: "CopilotConstructMgr",
+    name: "spec",
+  });
+  if (existing)
+    await db.update("_sc_metadata", { body: specBody }, existing.id);
+  else
+    await MetaData.create({
+      type: "CopilotConstructMgr",
+      name: "spec",
+      user_id: req.user?.id || undefined,
+      body: specBody,
+    });
+  return { json: { reload_page: true } };
+};
+
 const submit_specs = async (table_id, viewname, config, body, { req, res }) => {
   const { _csrf, ...spec } = body;
   const existing = await MetaData.findOne({
@@ -168,6 +359,35 @@ const submit_specs = async (table_id, viewname, config, body, { req, res }) => {
   };
 };
 
+// Creates fix tasks for application errors recorded while self-healing was off.
+const healPendingErrors = async () => {
+  const settings = await MetaData.findOne({
+    type: "CopilotConstructMgr",
+    name: "settings",
+  });
+  if (!settings?.body?.error_heal) return;
+  const spec = await MetaData.findOne({
+    type: "CopilotConstructMgr",
+    name: "spec",
+  });
+  if (!spec) return;
+  const errors = await MetaData.find({
+    type: "CopilotConstructMgr",
+    name: "error",
+  });
+  for (const err of errors) {
+    if (
+      err.body.source !== "application" ||
+      err.body.fix_task_created ||
+      err.body.cannot_fix_reason
+    )
+      continue;
+    doCreateErrorFixTask(err, null).catch((e) =>
+      console.error("healPendingErrors failed", e)
+    );
+  }
+};
+
 const virtual_triggers = () => {
   return [
     {
@@ -177,20 +397,49 @@ const virtual_triggers = () => {
           type: "CopilotConstructMgr",
           name: "error",
         });
-        const messages = new Set(existing.map((m) => m.body?.error?.stack));
-        if (!messages.has(row.stack))
-          await MetaData.create({
+        // Allow a new record once a fix task exists — same error recurring after a fix gets another task.
+        const unfixedDuplicate =
+          row.stack &&
+          existing.find(
+            (m) =>
+              m.body?.error?.stack === row.stack && !m.body?.fix_task_created
+          );
+        if (unfixedDuplicate) return;
+
+        const source = (row.stack || "").includes("/app-constructor/")
+          ? "constructor"
+          : "application";
+
+        const errorMd = await MetaData.create({
+          type: "CopilotConstructMgr",
+          name: "error",
+          body: { status: "New", error: row, source },
+          user_id: null,
+        });
+
+        if (source === "application") {
+          const settings = await MetaData.findOne({
             type: "CopilotConstructMgr",
-            name: "error",
-            body: { status: "New", error: row },
-            user_id: null,
+            name: "settings",
           });
+          if (settings?.body?.error_heal) {
+            const spec = await MetaData.findOne({
+              type: "CopilotConstructMgr",
+              name: "spec",
+            });
+            if (spec)
+              doCreateErrorFixTask(errorMd, null).catch((e) =>
+                console.error("auto error fix task failed", e)
+              );
+          }
+        }
       },
     },
     {
       when_trigger: "Often",
       run: async () => {
         await runNextTask();
+        await healPendingErrors();
       },
     },
   ];
@@ -205,12 +454,16 @@ module.exports = {
   run,
   routes: {
     submit_specs,
+    check_spec_dependencies,
+    clear_and_save_spec,
     ...req_routes,
+    ...research_routes,
     ...task_routes,
     ...error_routes,
     ...feedback_routes,
     ...progress_routes,
     ...schema_routes,
+    ...phase_routes,
   },
   virtual_triggers,
 };

package/builder-gen.js

@@ -27,13 +27,9 @@ designated as ownership fields (set automatically from the logged-in user). \
 All other foreign key fields — including parent-context keys like trip_id — MUST appear as \
 selector inputs; Saltcorn auto-fills them from the URL state when opened in context.`;
 
-const EDIT_FIELDVIEW_SELECTION = `\
-For date fields always prefer fieldview "flatpickr" when available — it provides the best user experience \
-and works for both regular dates and day-only dates. \
-Only use fieldview "edit_day" as a fallback when the field has day_only=true and flatpickr is not installed. \
-For String fields that have an options attribute (a comma-separated list of fixed choices), \
-use fieldview "select" — this renders a dropdown with those options. \
-Do not use "select_by_code" for fields with fixed options.`;
+const {
+  fieldview_selection_rules: EDIT_FIELDVIEW_SELECTION,
+} = require("./app-constructor/prompts");
 
 const EDIT_LAYOUT_STRUCTURE = `\
 Every field MUST be preceded by a label. \
@@ -1309,9 +1305,16 @@ const buildContext = async (mode, tableName) => {
     if (!fieldviews.length && String(typeName).startsWith("Key")) {
       fieldviews = ["select", "show"];
     }
+    // File fields store their fieldviews in state.fileviews, not on the type object
+    let fileFvDefs = {};
+    if (String(typeName) === "File") {
+      fileFvDefs = getState().fileviews || {};
+      fieldviews = Object.keys(fileFvDefs);
+    }
     // editFieldviews: only fieldviews where isEdit is not explicitly false
+    const effectiveFvDefs = String(typeName) === "File" ? fileFvDefs : fvDefs;
     const editFieldviews = fieldviews.filter(
-      (fv) => fvDefs[fv]?.isEdit !== false
+      (fv) => effectiveFvDefs[fv]?.isEdit !== false
     );
 
     const isPkName =

package/builder-schema.js

@@ -327,7 +327,13 @@ const buildBuilderSchema = ({ mode, ctx }) => {
   const actionDef = buildSegmentDef({
     type: "action",
     description: "Action button segment.",
+    required: ["rndid"],
     properties: {
+      rndid: {
+        type: "string",
+        description:
+          "Unique short identifier for this action (e.g. 'act1', 'del2'). REQUIRED — omitting it causes a server crash.",
+      },
       action_name: defs.action_name_enum
         ? { $ref: "#/$defs/action_name_enum", description: "Action name." }
         : { type: "string", description: "Action name." },

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@saltcorn/copilot",
-  "version": "0.8.2",
+  "version": "0.8.3",
   "description": "AI assistant for building Saltcorn applications",
   "main": "index.js",
   "dependencies": {

package/app-constructor/prompts.js

@@ -1,120 +0,0 @@
-const saltcorn_description = `This application will be implemented in Saltcorn, a database application development
-environment. 
-
-Saltcorn applications contain the following entity types:
-
-* Tables: These are relational database tables and consists of fields of specified types and rows 
-with a value for each field. Fields optionally can be required and/or unique. Every field has a name, 
-which is a an identifier that is balid in both JavaScript and SQL, and a label, which is any short 
-user-friendly string. Every table has a primary key
-(composite primary keys are not supported) which by default is an auto-incrementing integer with 
-name \`id\` and label ID. Fields can also be of Key type (foreign key) referencing a primary key 
-in another table, or its own table for a self-join. Tables can have 
-calculated fields, which can be stored or non-stored. Both stored and non-stored fields are 
-defined by a JavaScript expression, but only stored fields can reference other tables with join 
-fields and aggregations.
-
-* Views: Views are elementary user interfaces into a database table. A view is defined by applying a 
-view template (also sometime called a view pattern, the two are synonymous) to a table with a certain
-configuration. The view template defines the fundamental relationship between the UI and the table. For
-instance, the Show view template displays a single database row, the Edit view template is a form that 
-can create a new row or edit an existing row, the List view template displays multiple rows in a grid. 
-Views can embed views, for instance Show can embed another row through a Key field relationship, or 
-some views are defined by an underlying view. For instance, the Feed view repeats an underlying view 
-for multiple tables. New viewtemplates are provided by plugin modules.
-
-* Triggers: Triggers connect elementary actions (provided by plugin modules) to either a button in the
-user interface, or a periodic (hourly, daily etc) or table (for instance insert on specifc table) event.
-The elementary action each has a number of configuration fields that must be filled in after connecting
-the action to an event, table or button.
-
-* Page: A page has static content but can also embed views for synamic content. Pages can be either 
-defined by a Saltcorn layout, for pages that can be edited with drag and drop, or by HTML for more 
-flexible graphic designs. HTML pages should be used for landing pages.
-
-* Plugin modules: plugin modules can supply new field types, view templates or actions. Before they can be used,
-they need to be installed before they can be used. A plugin may also have a configuration that sets options
-for that plugin. Layout themes is Saltcorn are plugin modules.
-`;
-
-const existing_tables_list = (tables) => {
-  const tableLines = [];
-  tables.forEach((table) => {
-    const fieldLines = table.fields.map(
-      (f) =>
-        `  * ${f.name} with type: ${f.pretty_type}.${
-          f.description ? ` ${f.description}` : ""
-        }`
-    );
-    tableLines.push(
-      `${table.name}${
-        table.description ? `: ${table.description}.` : "."
-      } Contains the following fields:\n${fieldLines.join("\n")}`
-    );
-  });
-  return `The database already contains the following tables: 
-
-${tableLines.join("\n\n")}`;
-};
-
-const existing_entities_list = ({ views, triggers, pages, tableById = {} }) => {
-  const sections = [];
-  if (views.length)
-    sections.push(
-      `The following views are already implemented — do NOT plan tasks to create them:\n` +
-        views
-          .map((v) => {
-            const tablePart =
-              v.table?.name ||
-              (v.table_id && tableById[v.table_id]) ||
-              v.exttable_name;
-            return `- ${v.name} (${v.viewtemplate}${
-              tablePart ? ` on ${tablePart}` : ""
-            })`;
-          })
-          .join("\n")
-    );
-  if (triggers.length)
-    sections.push(
-      `The following triggers are already implemented — do NOT plan tasks to create them:\n` +
-        triggers
-          .map(
-            (t) =>
-              `- ${t.name} (${t.action}${
-                t.when_trigger ? `, ${t.when_trigger}` : ""
-              })`
-          )
-          .join("\n")
-    );
-  if (pages.length)
-    sections.push(
-      `The following pages are already implemented — do NOT plan tasks to create them:\n` +
-        pages.map((p) => `- ${p.name}`).join("\n")
-    );
-  return sections.join("\n\n");
-};
-
-const available_plugins_list = (storePlugins, installedNames) => {
-  const uninstalled = storePlugins.filter((p) => !installedNames.has(p.name));
-  if (!uninstalled.length) return "";
-  const lines = uninstalled.map((p) => {
-    let line = `### ${p.name}`;
-    if (p.description) line += `\n${p.description}`;
-    if (p.contents) line += `\n${p.contents}`;
-    return line;
-  });
-  return (
-    `The following plugins are available in the Saltcorn store but not yet installed. ` +
-    `If a task requires functionality provided by one of these plugins (e.g. a specific view template, field type, or action), ` +
-    `include an explicit "Install plugin <name>" task before it with the exact plugin name as listed here. ` +
-    `The executor will use that name directly without needing to look it up.\n\n` +
-    lines.join("\n\n")
-  );
-};
-
-module.exports = {
-  saltcorn_description,
-  existing_tables_list,
-  existing_entities_list,
-  available_plugins_list,
-};