@saltcorn/server 1.6.0-beta.1 → 1.6.0-beta.10

package/routes/tables.js

@@ -8,6 +8,7 @@ const Router = require("express-promise-router");
 
 const db = require("@saltcorn/data/db");
 const Table = require("@saltcorn/data/models/table");
+const Field = require("@saltcorn/data/models/field");
 const File = require("@saltcorn/data/models/file");
 const View = require("@saltcorn/data/models/view");
 const User = require("@saltcorn/data/models/user");
@@ -63,6 +64,9 @@ const {
   discoverable_tables,
   discover_tables,
   implement_discovery,
+  reconcile_table,
+  make_field,
+  findType,
 } = require("@saltcorn/data/models/discovery");
 const { getState } = require("@saltcorn/data/db/state");
 const { cardHeaderTabs } = require("@saltcorn/markup/layout_utils");
@@ -792,6 +796,311 @@ const attribBadges = (f) => {
   return s;
 };
 
+/**
+ * Rescan Table Fields (GET Handler)
+ * @name get/rescan/:idorname
+ * @function
+ * @memberof module:routes/tables~tablesRouter
+ */
+router.get(
+  "/rescan/:idorname",
+  isAdmin,
+  error_catcher(async (req, res) => {
+    const { idorname } = req.params;
+    let id = parseInt(idorname);
+    let table;
+    if (id) [table] = await Table.find({ id });
+    if (!table) [table] = await Table.find({ name: idorname });
+    if (!table) {
+      req.flash("error", req.__(`Table not found`));
+      res.redirect(`/table`);
+      return;
+    }
+
+    const result = await reconcile_table(table);
+    const ghosts = result.fields.filter((f) => f.status === "ghost");
+    const orphans = result.fields.filter((f) => f.status === "orphan");
+    const matches = result.fields.filter((f) => f.status === "match");
+
+    const statusBadge = (status) => {
+      switch (status) {
+        case "match": return badge("success", req.__("Match"));
+        case "ghost": return badge("danger", req.__("Ghost"));
+        case "orphan": return badge("primary", req.__("Orphan"));
+        default: return "";
+      }
+    };
+
+    let ghostForm = "";
+    if (ghosts.length > 0) {
+      ghostForm = form(
+        {
+          method: "post",
+          action: `/table/rescan-delete/${table.id}`,
+        },
+        input({ type: "hidden", name: "_csrf", value: req.csrfToken() }),
+        h4({ class: "text-danger" }, req.__("Ghost fields (in Saltcorn, not in DB)")),
+        mkTable(
+          [
+            {
+              label: "",
+              key: (r) =>
+                input({
+                  type: "checkbox",
+                  name: "fields",
+                  value: r.name,
+                  checked: true,
+                }),
+            },
+            { label: req.__("Name"), key: "name" },
+            { label: req.__("Type"), key: (r) => r.type || "—" },
+            { label: req.__("Status"), key: (r) => statusBadge(r.status) },
+          ],
+          ghosts,
+          { hover: true }
+        ),
+        button(
+          {
+            type: "submit",
+            class: "btn btn-danger mt-2",
+          },
+          req.__("Remove selected ghosts from Saltcorn")
+        )
+      );
+    }
+
+    let orphanForm = "";
+    if (orphans.length > 0) {
+      orphanForm = form(
+        {
+          method: "post",
+          action: `/table/rescan-import/${table.id}`,
+        },
+        input({ type: "hidden", name: "_csrf", value: req.csrfToken() }),
+        h4({ class: "text-primary" }, req.__("Orphan columns (in DB, not in Saltcorn)")),
+        mkTable(
+          [
+            {
+              label: "",
+              key: (r) =>
+                input({
+                  type: "checkbox",
+                  name: "columns",
+                  value: r.name,
+                  checked: true,
+                }),
+            },
+            { label: req.__("Column"), key: "name" },
+            { label: req.__("SQL Type"), key: (r) => r.type || "—" },
+            { label: req.__("Status"), key: (r) => statusBadge(r.status) },
+          ],
+          orphans,
+          { hover: true }
+        ),
+        button(
+          {
+            type: "submit",
+            class: "btn btn-primary mt-2",
+          },
+          req.__("Import selected orphans into Saltcorn")
+        )
+      );
+    }
+
+    let matchCard = "";
+    if (matches.length > 0) {
+      matchCard = div(
+        h4(req.__("Matching fields (%d)", matches.length)),
+        mkTable(
+          [
+            { label: req.__("Name"), key: "name" },
+            { label: req.__("Type"), key: (r) => r.type || "—" },
+            { label: req.__("Status"), key: (r) => statusBadge(r.status) },
+          ],
+          matches,
+          { hover: true }
+        )
+      );
+    }
+
+    const allMatch = ghosts.length === 0 && orphans.length === 0;
+
+    res.sendWrap(req.__("Rescan fields: %s", table.name), {
+      above: [
+        {
+          type: "breadcrumbs",
+          crumbs: [
+            { text: req.__("Tables"), href: "/table" },
+            {
+              text: table.name,
+              href: `/table/${encodeURIComponent(table.name)}`,
+            },
+            { text: req.__("Rescan fields") },
+          ],
+        },
+        allMatch && { type: "card", title: req.__("All fields match"), contents: p(req.__("All Saltcorn fields match the database columns.")) + a({ href: `/table/${table.id}`, class: "btn btn-primary" }, req.__("Back to table")) },
+        ghosts.length > 0 && { type: "card", title: req.__("Ghosts (%d)", ghosts.length), contents: ghostForm },
+        orphans.length > 0 && { type: "card", title: req.__("Orphans (%d)", orphans.length), contents: orphanForm },
+        matches.length > 0 && { type: "card", title: req.__("Matches"), contents: matchCard },
+      ].filter(Boolean),
+    });
+  })
+);
+
+/**
+ * Rescan Delete Ghosts (POST Handler)
+ * @name post/rescan-delete/:idorname
+ * @function
+ * @memberof module:routes/tables~tablesRouter
+ */
+router.post(
+  "/rescan-delete/:idorname",
+  isAdmin,
+  error_catcher(async (req, res) => {
+    const { idorname } = req.params;
+    let id = parseInt(idorname);
+    let table;
+    if (id) [table] = await Table.find({ id });
+    if (!table) [table] = await Table.find({ name: idorname });
+    if (!table) {
+      req.flash("error", req.__(`Table not found`));
+      res.redirect(`/table`);
+      return;
+    }
+
+    const fieldNames = Array.isArray(req.body.fields)
+      ? req.body.fields
+      : req.body.fields
+        ? [req.body.fields]
+        : [];
+    const fields = table.getFields();
+    let removed = 0;
+
+    for (const fname of fieldNames) {
+      const field = fields.find(
+        (f) => f.name === fname
+      );
+      if (field) {
+        await db.deleteWhere("_sc_fields", { id: field.id });
+        removed++;
+      }
+    }
+
+    await getState().refresh_tables(true);
+    req.flash(
+      "success",
+      req.__("Removed %d ghost field(s) from Saltcorn", removed)
+    );
+    res.redirect(`/table/rescan/${table.id}`);
+  })
+);
+
+/**
+ * Rescan Import Orphans (POST Handler)
+ * @name post/rescan-import/:idorname
+ * @function
+ * @memberof module:routes/tables~tablesRouter
+ */
+router.post(
+  "/rescan-import/:idorname",
+  isAdmin,
+  error_catcher(async (req, res) => {
+    const { idorname } = req.params;
+    let id = parseInt(idorname);
+    let table;
+    if (id) [table] = await Table.find({ id });
+    if (!table) [table] = await Table.find({ name: idorname });
+    if (!table) {
+      req.flash("error", req.__(`Table not found`));
+      res.redirect(`/table`);
+      return;
+    }
+
+    const colNames = Array.isArray(req.body.columns)
+      ? req.body.columns
+      : req.body.columns
+        ? [req.body.columns]
+        : [];
+    let imported = 0;
+    const skipped = [];
+
+    const schema = db.getTenantSchema();
+
+    for (const colName of colNames) {
+      // Get column info from DB and build a field config
+      let fieldCfg;
+      if (!db.isSQLite) {
+        const { rows } = await db.query(
+          "SELECT * FROM information_schema.columns WHERE table_schema = $1 AND table_name = $2 AND column_name = $3",
+          [schema, table.name, colName]
+        );
+        if (rows.length) {
+          fieldCfg = await make_field(rows[0]);
+          // Fallback: if make_field couldn't determine the type, try findType directly
+          if (!fieldCfg || !fieldCfg.type) {
+            const sqlType =
+              findType(rows[0].data_type) ||
+              findType(rows[0].udt_name) ||
+              "String";
+            fieldCfg = {
+              name: rows[0].column_name,
+              label: rows[0].column_name,
+              type: sqlType,
+              required: rows[0].is_nullable === "NO",
+            };
+          }
+        }
+      } else {
+        const { rows } = await db.query(
+          `PRAGMA table_info("${table.name}")`
+        );
+        const col = rows.find((r) => r.name === colName);
+        if (col) {
+          const type = findType(col.type) || "String";
+          fieldCfg = {
+            name: colName,
+            label: colName,
+            type,
+            required: col.notnull === 1,
+          };
+        }
+      }
+
+      if (fieldCfg && fieldCfg.type) {
+        // Insert directly into _sc_fields — the physical column already exists,
+        // so we must NOT use Field.create (which does ALTER TABLE ADD COLUMN).
+        const typeName =
+          typeof fieldCfg.type === "string" ? fieldCfg.type : fieldCfg.type?.name;
+        await db.insert("_sc_fields", {
+          table_id: table.id,
+          name: fieldCfg.name,
+          label: fieldCfg.label || fieldCfg.name,
+          type: typeName,
+          required: !!fieldCfg.required,
+          is_unique: !!fieldCfg.is_unique,
+          primary_key: !!fieldCfg.primary_key,
+        });
+        imported++;
+      } else {
+        skipped.push(colName);
+      }
+    }
+
+    await getState().refresh_tables(true);
+    const msgs = [req.__("Imported %d orphan column(s) into Saltcorn", imported)];
+    if (skipped.length)
+      msgs.push(
+        req.__(
+          "Skipped %d column(s) with unrecognized type: %s",
+          skipped.length,
+          skipped.join(", ")
+        )
+      );
+    req.flash("success", msgs.join(". "));
+    res.redirect(`/table/rescan/${table.id}`);
+  })
+);
+
 /**
  * Table Constructor (GET Handler)
  * @name get/:idorname
@@ -847,6 +1156,18 @@ router.get(
     let fieldCard;
     const primaryKeys = fields.filter((f) => f.primary_key);
     const nPrimaryKeys = primaryKeys.length;
+    const standardFieldNames = [
+      "name",
+      "description",
+      "created_at",
+      "updated_at",
+      "created_by",
+      "updated_by",
+    ];
+    const fieldNameSet = new Set(fields.map((f) => f.name));
+    const hasAllStandardFields = standardFieldNames.every((n) =>
+      fieldNameSet.has(n)
+    );
 
     if (fields.length === 0) {
       fieldCard = [
@@ -862,6 +1183,17 @@ router.get(
             },
             req.__("Add field to table")
           ),
+        !table.external &&
+          !table.provider_name &&
+          user_can_edit_tables &&
+          !hasAllStandardFields &&
+          a(
+            {
+              class: "btn btn-outline-secondary ms-2",
+              href: `javascript:ajax_modal('/table/create-standard-fields/${table.id}')`,
+            },
+            req.__("Create standard fields")
+          ),
       ];
     } else {
       const tableHtml = mkTable(
@@ -946,6 +1278,17 @@ router.get(
             },
             req.__("Add field")
           ),
+        !table.external &&
+          !table.provider_name &&
+          user_can_edit_tables &&
+          !hasAllStandardFields &&
+          a(
+            {
+              class: "btn btn-outline-secondary ms-2 mt-2",
+              href: `javascript:ajax_modal('/table/create-standard-fields/${table.id}')`,
+            },
+            req.__("Create standard fields")
+          ),
       ];
     }
     let viewCard;
@@ -1140,6 +1483,13 @@ router.get(
         div(
           { class: "mx-auto" },
           settingsDropdown(`dataMenuButton`, [
+            a(
+              {
+                class: "dropdown-item",
+                href: `/table/rescan/${table.id}`,
+              },
+              '<i class="fas fa-search"></i>&nbsp;' + req.__("Rescan fields")
+            ),
             // rename table doesnt supported for sqlite
             !db.isSQLite &&
               user_can_edit_tables &&
@@ -1789,8 +2139,13 @@ const constraintForm = (req, table, fields, type) => {
             name: "formula",
             label: req.__("Constraint formula"),
             validator: expressionValidator,
-            type: "String",
-            class: "validate-expression",
+            input_type: "code",
+            attributes: {
+              mode: "application/javascript",
+              singleline: true,
+              table: table.name,
+              expression_type: "boolean",
+            },
             help: {
               topic: "Table formula constraint",
               context: { table: table.name },
@@ -1946,6 +2301,8 @@ router.post(
           configuration,
         });
       });
+      // set updated_at
+      await table.update({});
       await getState().refresh_tables();
 
       Trigger.emitEvent(
@@ -2072,6 +2429,8 @@ router.post(
     await db.withTransaction(async () => {
       await cons.delete();
     });
+    //set updated_at
+    await Table.findOne(cons.table_id).update({});
     await getState().refresh_tables();
     res.redirect(`/table/constraints/${cons.table_id}`);
   })
@@ -2678,3 +3037,134 @@ router.post(
     res.redirect(`/table/${table.id}`);
   })
 );
+
+const standardFieldDefs = (req) => [
+  {
+    name: "name",
+    label: req.__("Name"),
+    type: "String",
+    fieldType: "String",
+  },
+  {
+    name: "description",
+    label: req.__("Description"),
+    type: "String",
+    fieldType: "String",
+  },
+  {
+    name: "created_at",
+    label: req.__("Created at"),
+    type: "Date",
+    fieldType: "Date",
+    required: true,
+    attributes: { default_expression: "new Date()" },
+  },
+  {
+    name: "updated_at",
+    label: req.__("Updated at"),
+    type: "Date",
+    fieldType: "Date",
+    calculated: true,
+    stored: true,
+    expression: "new Date()",
+  },
+  {
+    name: "created_by",
+    label: req.__("Created by"),
+    type: "Key to user",
+    fieldType: "Key",
+    reftable_name: "users",
+    attributes: { default_expression: "user?.id", summary_field: "email" },
+  },
+  {
+    name: "updated_by",
+    label: req.__("Updated by"),
+    type: "Key to user",
+    fieldType: "Key",
+    reftable_name: "users",
+    calculated: true,
+    stored: true,
+    expression: "user?.id ?? row?.updated_by",
+    attributes: { summary_field: "email" },
+  },
+];
+
+const standardFieldForm = (table, req, existingNames = new Set()) => {
+  const defs = standardFieldDefs(req).filter(
+    (def) => !existingNames.has(def.name)
+  );
+  return new Form({
+    submitLabel: req.__("Create fields"),
+    action: `/table/create-standard-fields/${table.id}`,
+    formStyle: "vert",
+    fields: defs.map((def) => ({
+      type: "Bool",
+      label: `${def.label} (${def.type})`,
+      name: def.name,
+      default: true,
+    })),
+  });
+};
+
+router.get(
+  "/create-standard-fields/:id",
+  isAdminOrHasConfigMinRole("min_role_edit_tables"),
+  error_catcher(async (req, res) => {
+    const { id } = req.params;
+    const table = Table.findOne({ id });
+    if (!table) {
+      req.flash("error", `Table not found`);
+      res.redirect(`/table`);
+      return;
+    }
+    const existingFields = await table.getFields();
+    const existingNames = new Set(existingFields.map((f) => f.name));
+    res.set("Page-Title", req.__("Create standard fields"));
+    const form = standardFieldForm(table, req, existingNames);
+    res.send(renderForm(form, req.csrfToken()));
+  })
+);
+
+router.post(
+  "/create-standard-fields/:id",
+  isAdminOrHasConfigMinRole("min_role_edit_tables"),
+  error_catcher(async (req, res) => {
+    const { id } = req.params;
+    const table = Table.findOne({ id });
+    if (!table) {
+      req.flash("error", `Table not found`);
+      res.redirect(`/table`);
+      return;
+    }
+    const existingFields = await table.getFields();
+    const existingNames = new Set(existingFields.map((f) => f.name));
+    const form = standardFieldForm(table, req, existingNames);
+    form.validate(req.body || {});
+    if (form.hasErrors) {
+      req.flash("error", req.__("An error occurred"));
+      res.redirect(`/table/${table.id}`);
+      return;
+    }
+    const defs = standardFieldDefs(req);
+    await db.withTransaction(async () => {
+      for (const def of defs) {
+        if (!form.values[def.name]) continue;
+        if (existingNames.has(def.name)) continue;
+        await Field.create({
+          table_id: table.id,
+          name: def.name,
+          label: def.label,
+          type: def.fieldType,
+          reftable_name: def.reftable_name,
+          required: def.required || false,
+          calculated: def.calculated || false,
+          stored: def.stored || false,
+          expression: def.expression,
+          attributes: def.attributes || {},
+        });
+      }
+    });
+    await getState().refresh_tables();
+    res.redirect(`/table/${table.id}`);
+  })
+);

package/routes/tenant.js

@@ -43,7 +43,7 @@ const {
 } = require("@saltcorn/markup/tags");
 const db = require("@saltcorn/data/db");
 
-const { loadAllPlugins, loadAndSaveNewPlugin } = require("../load_plugins");
+const Plugin = require("@saltcorn/data/models/plugin");
 const {
   isAdmin,
   error_catcher,
@@ -309,9 +309,9 @@ router.post(
 
         await create_tenant({
           t: subdomain,
-          plugin_loader: loadAllPlugins,
+          plugin_loader: Plugin.loadAllPlugins,
           noSignalOrDB: false,
-          loadAndSaveNewPlugin: loadAndSaveNewPlugin,
+          loadAndSaveNewPlugin: Plugin.loadAndSaveNewPlugin,
           tenant_template,
         });
         let new_url_create = newurl;

package/routes/utils.js

@@ -13,7 +13,11 @@ const {
   features,
 } = require("@saltcorn/data/db/state");
 const { get_base_url } = require("@saltcorn/data/models/config");
-const { hash } = require("@saltcorn/data/utils");
+const {
+  hash,
+  is_relative_url,
+  normalize_relative_url,
+} = require("@saltcorn/data/utils");
 const { input, script, domReady, a, text } = require("@saltcorn/markup/tags");
 const session = require("express-session");
 const cookieSession = require("cookie-session");
@@ -355,7 +359,7 @@ const getSessionStore = (pruneInterval) => {
       secret: db.connectObj.session_secret || is.str.generate(),
       resave: false,
       saveUninitialized: false,
-      cookie: { maxAge: 30 * 24 * 60 * 60 * 1000, sameSite }, // 30 days
+      cookie: { maxAge: 30 * 24 * 60 * 60 * 1000, sameSite, secure: "auto" }, // 30 days
     });
   } else {
     const pgSession = require("connect-pg-simple")(session);
@@ -369,7 +373,7 @@ const getSessionStore = (pruneInterval) => {
       secret: db.connectObj.session_secret || is.str.generate(),
       resave: false,
       saveUninitialized: false,
-      cookie: { maxAge: 30 * 24 * 60 * 60 * 1000, sameSite }, // 30 days
+      cookie: { maxAge: 30 * 24 * 60 * 60 * 1000, sameSite, secure: "auto" }, // 30 days
     });
   }
 };
@@ -389,9 +393,14 @@ const addOnDoneRedirect = (oldPath, req) => {
   return oldPath;
 };
 
-//https://stackoverflow.com/a/38979205/19839414
-const is_relative_url = (url) => {
-  return typeof url === "string" && !url.includes(":/") && !url.includes("//");
+const safe_redirect = (res, url, default_url) => {
+  if (!url) {
+    res.redirect(default_url);
+    return;
+  }
+  const dest = normalize_relative_url(url);
+  if (dest !== null) res.redirect(dest);
+  else res.redirect(default_url);
 };
 
 /**
@@ -647,6 +656,8 @@ module.exports = {
   get_tenant_from_req,
   addOnDoneRedirect,
   is_relative_url,
+  normalize_relative_url,
+  safe_redirect,
   is_ip_address,
   get_sys_info,
   admin_config_route,

package/routes/viewedit.js

@@ -131,8 +131,8 @@ const mapObjectValues = (o, f) =>
  * @returns {Form}
  */
 const viewForm = async (req, tableOptions, roles, pages, values) => {
-  const isEdit =
-    values && values.id && !getState().getConfig("development_mode", false);
+  const devmode = getState().getConfig("development_mode", false);
+  const isEdit = values && values.id && !devmode;
   const hasTable = Object.entries(getState().viewtemplates)
     .filter(([k, v]) => !v.tableless && !v.table_optional)
     .map(([k, v]) => k);
@@ -141,7 +141,7 @@ const viewForm = async (req, tableOptions, roles, pages, values) => {
     .map(([k, v]) => k);
   const slugOptions = await Table.allSlugOptions();
   const viewpatternOptions = Object.values(getState().viewtemplates)
-    .filter((vt) => !vt.singleton)
+    .filter((vt) => !vt.singleton && !(!values && vt.deprecated && !devmode))
     .map((vt) => vt.name);
   return new Form({
     action: addOnDoneRedirect("/viewedit/save", req),
@@ -173,7 +173,10 @@ const viewForm = async (req, tableOptions, roles, pages, values) => {
         attributes: {
           explainers: mapObjectValues(
             getState().viewtemplates,
-            ({ description }) => description
+            ({ description, deprecated }) =>
+              deprecated && description
+                ? `${description}. <i>${req.__("Deprecated")}</i>`
+                : description || (deprecated ? req.__("Deprecated") : undefined)
           ),
         },
         disabled: isEdit,
@@ -690,6 +693,7 @@ const respondWorkflow = (view, wf, wfres, req, res, table) => {
     );
   else if (wfres.renderBuilder) {
     const locale = req.getLocale();
+    wfres.renderBuilder.options.isRTL = req.isRTL;
     wfres.renderBuilder.options.translations =
       locale === "en" ? {} : req.getCatalog(locale) || {};
     wfres.renderBuilder.options.view_id = view.id;
@@ -1018,7 +1022,7 @@ router.post(
   "/copilot-generate-layout",
   isAdminOrHasConfigMinRole(["min_role_edit_views", "min_role_edit_pages"]),
   error_catcher(async (req, res) => {
-    const { mode, table, prompt } = req.body;
+    const { mode, table, prompt, existing } = req.body;
     const f = getState().functions.copilot_generate_layout;
     if (!f) {
       res.json({
@@ -1026,7 +1030,7 @@ router.post(
       });
       return;
     }
-    const layout = await f.run(prompt, mode, table);
+    const layout = await f.run(prompt, mode, table, existing);
     res.json({ success: "ok", layout });
   })
 );