@saltcorn/server 1.1.4-beta.1 → 1.1.4-beta.10

package/routes/admin.js

@@ -15,6 +15,8 @@ const {
   tenant_letsencrypt_name,
   isAdminOrHasConfigMinRole,
   checkEditPermission,
+  addOnDoneRedirect,
+  is_relative_url,
 } = require("./utils.js");
 const Table = require("@saltcorn/data/models/table");
 const Plugin = require("@saltcorn/data/models/plugin");
@@ -95,7 +97,10 @@ const {
 const packagejson = require("../package.json");
 const Form = require("@saltcorn/data/models/form");
 const { get_latest_npm_version } = require("@saltcorn/data/models/config");
-const { getMailTransport } = require("@saltcorn/data/models/email");
+const {
+  getMailTransport,
+  getOauth2Client,
+} = require("@saltcorn/data/models/email");
 const {
   getBaseDomain,
   hostname_matches_baseurl,
@@ -214,8 +219,22 @@ admin_config_route({
   flash: "Email settings updated",
   field_names: [
     "smtp_host",
-    "smtp_username",
-    "smtp_password",
+    "smtp_auth_method",
+    { name: "smtp_username", showIf: { smtp_auth_method: "password" } },
+    { name: "smtp_password", showIf: { smtp_auth_method: "password" } },
+    {
+      name: "smtp_api_option",
+      showIf: {
+        smtp_host: "outlook.office365.com",
+        smtp_auth_method: "oauth2",
+      },
+    },
+    { name: "smtp_authorize_url", showIf: { smtp_auth_method: "oauth2" } },
+    { name: "smtp_token_url", showIf: { smtp_auth_method: "oauth2" } },
+    { name: "smtp_redirect_uri", showIf: { smtp_auth_method: "oauth2" } },
+    { name: "smtp_client_id", showIf: { smtp_auth_method: "oauth2" } },
+    { name: "smtp_client_secret", showIf: { smtp_auth_method: "oauth2" } },
+    { name: "smtp_outh_scopes", showIf: { smtp_auth_method: "oauth2" } },
     "smtp_port",
     "smtp_secure",
     "smtp_allow_self_signed",
@@ -241,6 +260,40 @@ admin_config_route({
             },
             req.__("Send test email")
           ),
+          a(
+            {
+              id: "authenticate_oauth",
+              href: "/admin/authorize-mail-oauth",
+              class: "btn btn-primary ms-2",
+              onclick: "spin_action_link(this)",
+            },
+            req.__("Authorize")
+          ),
+          script(`
+  function authMethodChange(method) {
+    const btn = document.getElementById("authenticate_oauth");
+    if (method === "oauth2") {
+      btn.classList.remove("d-none");
+      btn.classList.add("d-inline");
+      // get 
+    }
+    else {
+      btn.classList.add("d-none");
+      btn.classList.remove("d-inline");
+    }
+
+  }
+  ${domReady(`
+  const authMethod = document.getElementById('inputsmtp_auth_method');
+  if (authMethod) {
+    const authMethodValue = authMethod.value;
+    authMethodChange(authMethodValue);
+    authMethod.addEventListener('change', (e) => {
+      const authMethod = e.target.value;
+      authMethodChange(authMethod);
+    });
+  }`)}
+`),
         ],
       },
     });
@@ -264,7 +317,7 @@ router.get(
       html: req.__("Hello from Saltcorn"),
     };
     try {
-      const sendres = await getMailTransport().sendMail(email);
+      const sendres = await (await getMailTransport()).sendMail(email);
       getState().log(6, sendres);
       req.flash(
         "success",
@@ -279,6 +332,27 @@ router.get(
   })
 );
 
+/**
+ * do the redirect to the oauth2 provider
+ */
+router.get(
+  "/authorize-mail-oauth",
+  isAdmin,
+  error_catcher(async (req, res) => {
+    const client = getOauth2Client();
+    const smtpRedirectUri = getState().getConfig("smtp_redirect_uri");
+    const scopes = getState().getConfig("smtp_outh_scopes");
+    const scopeArray = scopes.split(" ").map((s) => s.trim());
+
+    // Mail.Send SMTP.Send profile openid email https://outlook.office365.com/IMAP.AccessAsUser.All
+    const authorizeUrl = client.authorizeURL({
+      redirect_uri: smtpRedirectUri,
+      scope: scopeArray,
+    });
+    res.redirect(authorizeUrl);
+  })
+);
+
 /**
  * @name get/send-test-email
  * @function
@@ -435,7 +509,10 @@ router.get(
                     req.csrfToken(),
                     {
                       btnClass: "btn-outline-primary",
-                      spinner: true,
+                      klass: "backupadminbtn",
+                      //TODO not sure how to tell when backup done
+                      onClick:
+                        "spin_action_link($('.backupadminbtn'));setTimeout(()=> reset_spinners($('.backupadminbtn')), 3000)",
                     }
                   )
                 ),
@@ -723,7 +800,10 @@ router.get(
             label: req.__("Restore"),
             key: (r) =>
               post_btn(
-                `/admin/snapshot-restore/${type}/${name}/${r.id}`,
+                addOnDoneRedirect(
+                  `/admin/snapshot-restore/${type}/${name}/${r.id}`,
+                  req
+                ),
                 req.__("Restore"),
                 req.csrfToken()
               ),
@@ -762,13 +842,16 @@ router.post(
     }
     await getState().refresh();
     res.redirect(
-      type == "codepage"
+      req.query.on_done_redirect &&
+        is_relative_url("/" + req.query.on_done_redirect)
+        ? `/${req.query.on_done_redirect}`
+        : type == "codepage"
         ? `/admin/edit-codepage/${name}`
         : type === "trigger"
-          ? `/actions`
-          : /^[a-z]+$/g.test(type)
-            ? `/${type}edit`
-            : "/"
+        ? `/actions`
+        : /^[a-z]+$/g.test(type)
+        ? `/${type}edit`
+        : "/"
     );
   })
 );
@@ -1269,20 +1352,20 @@ router.get(
                             }
                           )
                         : isRoot && is_latest
-                          ? span(
-                              { class: "badge bg-primary ms-2" },
-                              req.__("Latest")
-                            ) +
-                            post_btn(
-                              "/admin/check-for-upgrade",
-                              req.__("Check updates"),
-                              req.csrfToken(),
-                              {
-                                btnClass: "btn-primary btn-sm px-1 py-0",
-                                formClass: "d-inline",
-                              }
-                            )
-                          : "",
+                        ? span(
+                            { class: "badge bg-primary ms-2" },
+                            req.__("Latest")
+                          ) +
+                          post_btn(
+                            "/admin/check-for-upgrade",
+                            req.__("Check updates"),
+                            req.csrfToken(),
+                            {
+                              btnClass: "btn-primary btn-sm px-1 py-0",
+                              formClass: "d-inline",
+                            }
+                          )
+                        : "",
                       isRoot &&
                         !git_commit &&
                         a(
@@ -2051,8 +2134,9 @@ router.get(
     const filename = `${moment(start).format("YYYYMMDDHHmm")}.html`;
     await File.new_folder("configuration_checks");
     const go = async () => {
-      const { passes, errors, pass, warnings } =
-        await runConfigurationCheck(req);
+      const { passes, errors, pass, warnings } = await runConfigurationCheck(
+        req
+      );
       const end = new Date();
       const secs = Math.round((end.getTime() - start.getTime()) / 1000);
 
@@ -3329,8 +3413,8 @@ router.get(
       mode === "prepare"
         ? "_prepare_step"
         : mode === "finish"
-          ? "_finish_step"
-          : "";
+        ? "_finish_step"
+        : "";
     res.json({
       finished: await checkFiles(out_dir_name, [
         `logs${stepDesc}.txt`,
@@ -3405,8 +3489,8 @@ router.get(
       mode === "prepare"
         ? "_prepare_step"
         : mode === "finish"
-          ? "_finish_step"
-          : "";
+        ? "_finish_step"
+        : "";
     const resultMsg = files.find(
       (file) => file.filename === `logs${stepDesc}.txt`
     )

package/routes/api.js

@@ -407,23 +407,35 @@ router.get(
                   target: field?.attributes?.summary_field,
                 };
             });
-            rows = await table.getJoinedRows({
-              where: qstate,
-              joinFields,
-              limit: use_limit,
-              offset: use_offset,
-              orderDesc:
-                (sortDesc && sortDesc !== "false") || tabulator_dir == "desc",
-              orderBy: orderByField?.name || undefined,
-              forPublic: !(req.user || user),
-              forUser: req.user || user,
-            });
+            try {
+              rows = await table.getJoinedRows({
+                where: qstate,
+                joinFields,
+                limit: use_limit,
+                offset: use_offset,
+                orderDesc:
+                  (sortDesc && sortDesc !== "false") || tabulator_dir == "desc",
+                orderBy: orderByField?.name || undefined,
+                forPublic: !(req.user || user),
+                forUser: req.user || user,
+              });
+            } catch (e) {
+              console.error(e);
+              res.json({ error: "API error" });
+              return;
+            }
           }
           if (tabulator_pagination_format) {
-            res.json({
-              last_page: Math.ceil((await table.countRows()) / +tabulator_size),
-              data: rows.map(limitFields(fields)),
-            });
+            const count = await table.countRows();
+            if (count === null)
+              res.json({
+                data: rows.map(limitFields(fields)),
+              });
+            else
+              res.json({
+                last_page: Math.ceil(count / +tabulator_size),
+                data: rows.map(limitFields(fields)),
+              });
           } else res.json({ success: rows.map(limitFields(fields)) });
         } else {
           getState().log(3, `API get ${table.name} not authorized`);

package/routes/common_lists.js

@@ -205,7 +205,7 @@ const view_dropdown = (view, req, on_done_redirect_str = "") =>
       a(
         {
           class: "dropdown-item",
-          href: `javascript:ajax_modal('/admin/snapshot-restore/view/${view.name}')`,
+          href: `javascript:ajax_modal('/admin/snapshot-restore/view/${view.name}${on_done_redirect_str}')`,
         },
         '<i class="fas fa-undo-alt"></i>&nbsp;' + req.__("Restore")
       ),
@@ -284,7 +284,7 @@ const tagsDropdown = (tags, altHeader, req) =>
         {
           class: "dropdown-item",
           // TODO check url why view for page, what do we need for page group
-          href: `tag`,
+          href: `/tag`,
         },
         req.__("Manage tags...")
       )
@@ -391,7 +391,7 @@ const viewsList = async (
                   filterOnTag ? `Tag:${filterOnTag.name}` : undefined,
                   req
                 ),
-                key: (r) => tagBadges(r),
+                key: (r) => (r.id ? tagBadges(r) : ""),
               },
             ]),
         {
@@ -674,12 +674,12 @@ const trigger_dropdown = (trigger, req, on_done_redirect_str = "") =>
     a(
       {
         class: "dropdown-item",
-        href: `javascript:ajax_modal('/admin/snapshot-restore/trigger/${trigger.name}')`,
+        href: `javascript:ajax_modal('/admin/snapshot-restore/trigger/${trigger.name}${on_done_redirect_str}')`,
       },
       '<i class="fas fa-undo-alt"></i>&nbsp;' + req.__("Restore")
     ),
     post_dropdown_item(
-      `/actions/clone/${trigger.id}`,
+      `/actions/clone/${trigger.id}${on_done_redirect_str}`,
       '<i class="far fa-copy"></i>&nbsp;' + req.__("Duplicate"),
       req
     ),
@@ -734,11 +734,22 @@ const getTriggerList = async (
       { label: req.__("Name"), key: "name" },
       {
         label: req.__("Test run"),
-        key: (r) => link(`/actions/testrun/${r.id}`, req.__("Test run")),
+        key: (r) =>
+          link(
+            `/actions/testrun/${r.id}${on_done_redirect_str}`,
+            req.__("Test run"),
+            {
+              onclick: `press_store_button(this)`,
+            }
+          ),
       },
       {
         label: req.__("Configure"),
-        key: (r) => link(`/actions/configure/${r.id}`, req.__("Configure")),
+        key: (r) =>
+          link(
+            `/actions/configure/${r.id}${on_done_redirect_str}`,
+            req.__("Configure")
+          ),
       },
       ...(tagId
         ? []

package/routes/infoarch.js

@@ -16,12 +16,16 @@ const {
   send_admin_page,
   config_fields_form,
   save_config_from_form,
+  upload_language_pack,
 } = require("../markup/admin.js");
 const { getState } = require("@saltcorn/data/db/state");
 const { div, a, i, text, button } = require("@saltcorn/markup/tags");
 const { mkTable, renderForm, post_delete_btn } = require("@saltcorn/markup");
 const Form = require("@saltcorn/data/models/form");
 const Snapshot = require("@saltcorn/admin-models/models/snapshot");
+const { stringify } = require("csv-stringify");
+const csvtojson = require("csvtojson");
+const { hasLLM, translate } = require("@saltcorn/data/translate");
 
 /**
  * @type {object}
@@ -175,6 +179,17 @@ router.get(
                       })
                     : "",
               },
+              {
+                label: req.__("Language CSV"),
+                key: (r) =>
+                  a(
+                    {
+                      href: `/site-structure/localizer/download-pack/${r.locale}`,
+                    },
+                    i({ class: "fas fa-download me-2" }),
+                    req.__("Download")
+                  ),
+              },
               {
                 label: req.__("Delete"),
                 key: (r) =>
@@ -187,13 +202,17 @@ router.get(
             ],
             Object.values(cfgLangs)
           ),
-          a(
-            {
-              href: "/site-structure/localizer/add-lang",
-              class: "btn btn-primary mt-1",
-            },
-            i({ class: "fas fa-plus-square me-1" }),
-            req.__("Add language")
+          div(
+            { class: "d-flex mt-1" },
+            a(
+              {
+                href: "/site-structure/localizer/add-lang",
+                class: "btn btn-primary me-2",
+              },
+              i({ class: "fas fa-plus-square me-1" }),
+              req.__("Add language")
+            ),
+            upload_language_pack(req)
           )
         ),
       },
@@ -201,6 +220,75 @@ router.get(
   })
 );
 
+router.get(
+  "/localizer/download-pack/:lang",
+  isAdmin,
+  error_catcher(async (req, res) => {
+    const { lang } = req.params;
+    if (lang === "__proto__" || lang === "constructor") {
+      res.redirect(`/`);
+      return;
+    }
+    const cfgLangs = getState().getConfig("localizer_languages");
+
+    if (!cfgLangs[lang]) {
+      req.flash("error", req.__("Language not found"));
+      return res.redirect(`/site-structure/localizer`);
+    }
+    const default_lang =
+      Object.values(cfgLangs).find((lobj) => lobj.is_default)?.locale ||
+      getState().getConfig("default_locale", "en");
+
+    const cfgStrings = getState().getConfig("localizer_strings", {});
+    const translation = cfgStrings[lang] || {};
+    const strings = getState()
+      .getStringsForI18n()
+      .map((s) => ({ [default_lang]: s, [lang]: translation[s] || s }));
+    res.type("text/csv");
+    res.setHeader("Content-Disposition", `attachment; filename="${lang}.csv"`);
+    res.setHeader("Cache-Control", "no-cache");
+    res.setHeader("Pragma", "no-cache");
+    stringify(strings, {
+      header: true,
+      columns: [default_lang, lang],
+      quoted: true,
+    }).pipe(res);
+  })
+);
+
+router.post(
+  "/localizer/upload-language-pack",
+  isAdmin,
+  setTenant, // TODO why is this needed?????
+  error_catcher(async (req, res) => {
+    if (req.files?.file?.tempFilePath) {
+      const cfgLangs = getState().getConfig("localizer_languages");
+      const default_lang =
+        Object.values(cfgLangs).find((lobj) => lobj.is_default)?.locale ||
+        getState().getConfig("default_locale", "en");
+      const cfgStrings = getState().getConfigCopy("localizer_strings");
+
+      try {
+        const rows = await csvtojson().fromFile(req.files?.file?.tempFilePath);
+        const langs = Object.keys(rows[0]).filter((k) => k !== default_lang);
+        for (const lang of langs)
+          for (const row of rows) {
+            const defstring = row[default_lang];
+            if (cfgStrings[lang]) cfgStrings[lang][defstring] = row[lang];
+            else cfgStrings[lang] = { [defstring]: row[lang] };
+          }
+        await getState().setConfig("localizer_strings", cfgStrings);
+
+        req.flash("success", `Updated languages: ${langs.join(", ")}`);
+      } catch (e) {
+        console.error(e);
+        req.flash("error", e.message);
+      }
+    }
+    res.redirect(`/site-structure/localizer`);
+  })
+);
+
 /**
  * @name get/localizer/add-lang
  * @function
@@ -253,7 +341,21 @@ router.get(
         above: [
           {
             type: "card",
-            contents: [renderForm(form, req.csrfToken())],
+            contents: [
+              renderForm(form, req.csrfToken()),
+              hasLLM() &&
+                renderForm(
+                  new Form({
+                    fields: [],
+                    action: `/site-structure/localizer/translate-llm/${lang}`,
+                    submitLabel: req.__("Translate with LLM"),
+                    onSubmit: "press_store_button(this)",
+
+                    submitButtonClass: "btn-secondary",
+                  }),
+                  req.csrfToken()
+                ),
+            ],
           },
           !is_default && {
             type: "card",
@@ -288,6 +390,55 @@ router.get(
   })
 );
 
+router.post(
+  "/localizer/translate-llm/:lang/",
+  isAdmin,
+  error_catcher(async (req, res) => {
+    const { lang, defstring } = req.params;
+    if (
+      lang === "__proto__" ||
+      defstring === "__proto__" ||
+      lang === "constructor"
+    ) {
+      res.redirect(`/`);
+      return;
+    }
+    const cfgLangs = getState().getConfig("localizer_languages");
+
+    if (!cfgLangs[lang]) {
+      req.flash("error", req.__("Language not found"));
+      return res.redirect(`/site-structure/localizer`);
+    }
+    const default_lang =
+      Object.values(cfgLangs).find((lobj) => lobj.is_default)?.locale ||
+      getState().getConfig("default_locale", "en");
+    let count = 0;
+    for (const defstring of getState().getStringsForI18n()) {
+      const cfgStrings = getState().getConfigCopy("localizer_strings", {});
+      if (
+        cfgStrings[lang][defstring] &&
+        cfgStrings[lang][defstring] !== defstring
+      )
+        continue;
+      if (count >= 20) break;
+      count += 1;
+      const translated = await translate(defstring, lang, default_lang);
+      if (cfgStrings[lang]) cfgStrings[lang][defstring] = translated;
+      else cfgStrings[lang] = { [defstring]: translated };
+      await getState().setConfig("localizer_strings", cfgStrings);
+    }
+    if (count == 20)
+      req.flash(
+        "success",
+        req.__(
+          `Translated %s strings. Click 'Translate with LLM' again to continue`, count
+        )
+      );
+    else req.flash("success", req.__(`Finished translating %s strings.`, count));
+    res.redirect(`/site-structure/localizer/edit/${lang}`);
+  })
+);
+
 /**
  * @name post/localizer/save-string/:lang/:defstring
  * @function

package/routes/list.js

@@ -19,6 +19,7 @@ const {
   i,
   text,
   button,
+  a,
   input,
   label,
   form,
@@ -31,6 +32,7 @@ const {
 } = require("./utils");
 const moment = require("moment");
 const { getState } = require("@saltcorn/data/db/state");
+const { comparingCaseInsensitive } = require("@saltcorn/data/utils");
 
 /**
  * @type {object}
@@ -127,19 +129,26 @@ router.post(
 const typeToGridType = (t, field) => {
   const jsgField = { field: field.name, title: field.label, editor: true };
   if (t.name === "String" && field.attributes && field.attributes.options) {
-    jsgField.editor = "select";
+    jsgField.editor = "list";
 
     const values = field.attributes.options.split(",").map((o) => o.trim());
     if (!field.required) values.unshift("");
 
     jsgField.editorParams = { values };
   } else if (t === "Key" || t === "File") {
-    jsgField.editor = "select";
-    const values = {};
-
-    field.options.forEach(({ label, value }) => (values[value] = label));
+    jsgField.editor = "list";
+    const values = [];
+    const valuesObj = {};
+    field.options
+      .sort(comparingCaseInsensitive("label"))
+      .forEach(({ label, value }) => {
+        //en space. tabulator workaround
+        const l = label === "" ? "\u2002" : label;
+        values.push({ label: l, value });
+        valuesObj[value] = l;
+      });
     jsgField.editorParams = { values };
-    jsgField.formatterParams = { values };
+    jsgField.formatterParams = { values: valuesObj };
     jsgField.formatter = "__lookupIntToString";
   } else if (t.name === "Float" || t.name === "Integer") {
     jsgField.editor = "number";
@@ -373,6 +382,21 @@ router.get(
                   },
                   form(
                     { class: "px-2" },
+                    a(
+                      {
+                        onclick: `event.stopPropagation();allnonecols(true,this)`,
+                        href: "javascript:;",
+                      },
+                      "All"
+                    ),
+                    " | ",
+                    a(
+                      {
+                        onclick: `event.stopPropagation();allnonecols(false,this)`,
+                        href: "javascript:;",
+                      },
+                      "None"
+                    ),
                     fields.map((f) =>
                       div(
                         { class: "form-check" },
@@ -403,7 +427,7 @@ router.get(
                   if(typeof v === "string" && v.startsWith("__"))
                     col[k] = window[v.substring(2)];
                 })
-              })   
+              })
               window.tabulator_table_primary_key = "${table.pk_name}";
               window.tabulator_table = new Tabulator("#jsGrid", {
                   ajaxURL:"/api/${encodeURIComponent(
@@ -411,22 +435,34 @@ router.get(
                   )}?tabulator_pagination_format=true${
                     table.versioned ? "&versioncount=on" : ""
                   }",                   
-                  layout:"fitColumns", 
+                  layout:"fitData", 
                   columns,
                   height:"100%",
                   pagination:true,
                   paginationMode:"remote",
                   paginationSize:20,
-                  clipboard:true,
-                  persistence:true, 
-                  persistenceID:"table_tab_${table.name}",
+                  clipboard:true,                 
                   movableColumns: true,
                   ajaxContentType:"json",
                   sortMode:"remote",
+                  resizableColumnGuide:true,                  
+                  columnDefaults:{
+                      resizable:true,
+                      maxWidth:500
+                  },
                   initialSort:[
                     {column:"${table.pk_name}", dir:"asc"},
                   ],                 
               });
+              window.allnonecols= (do_show, e) =>{
+                columns.forEach(col=>{
+                  if(col.frozen && !do_show) return;
+                  if (do_show) window.tabulator_table.showColumn(col.field);
+                  else window.tabulator_table.hideColumn(col.field);
+                  $(e).closest("form").find("input").prop("checked", do_show)
+                })            
+              }
+
               window.tabulator_table.on("cellEdited", function(cell){
                 const row = cell.getRow().getData()
                 const fieldName = cell.getColumn().getField()