@saltcorn/server 0.8.8-beta.1 → 0.8.8-beta.3

package/load_plugins.js

@@ -187,7 +187,14 @@ const loadAndSaveNewPlugin = async (plugin, force, noSignalOrDB) => {
   if (!isRoot && !tenants_unsafe_plugins) {
     if (plugin.source !== "npm") return;
     //get allowed plugins
-    const instore = await Plugin.store_plugins_available();
+
+    //refresh root store
+    await db.runWithTenant(
+      db.connectObj.default_schema,
+      async () => await Plugin.store_plugins_available()
+    );
+
+    const instore = getRootState().getConfig("available_plugins", []);
     const safes = instore.filter((p) => !p.unsafe).map((p) => p.location);
     if (!safes.includes(plugin.location)) return;
   }

package/locales/en.json

@@ -1237,5 +1237,10 @@
 	"Formula for JavaScript object that will be added to state parameters": "Formula for JavaScript object that will be added to state parameters",
 	"Alignment": "Alignment",
 	"Click to edit?": "Click to edit?",
-	"Format": "Format"
+	"Format": "Format",
+	"Table Synchronization": "Table Synchronization",
+	"unsynched": "unsynched",
+	"synched": "synched",
+	"Sync information": "Sync information",
+	"Sync information tracks the last modification or deletion timestamp so that the table data can be synchronized with the mobile app": "Sync information tracks the last modification or deletion timestamp so that the table data can be synchronized with the mobile app"
 }

package/locales/ru.json

@@ -1095,5 +1095,8 @@
 	"Backup settings": "Настройки резервного копироваия",
 	"Click to edit?": "Редактировать по клику?",
 	"Format": "Формат",
-	"Page '%s' was loaded": "Страница '%s' была загружена"
+	"Page '%s' was loaded": "Страница '%s' была загружена",
+	"Table provider": "Провайдер данных",
+	"Database table": "Таблица БД",
+	"Disable on mobile": "Disable on mobile"
 }

package/package.json

@@ -1,18 +1,18 @@
 {
   "name": "@saltcorn/server",
-  "version": "0.8.8-beta.1",
+  "version": "0.8.8-beta.3",
   "description": "Server app for Saltcorn, open-source no-code platform",
   "homepage": "https://saltcorn.com",
   "main": "index.js",
   "license": "MIT",
   "dependencies": {
-    "@saltcorn/base-plugin": "0.8.8-beta.1",
-    "@saltcorn/builder": "0.8.8-beta.1",
-    "@saltcorn/data": "0.8.8-beta.1",
-    "@saltcorn/admin-models": "0.8.8-beta.1",
-    "@saltcorn/filemanager": "0.8.8-beta.1",
-    "@saltcorn/markup": "0.8.8-beta.1",
-    "@saltcorn/sbadmin2": "0.8.8-beta.1",
+    "@saltcorn/base-plugin": "0.8.8-beta.3",
+    "@saltcorn/builder": "0.8.8-beta.3",
+    "@saltcorn/data": "0.8.8-beta.3",
+    "@saltcorn/admin-models": "0.8.8-beta.3",
+    "@saltcorn/filemanager": "0.8.8-beta.3",
+    "@saltcorn/markup": "0.8.8-beta.3",
+    "@saltcorn/sbadmin2": "0.8.8-beta.3",
     "@socket.io/cluster-adapter": "^0.2.1",
     "@socket.io/sticky": "^1.0.1",
     "adm-zip": "0.5.10",

package/public/saltcorn.js

@@ -282,6 +282,10 @@ function ensure_modal_exists_and_closed() {
     </div>
   </div>`);
   } else if ($("#scmodal").hasClass("show")) {
+    // remove reload handler added by edit, for when we have popup link
+    // in autosave edit in popup
+    $("#scmodal").off("hidden.bs.modal");
+
     close_saltcorn_modal();
   }
 }
@@ -319,7 +323,9 @@ function ajax_modal(url, opts = {}) {
       if (title) $("#scmodal .modal-title").html(decodeURIComponent(title));
       $("#scmodal .modal-body").html(res);
       $("#scmodal").prop("data-modal-state", url);
-      new bootstrap.Modal($("#scmodal")).show();
+      new bootstrap.Modal($("#scmodal"), {
+        focus: false,
+      }).show();
       initialize_page();
       (opts.onOpen || function () {})(res);
       $("#scmodal").on("hidden.bs.modal", function (e) {
@@ -669,6 +675,9 @@ function build_mobile_app(button) {
   form.serializeArray().forEach((item) => {
     params[item.name] = item.value;
   });
+  params.synchedTables = Array.from($("#synched-tbls-select-id")[0].options)
+    .filter((option) => !option.hidden)
+    .map((option) => option.value);
   ajax_post("/admin/build-mobile-app", {
     data: params,
     success: (data) => {
@@ -682,6 +691,40 @@ function build_mobile_app(button) {
   });
 }
 
+function move_to_synched() {
+  const opts = $("#unsynched-tbls-select-id");
+  $("#synched-tbls-select-id").removeAttr("selected");
+  for (const selected of opts.val()) {
+    const jUnsOpt = $(`[id='${selected}_unsynched_opt']`);
+    jUnsOpt.attr("hidden", "true");
+    jUnsOpt.removeAttr("selected");
+    const jSynOpt = $(`[id='${selected}_synched_opt']`);
+    jSynOpt.removeAttr("hidden");
+    jSynOpt.removeAttr("selected");
+  }
+}
+
+function move_to_unsynched() {
+  const opts = $("#synched-tbls-select-id");
+  $("#unsynched-tbls-select-id").removeAttr("selected");
+  for (const selected of opts.val()) {
+    const jSynOpt = $(`[id='${selected}_synched_opt']`);
+    jSynOpt.attr("hidden", "true");
+    jSynOpt.removeAttr("selected");
+    const jUnsOpt = $(`[id='${selected}_unsynched_opt']`);
+    jUnsOpt.removeAttr("hidden");
+    jUnsOpt.removeAttr("selected");
+  }
+}
+
+function toggle_tbl_sync() {
+  if ($("#offlineModeBoxId")[0].checked === true) {
+    $("#tblSyncSelectorId").attr("hidden", false);
+  } else {
+    $("#tblSyncSelectorId").attr("hidden", true);
+  }
+}
+
 function join_field_clicked(e, fieldPath) {
   $("#inputjoin_field").val(fieldPath);
   apply_showif();

package/routes/admin.js

@@ -1491,7 +1491,7 @@ router.get(
     const images = (await File.find({ mime_super: "image" })).filter((image) =>
       image.filename?.endsWith(".png")
     );
-
+    const withSyncInfo = await Table.find({ has_sync_info: true });
     send_admin_page({
       res,
       req,
@@ -1752,7 +1752,6 @@ router.get(
                   ),
 
                   div(
-                    // TODO only for some tables?
                     { class: "row pb-2" },
                     div(
                       { class: "col-sm-4" },
@@ -1761,6 +1760,8 @@ router.get(
                         id: "offlineModeBoxId",
                         class: "form-check-input me-2",
                         name: "allowOfflineMode",
+                        value: "allowOfflineMode",
+                        onClick: "toggle_tbl_sync()",
                         checked: true,
                       }),
                       label(
@@ -1771,6 +1772,98 @@ router.get(
                         req.__("Allow offline mode")
                       )
                     )
+                  ),
+                  div(
+                    {
+                      id: "tblSyncSelectorId",
+                      class: "row pb-2",
+                    },
+                    div(
+                      label(
+                        { class: "form-label fw-bold" },
+                        req.__("Table Synchronization")
+                      )
+                    ),
+                    div(
+                      { class: "container" },
+                      div(
+                        { class: "row" },
+                        div(
+                          { class: "col-sm-4 text-center" },
+                          req.__("unsynched")
+                        ),
+                        div({ class: "col-sm-1" }),
+                        div(
+                          { class: "col-sm-4 text-center" },
+                          req.__("synched")
+                        )
+                      ),
+                      div(
+                        { class: "row" },
+                        div(
+                          { class: "col-sm-4" },
+                          select(
+                            {
+                              id: "unsynched-tbls-select-id",
+                              class: "form-control form-select",
+                              multiple: true,
+                            },
+                            withSyncInfo.map((table) =>
+                              option({
+                                id: `${table.name}_unsynched_opt`,
+                                value: table.name,
+                                label: table.name,
+                              })
+                            )
+                          )
+                        ),
+                        div(
+                          { class: "col-sm-1 d-flex justify-content-center" },
+                          div(
+                            div(
+                              button(
+                                {
+                                  id: "move-right-btn-id",
+                                  type: "button",
+                                  onClick: `move_to_synched()`,
+                                  class: "btn btn-light pt-1 mb-1",
+                                },
+                                i({ class: "fas fa-arrow-right" })
+                              )
+                            ),
+                            div(
+                              button(
+                                {
+                                  id: "move-left-btn-id",
+                                  type: "button",
+                                  onClick: `move_to_unsynched()`,
+                                  class: "btn btn-light pt-1",
+                                },
+                                i({ class: "fas fa-arrow-left" })
+                              )
+                            )
+                          )
+                        ),
+                        div(
+                          { class: "col-sm-4" },
+                          select(
+                            {
+                              id: "synched-tbls-select-id",
+                              class: "form-control form-select",
+                              multiple: true,
+                            },
+                            withSyncInfo.map((table) =>
+                              option({
+                                id: `${table.name}_synched_opt`,
+                                value: table.name,
+                                label: table.name,
+                                hidden: "true",
+                              })
+                            )
+                          )
+                        )
+                      )
+                    )
                   )
                 ),
                 button(
@@ -1877,6 +1970,7 @@ router.post(
       serverURL,
       splashPage,
       allowOfflineMode,
+      synchedTables,
     } = req.body;
     if (!androidPlatform && !iOSPlatform) {
       return res.json({
@@ -1928,6 +2022,8 @@ router.post(
     if (serverURL) spawnParams.push("-s", serverURL);
     if (splashPage) spawnParams.push("--splashPage", splashPage);
     if (allowOfflineMode) spawnParams.push("--allowOfflineMode");
+    if (synchedTables?.length > 0)
+      spawnParams.push("--synchedTables", ...synchedTables.map((tbl) => tbl));
     if (
       db.is_it_multi_tenant() &&
       db.getTenantSchema() !== db.connectObj.default_schema

package/routes/fields.js

@@ -542,14 +542,6 @@ const fieldFlow = (req) =>
                 type: "Bool",
                 showIf: { summary_field: textfields },
               }),
-              /*new Field({
-                name: "on_delete_cascade",
-                label: req.__("On delete cascade"),
-                type: "Bool",
-                sublabel: req.__(
-                  "If the parent row is deleted, automatically delete the child rows."
-                ),
-              }),*/
               new Field({
                 name: "on_delete",
                 label: req.__("On delete"),

package/routes/scapi.js

@@ -101,7 +101,7 @@ router.get(
       { session: false },
       async function (err, user, info) {
         if (accessAllowedRead(req, user)) {
-          const views = await View.find({});
+          const views = await View.find({}, { cached: true });
 
           res.json({ success: views });
         } else {

package/routes/sync.js

@@ -3,83 +3,319 @@ const Router = require("express-promise-router");
 const db = require("@saltcorn/data/db");
 const { getState } = require("@saltcorn/data/db/state");
 const Table = require("@saltcorn/data/models/table");
+const File = require("@saltcorn/data/models/file");
+const { getSafeSaltcornCmd } = require("@saltcorn/data/utils");
+const { spawn, spawnSync } = require("child_process");
+const path = require("path");
+const fs = require("fs").promises;
 
 const router = new Router();
 module.exports = router;
 
-const pickFields = (table, row) => {
-  const result = {};
-  const fields = table.getFields();
-  for (const { name, type, calculated } of table.getFields()) {
-    if (name === "id" || calculated) continue;
-    if (type?.name === "Date") {
-      result[name] = row[name] ? new Date(row[name]) : undefined;
-    } else {
-      result[name] = row[name];
+router.get(
+  "/sync_timestamp",
+  error_catcher(async (req, res) => {
+    try {
+      res.json({ syncTimestamp: (await db.time()).valueOf() });
+    } catch (error) {
+      getState().log(2, `GET /sync_timestamp: '${error.message}'`);
+      res.status(400).json({ error: error.message || error });
+    }
+  })
+);
+
+const getSyncRows = async (syncInfo, table, syncUntil, client) => {
+  const tblName = table.name;
+  const pkName = table.pk_name;
+  const schema = db.getTenantSchemaPrefix();
+  if (!syncInfo.syncFrom) {
+    const { rows } = await client.query(
+      `select 
+         info_tbl.ref "_sync_info_tbl_ref_", 
+         info_tbl.last_modified "_sync_info_tbl_last_modified_", 
+         info_tbl.deleted "_sync_info_tbl_deleted_",
+         data_tbl.*
+       from ${schema}"${db.sqlsanitize(
+        tblName
+      )}_sync_info" "info_tbl" right join "${db.sqlsanitize(
+        tblName
+      )}" "data_tbl"
+      on info_tbl.ref = data_tbl."${db.sqlsanitize(
+        pkName
+      )}" and info_tbl.deleted = false
+      where data_tbl."${db.sqlsanitize(pkName)}" > ${
+        syncInfo.maxLoadedId
+      } order by data_tbl."${db.sqlsanitize(pkName)}"`
+    );
+    for (const row of rows) {
+      if (row._sync_info_tbl_last_modified_)
+        row._sync_info_tbl_last_modified_ =
+          row._sync_info_tbl_last_modified_.valueOf();
+      else row._sync_info_tbl_last_modified_ = new Date(syncUntil).valueOf();
+      row._sync_info_tbl_ref_ = row[pkName];
     }
+    return rows;
+  } else {
+    const { rows } = await client.query(
+      `select 
+         info_tbl.ref "_sync_info_tbl_ref_", 
+         info_tbl.last_modified "_sync_info_tbl_last_modified_", 
+         info_tbl.deleted "_sync_info_tbl_deleted_",
+         data_tbl.*
+       from ${schema}"${db.sqlsanitize(
+        tblName
+      )}_sync_info" "info_tbl" join ${schema}"${db.sqlsanitize(
+        tblName
+      )}" "data_tbl"
+      on info_tbl.ref = data_tbl."${db.sqlsanitize(pkName)}"
+      where date_trunc('milliseconds', info_tbl.last_modified) > to_timestamp(${
+        new Date(syncInfo.syncFrom).valueOf() / 1000.0
+      }) 
+      and date_trunc('milliseconds', info_tbl.last_modified) < to_timestamp(${
+        new Date(syncUntil).valueOf() / 1000.0
+      }) 
+      and info_tbl.deleted = false
+      and info_tbl.ref > ${syncInfo.maxLoadedId}
+      order by info_tbl.ref`
+    );
+    for (const row of rows) {
+      if (row._sync_info_tbl_last_modified_)
+        row._sync_info_tbl_last_modified_ =
+          row._sync_info_tbl_last_modified_.valueOf();
+      else row._sync_info_tbl_last_modified_ = syncUntil.valueOf();
+    }
+    return rows;
   }
-  return result;
 };
 
-const allowInsert = (table, user) => {
-  const role = user?.role_id || 100;
-  return table.min_role_write >= role;
-};
+/*
+  load inserts/updates after syncFrom
+  If a table has no syncFrom then it's the first sync and we have to send everything
+*/
+router.post(
+  "/load_changes",
+  error_catcher(async (req, res) => {
+    const result = {};
+    const { syncInfos, loadUntil } = req.body;
+    if (!loadUntil) {
+      getState().log(2, `POST /load_changes: loadUntil is missing`);
+      return res.status(400).json({ error: "loadUntil is missing" });
+    }
+    if (!syncInfos) {
+      getState().log(2, `POST /load_changes: syncInfos is missing`);
+      return res.status(400).json({ error: "syncInfos is missing" });
+    }
+    const role = req.user ? req.user.role_id : 100;
+    const client = await db.getClient();
+    let rowLimit = 1000;
+    try {
+      await client.query(`BEGIN`);
+      for (const [tblName, syncInfo] of Object.entries(syncInfos)) {
+        const table = Table.findOne({ name: tblName });
+        if (!table) throw new Error(`The table '${tblName}' does not exists`);
+        const pkName = table.pk_name;
+        let rows = await getSyncRows(syncInfo, table, loadUntil, client);
+        if (role > table.min_role_read) {
+          if (
+            role === 100 ||
+            (!table.ownership_field_id && !table.ownership_formula)
+          )
+            continue;
+          else if (table.ownership_field_id) {
+          } else if (table.ownership_formula) {
+            rows = rows.filter((row) => table.is_owner(req.user, row));
+          }
+        }
+        if (rows.length > rowLimit) {
+          rows.splice(rowLimit);
+        }
+        rowLimit -= rows.length;
+        result[tblName] = {
+          rows,
+          maxLoadedId: rows.length > 0 ? rows[rows.length - 1][pkName] : 0,
+        };
+      }
+      await client.query("COMMIT");
+      res.json(result);
+    } catch (error) {
+      await client.query("ROLLBACK");
+      getState().log(2, `POST /load_changes: '${error.message}'`);
+      res.status(400).json({ error: error.message || error });
+    } finally {
+      client.release(true);
+    }
+  })
+);
 
-const throwWithCode = (message, code) => {
-  const err = new Error(message);
-  err.statusCode = code;
-  throw err;
+const getDelRows = async (tblName, syncFrom, syncUntil, client) => {
+  const schema = db.getTenantSchemaPrefix();
+  const dbRes = await client.query(
+    `select * 
+     from (
+      select ref, max(last_modified) from ${schema}"${db.sqlsanitize(
+      tblName
+    )}_sync_info" 
+      group by ref, deleted having deleted = true) as alias 
+      where alias.max < to_timestamp(${syncUntil.valueOf() / 1000.0}) 
+        and alias.max > to_timestamp(${syncFrom.valueOf() / 1000.0})`
+  );
+  for (const row of dbRes.rows) {
+    if (row.last_modified) row.last_modified = row.last_modified.valueOf();
+    if (row.max) row.max = row.max.valueOf();
+  }
+  return dbRes.rows;
 };
 
-/**
- * insert the offline data uploaded by the mobile-app
- */
+/*
+  load deletes after syncFrom
+  If a table has no syncFrom then it's the first sync and there is nothing to delete
+*/
 router.post(
-  "/table_data",
+  "/deletes",
   error_catcher(async (req, res) => {
-    // TODO sqlite
-    getState().log(
-      4,
-      `POST /sync/table_data user: '${req.user ? req.user.id : "public"}'`
-    );
-    let aborted = false;
-    req.socket.on("close", () => {
-      aborted = true;
-    });
-    req.socket.on("timeout", () => {
-      aborted = true;
-    });
-    const client = db.isSQLite ? db : await db.getClient();
+    const { syncInfos, syncTimestamp } = req.body;
+    const client = await db.getClient();
     try {
-      await client.query("BEGIN");
-      await client.query("SET CONSTRAINTS ALL DEFERRED");
-      for (const [tblName, offlineRows] of Object.entries(req.body.data) ||
-        []) {
-        const table = Table.findOne({ name: tblName });
-        if (!table) throw new Error(`The table '${tblName}' does not exist.`);
-        if (!allowInsert(table, req.user))
-          throwWithCode(req.__("Not authorized"), 401);
-        if (tblName !== "users") {
-          for (const newRow of offlineRows.map((row) =>
-            pickFields(table, row)
-          )) {
-            if (aborted) throw new Error("connection closed by client");
-            await db.insert(table.name, newRow, { client: client });
-          }
+      await client.query(`BEGIN`);
+      const syncUntil = new Date(syncTimestamp);
+      const result = {
+        deletes: {},
+      };
+      for (const [tblName, syncInfo] of Object.entries(syncInfos)) {
+        if (syncInfo.syncFrom) {
+          result.deletes[tblName] = await getDelRows(
+            tblName,
+            new Date(syncInfo.syncFrom),
+            syncUntil,
+            client
+          );
         }
       }
-      if (aborted) throw new Error("connection closed by client");
       await client.query("COMMIT");
-      res.json({ success: true });
+      res.json(result);
     } catch (error) {
       await client.query("ROLLBACK");
-      getState().log(2, `POST /sync/table_data error: '${error.message}'`);
-      res
-        .status(error.statusCode || 400)
-        .json({ error: error.message || error });
+      getState().log(2, `POST /sync/deletes: '${error.message}'`);
+      res.status(400).json({ error: error.message || error });
     } finally {
-      if (!db.isSQLite) await client.release(true);
+      client.release(true);
+    }
+  })
+);
+
+/*
+  insert the app offline data
+*/
+router.post(
+  "/offline_changes",
+  error_catcher(async (req, res) => {
+    const { changes, syncTimestamp } = req.body;
+    const rootFolder = await File.rootFolder();
+    try {
+      const syncDirName = `${syncTimestamp}_${req.user?.email || "public"}`;
+      const syncDir = path.join(
+        rootFolder.location,
+        "mobile_app",
+        "sync",
+        syncDirName
+      );
+      await fs.mkdir(syncDir, { recursive: true });
+      await fs.writeFile(
+        path.join(syncDir, "changes.json"),
+        JSON.stringify(changes)
+      );
+      const spawnParams = ["sync-upload-data"];
+      if (req.user?.email) spawnParams.push("--userEmail", req.user.email);
+      spawnParams.push("--directory", syncDir);
+      if (
+        db.is_it_multi_tenant() &&
+        db.getTenantSchema() !== db.connectObj.default_schema
+      ) {
+        spawnParams.push("--tenantAppName", db.getTenantSchema());
+      }
+      spawnParams.push("--syncTimestamp", syncTimestamp);
+
+      res.json({ syncDir: syncDirName });
+      const child = spawn(getSafeSaltcornCmd(), spawnParams, {
+        stdio: ["pipe", "pipe", "pipe"],
+        cwd: ".",
+      });
+
+      child.on("exit", async (exitCode, signal) => {
+        getState().log(
+          5,
+          `POST /sync/offline_changes: upload offline data finished with code: ${exitCode}`
+        );
+      });
+      child.on("error", (msg) => {
+        const message = msg.message ? msg.message : msg.code;
+        getState().log(
+          5,
+          `POST /sync/offline_changes: upload offline data failed: ${message}`
+        );
+      });
+    } catch (error) {
+      getState().log(2, `POST /sync/offline_changes: '${error.message}'`);
+      res.status(400).json({ error: error.message || error });
+    }
+  })
+);
+
+router.get(
+  "/upload_finished",
+  error_catcher(async (req, res) => {
+    const { dir_name } = req.query;
+    try {
+      const rootFolder = await File.rootFolder();
+      const syncDir = path.join(
+        rootFolder.location,
+        "mobile_app",
+        "sync",
+        dir_name
+      );
+      let entries = null;
+      try {
+        entries = await fs.readdir(syncDir);
+      } catch (error) {
+        return res.json({ finished: false });
+      }
+      if (entries.indexOf("translated-ids.json") >= 0) {
+        const translatedIds = JSON.parse(
+          await fs.readFile(path.join(syncDir, "translated-ids.json"))
+        );
+        res.json({ finished: true, translatedIds });
+      } else if (entries.indexOf("error.json") >= 0) {
+        const error = JSON.parse(
+          await fs.readFile(path.join(syncDir, "error.json"))
+        );
+        res.json({ finished: true, error });
+      } else {
+        res.json({ finished: false });
+      }
+    } catch (error) {
+      getState().log(2, `GET /sync/upload_finished: '${error.message}'`);
+      res.status(400).json({ error: error.message || error });
+    }
+  })
+);
+
+router.post(
+  "/clean_sync_dir",
+  error_catcher(async (req, res) => {
+    const { dir_name } = req.body;
+    try {
+      const rootFolder = await File.rootFolder();
+      const syncDir = path.join(
+        rootFolder.location,
+        "mobile_app",
+        "sync",
+        dir_name
+      );
+      await fs.rm(syncDir, { recursive: true, force: true });
+      res.status(200).send("");
+    } catch (error) {
+      getState().log(2, `POST /sync/clean_sync_dir: '${error.message}'`);
+      res.status(400).json({ error: error.message || error });
     }
   })
 );