@saltcorn/server 0.8.8-beta.0 → 0.8.8-beta.2

package/routes/sync.js

@@ -3,83 +3,319 @@ const Router = require("express-promise-router");
 const db = require("@saltcorn/data/db");
 const { getState } = require("@saltcorn/data/db/state");
 const Table = require("@saltcorn/data/models/table");
+const File = require("@saltcorn/data/models/file");
+const { getSafeSaltcornCmd } = require("@saltcorn/data/utils");
+const { spawn, spawnSync } = require("child_process");
+const path = require("path");
+const fs = require("fs").promises;
 
 const router = new Router();
 module.exports = router;
 
-const pickFields = (table, row) => {
-  const result = {};
-  const fields = table.getFields();
-  for (const { name, type, calculated } of table.getFields()) {
-    if (name === "id" || calculated) continue;
-    if (type?.name === "Date") {
-      result[name] = row[name] ? new Date(row[name]) : undefined;
-    } else {
-      result[name] = row[name];
+router.get(
+  "/sync_timestamp",
+  error_catcher(async (req, res) => {
+    try {
+      res.json({ syncTimestamp: (await db.time()).valueOf() });
+    } catch (error) {
+      getState().log(2, `GET /sync_timestamp: '${error.message}'`);
+      res.status(400).json({ error: error.message || error });
+    }
+  })
+);
+
+const getSyncRows = async (syncInfo, table, syncUntil, client) => {
+  const tblName = table.name;
+  const pkName = table.pk_name;
+  const schema = db.getTenantSchemaPrefix();
+  if (!syncInfo.syncFrom) {
+    const { rows } = await client.query(
+      `select 
+         info_tbl.ref "_sync_info_tbl_ref_", 
+         info_tbl.last_modified "_sync_info_tbl_last_modified_", 
+         info_tbl.deleted "_sync_info_tbl_deleted_",
+         data_tbl.*
+       from ${schema}"${db.sqlsanitize(
+        tblName
+      )}_sync_info" "info_tbl" right join "${db.sqlsanitize(
+        tblName
+      )}" "data_tbl"
+      on info_tbl.ref = data_tbl."${db.sqlsanitize(
+        pkName
+      )}" and info_tbl.deleted = false
+      where data_tbl."${db.sqlsanitize(pkName)}" > ${
+        syncInfo.maxLoadedId
+      } order by data_tbl."${db.sqlsanitize(pkName)}"`
+    );
+    for (const row of rows) {
+      if (row._sync_info_tbl_last_modified_)
+        row._sync_info_tbl_last_modified_ =
+          row._sync_info_tbl_last_modified_.valueOf();
+      else row._sync_info_tbl_last_modified_ = new Date(syncUntil).valueOf();
+      row._sync_info_tbl_ref_ = row[pkName];
     }
+    return rows;
+  } else {
+    const { rows } = await client.query(
+      `select 
+         info_tbl.ref "_sync_info_tbl_ref_", 
+         info_tbl.last_modified "_sync_info_tbl_last_modified_", 
+         info_tbl.deleted "_sync_info_tbl_deleted_",
+         data_tbl.*
+       from ${schema}"${db.sqlsanitize(
+        tblName
+      )}_sync_info" "info_tbl" join ${schema}"${db.sqlsanitize(
+        tblName
+      )}" "data_tbl"
+      on info_tbl.ref = data_tbl."${db.sqlsanitize(pkName)}"
+      where date_trunc('milliseconds', info_tbl.last_modified) > to_timestamp(${
+        new Date(syncInfo.syncFrom).valueOf() / 1000.0
+      }) 
+      and date_trunc('milliseconds', info_tbl.last_modified) < to_timestamp(${
+        new Date(syncUntil).valueOf() / 1000.0
+      }) 
+      and info_tbl.deleted = false
+      and info_tbl.ref > ${syncInfo.maxLoadedId}
+      order by info_tbl.ref`
+    );
+    for (const row of rows) {
+      if (row._sync_info_tbl_last_modified_)
+        row._sync_info_tbl_last_modified_ =
+          row._sync_info_tbl_last_modified_.valueOf();
+      else row._sync_info_tbl_last_modified_ = syncUntil.valueOf();
+    }
+    return rows;
   }
-  return result;
 };
 
-const allowInsert = (table, user) => {
-  const role = user?.role_id || 100;
-  return table.min_role_write >= role;
-};
+/*
+  load inserts/updates after syncFrom
+  If a table has no syncFrom then it's the first sync and we have to send everything
+*/
+router.post(
+  "/load_changes",
+  error_catcher(async (req, res) => {
+    const result = {};
+    const { syncInfos, loadUntil } = req.body;
+    if (!loadUntil) {
+      getState().log(2, `POST /load_changes: loadUntil is missing`);
+      return res.status(400).json({ error: "loadUntil is missing" });
+    }
+    if (!syncInfos) {
+      getState().log(2, `POST /load_changes: syncInfos is missing`);
+      return res.status(400).json({ error: "syncInfos is missing" });
+    }
+    const role = req.user ? req.user.role_id : 100;
+    const client = await db.getClient();
+    let rowLimit = 1000;
+    try {
+      await client.query(`BEGIN`);
+      for (const [tblName, syncInfo] of Object.entries(syncInfos)) {
+        const table = Table.findOne({ name: tblName });
+        if (!table) throw new Error(`The table '${tblName}' does not exists`);
+        const pkName = table.pk_name;
+        let rows = await getSyncRows(syncInfo, table, loadUntil, client);
+        if (role > table.min_role_read) {
+          if (
+            role === 100 ||
+            (!table.ownership_field_id && !table.ownership_formula)
+          )
+            continue;
+          else if (table.ownership_field_id) {
+          } else if (table.ownership_formula) {
+            rows = rows.filter((row) => table.is_owner(req.user, row));
+          }
+        }
+        if (rows.length > rowLimit) {
+          rows.splice(rowLimit);
+        }
+        rowLimit -= rows.length;
+        result[tblName] = {
+          rows,
+          maxLoadedId: rows.length > 0 ? rows[rows.length - 1][pkName] : 0,
+        };
+      }
+      await client.query("COMMIT");
+      res.json(result);
+    } catch (error) {
+      await client.query("ROLLBACK");
+      getState().log(2, `POST /load_changes: '${error.message}'`);
+      res.status(400).json({ error: error.message || error });
+    } finally {
+      client.release(true);
+    }
+  })
+);
 
-const throwWithCode = (message, code) => {
-  const err = new Error(message);
-  err.statusCode = code;
-  throw err;
+const getDelRows = async (tblName, syncFrom, syncUntil, client) => {
+  const schema = db.getTenantSchemaPrefix();
+  const dbRes = await client.query(
+    `select * 
+     from (
+      select ref, max(last_modified) from ${schema}"${db.sqlsanitize(
+      tblName
+    )}_sync_info" 
+      group by ref, deleted having deleted = true) as alias 
+      where alias.max < to_timestamp(${syncUntil.valueOf() / 1000.0}) 
+        and alias.max > to_timestamp(${syncFrom.valueOf() / 1000.0})`
+  );
+  for (const row of dbRes.rows) {
+    if (row.last_modified) row.last_modified = row.last_modified.valueOf();
+    if (row.max) row.max = row.max.valueOf();
+  }
+  return dbRes.rows;
 };
 
-/**
- * insert the offline data uploaded by the mobile-app
- */
+/*
+  load deletes after syncFrom
+  If a table has no syncFrom then it's the first sync and there is nothing to delete
+*/
 router.post(
-  "/table_data",
+  "/deletes",
   error_catcher(async (req, res) => {
-    // TODO sqlite
-    getState().log(
-      4,
-      `POST /sync/table_data user: '${req.user ? req.user.id : "public"}'`
-    );
-    let aborted = false;
-    req.socket.on("close", () => {
-      aborted = true;
-    });
-    req.socket.on("timeout", () => {
-      aborted = true;
-    });
-    const client = db.isSQLite ? db : await db.getClient();
+    const { syncInfos, syncTimestamp } = req.body;
+    const client = await db.getClient();
     try {
-      await client.query("BEGIN");
-      await client.query("SET CONSTRAINTS ALL DEFERRED");
-      for (const [tblName, offlineRows] of Object.entries(req.body.data) ||
-        []) {
-        const table = Table.findOne({ name: tblName });
-        if (!table) throw new Error(`The table '${tblName}' does not exist.`);
-        if (!allowInsert(table, req.user))
-          throwWithCode(req.__("Not authorized"), 401);
-        if (tblName !== "users") {
-          for (const newRow of offlineRows.map((row) =>
-            pickFields(table, row)
-          )) {
-            if (aborted) throw new Error("connection closed by client");
-            await db.insert(table.name, newRow, { client: client });
-          }
+      await client.query(`BEGIN`);
+      const syncUntil = new Date(syncTimestamp);
+      const result = {
+        deletes: {},
+      };
+      for (const [tblName, syncInfo] of Object.entries(syncInfos)) {
+        if (syncInfo.syncFrom) {
+          result.deletes[tblName] = await getDelRows(
+            tblName,
+            new Date(syncInfo.syncFrom),
+            syncUntil,
+            client
+          );
         }
       }
-      if (aborted) throw new Error("connection closed by client");
       await client.query("COMMIT");
-      res.json({ success: true });
+      res.json(result);
     } catch (error) {
       await client.query("ROLLBACK");
-      getState().log(2, `POST /sync/table_data error: '${error.message}'`);
-      res
-        .status(error.statusCode || 400)
-        .json({ error: error.message || error });
+      getState().log(2, `POST /sync/deletes: '${error.message}'`);
+      res.status(400).json({ error: error.message || error });
     } finally {
-      if (!db.isSQLite) await client.release(true);
+      client.release(true);
+    }
+  })
+);
+
+/*
+  insert the app offline data
+*/
+router.post(
+  "/offline_changes",
+  error_catcher(async (req, res) => {
+    const { changes, syncTimestamp } = req.body;
+    const rootFolder = await File.rootFolder();
+    try {
+      const syncDirName = `${syncTimestamp}_${req.user?.email || "public"}`;
+      const syncDir = path.join(
+        rootFolder.location,
+        "mobile_app",
+        "sync",
+        syncDirName
+      );
+      await fs.mkdir(syncDir, { recursive: true });
+      await fs.writeFile(
+        path.join(syncDir, "changes.json"),
+        JSON.stringify(changes)
+      );
+      const spawnParams = ["sync-upload-data"];
+      if (req.user?.email) spawnParams.push("--userEmail", req.user.email);
+      spawnParams.push("--directory", syncDir);
+      if (
+        db.is_it_multi_tenant() &&
+        db.getTenantSchema() !== db.connectObj.default_schema
+      ) {
+        spawnParams.push("--tenantAppName", db.getTenantSchema());
+      }
+      spawnParams.push("--syncTimestamp", syncTimestamp);
+
+      res.json({ syncDir: syncDirName });
+      const child = spawn(getSafeSaltcornCmd(), spawnParams, {
+        stdio: ["pipe", "pipe", "pipe"],
+        cwd: ".",
+      });
+
+      child.on("exit", async (exitCode, signal) => {
+        getState().log(
+          5,
+          `POST /sync/offline_changes: upload offline data finished with code: ${exitCode}`
+        );
+      });
+      child.on("error", (msg) => {
+        const message = msg.message ? msg.message : msg.code;
+        getState().log(
+          5,
+          `POST /sync/offline_changes: upload offline data failed: ${message}`
+        );
+      });
+    } catch (error) {
+      getState().log(2, `POST /sync/offline_changes: '${error.message}'`);
+      res.status(400).json({ error: error.message || error });
+    }
+  })
+);
+
+router.get(
+  "/upload_finished",
+  error_catcher(async (req, res) => {
+    const { dir_name } = req.query;
+    try {
+      const rootFolder = await File.rootFolder();
+      const syncDir = path.join(
+        rootFolder.location,
+        "mobile_app",
+        "sync",
+        dir_name
+      );
+      let entries = null;
+      try {
+        entries = await fs.readdir(syncDir);
+      } catch (error) {
+        return res.json({ finished: false });
+      }
+      if (entries.indexOf("translated-ids.json") >= 0) {
+        const translatedIds = JSON.parse(
+          await fs.readFile(path.join(syncDir, "translated-ids.json"))
+        );
+        res.json({ finished: true, translatedIds });
+      } else if (entries.indexOf("error.json") >= 0) {
+        const error = JSON.parse(
+          await fs.readFile(path.join(syncDir, "error.json"))
+        );
+        res.json({ finished: true, error });
+      } else {
+        res.json({ finished: false });
+      }
+    } catch (error) {
+      getState().log(2, `GET /sync/upload_finished: '${error.message}'`);
+      res.status(400).json({ error: error.message || error });
+    }
+  })
+);
+
+router.post(
+  "/clean_sync_dir",
+  error_catcher(async (req, res) => {
+    const { dir_name } = req.body;
+    try {
+      const rootFolder = await File.rootFolder();
+      const syncDir = path.join(
+        rootFolder.location,
+        "mobile_app",
+        "sync",
+        dir_name
+      );
+      await fs.rm(syncDir, { recursive: true, force: true });
+      res.status(200).send("");
+    } catch (error) {
+      getState().log(2, `POST /sync/clean_sync_dir: '${error.message}'`);
+      res.status(400).json({ error: error.message || error });
     }
   })
 );

package/routes/tables.js

@@ -168,6 +168,19 @@ const tableForm = async (table, req) => {
               name: "versioned",
               type: "Bool",
             },
+            ...(table.name === "users"
+              ? []
+              : [
+                  {
+                    label: req.__("Sync information"),
+                    sublabel: req.__(
+                      "Sync information tracks the last modification or deletion timestamp " +
+                        "so that the table data can be synchronized with the mobile app"
+                    ),
+                    name: "has_sync_info",
+                    type: "Bool",
+                  },
+                ]),
           ]),
     ],
   });
@@ -922,7 +935,10 @@ router.get(
             { href: `/table/constraints/${table.id}` },
             i({ class: "fas fa-2x fa-tasks" }),
             "<br/>",
-            req.__("Constraints")
+            req.__("Constraints") +
+              (table.constraints?.length
+                ? ` (${table.constraints.length})`
+                : "")
           )
         ),
 
@@ -1080,9 +1096,11 @@ router.post(
       const { id, _csrf, ...rest } = v;
       const table = Table.findOne({ id: parseInt(id) });
       const old_versioned = table.versioned;
+      const old_has_sync_info = table.has_sync_info;
       let hasError = false;
       let notify = "";
       if (!rest.versioned) rest.versioned = false;
+      if (!rest.has_sync_info) rest.has_sync_info = false;
       if (rest.ownership_field_id === "_formula") {
         rest.ownership_field_id = null;
         const fmlValidRes = expressionValidator(rest.ownership_formula);

package/tests/plugin_install.test.js

@@ -0,0 +1,114 @@
+const Table = require("@saltcorn/data/models/table");
+const Plugin = require("@saltcorn/data/models/plugin");
+const { getState, add_tenant } = require("@saltcorn/data/db/state");
+const { install_pack } = require("@saltcorn/admin-models/models/pack");
+const {
+  switchToTenant,
+  insertTenant,
+  create_tenant,
+} = require("@saltcorn/admin-models/models/tenant");
+const { resetToFixtures } = require("../auth/testhelp");
+const db = require("@saltcorn/data/db");
+const load_plugins = require("../load_plugins");
+
+beforeAll(async () => {
+  if (!db.isSQLite) await db.query(`drop schema if exists test101 CASCADE `);
+  await resetToFixtures();
+});
+afterAll(db.close);
+
+jest.setTimeout(30000);
+const plugin_pack = (plugin) => ({
+  tables: [],
+  views: [],
+  plugins: [
+    {
+      ...plugin,
+      configuration: null,
+    },
+  ],
+  pages: [],
+  roles: [],
+  library: [],
+  triggers: [],
+});
+
+describe("Tenant cannot install unsafe plugins", () => {
+  if (!db.isSQLite) {
+    it("creates a new tenant", async () => {
+      db.enable_multi_tenant();
+      const loadAndSaveNewPlugin = load_plugins.loadAndSaveNewPlugin;
+
+      await getState().setConfig("base_url", "http://example.com/");
+
+      add_tenant("test101");
+
+      await switchToTenant(
+        await insertTenant("test101", "foo@foo.com", ""),
+        "http://test101.example.com/"
+      );
+
+      await create_tenant({
+        t: "test101",
+        loadAndSaveNewPlugin,
+        plugin_loader() {},
+      });
+    });
+    it("can install safe plugins on tenant", async () => {
+      await db.runWithTenant("test101", async () => {
+        const loadAndSaveNewPlugin = load_plugins.loadAndSaveNewPlugin;
+
+        await install_pack(
+          plugin_pack({
+            name: "html",
+            source: "npm",
+            location: "@saltcorn/html",
+          }),
+          "Todo list",
+          loadAndSaveNewPlugin
+        );
+        const dbPlugin = await Plugin.findOne({ name: "html" });
+        expect(dbPlugin).not.toBe(null);
+      });
+    });
+    it("cannot install unsafe plugins on tenant", async () => {
+      await db.runWithTenant("test101", async () => {
+        const loadAndSaveNewPlugin = load_plugins.loadAndSaveNewPlugin;
+
+        await install_pack(
+          plugin_pack({
+            name: "sql-list",
+            source: "npm",
+            location: "@saltcorn/sql-list",
+          }),
+          "Todo list",
+          loadAndSaveNewPlugin
+        );
+        const dbPlugin = await Plugin.findOne({ name: "sql-list" });
+        expect(dbPlugin).toBe(null);
+      });
+    });
+    it("can install unsafe plugins on tenant when permitted", async () => {
+      await getState().setConfig("tenants_unsafe_plugins", true);
+      await db.runWithTenant("test101", async () => {
+        const loadAndSaveNewPlugin = load_plugins.loadAndSaveNewPlugin;
+
+        await install_pack(
+          plugin_pack({
+            name: "sql-list",
+            source: "npm",
+            location: "@saltcorn/sql-list",
+          }),
+          "Todo list",
+          loadAndSaveNewPlugin
+        );
+        const dbPlugin = await Plugin.findOne({ name: "sql-list" });
+        expect(dbPlugin).not.toBe(null);
+      });
+    });
+  } else {
+    it("does not support tenants on SQLite", async () => {
+      expect(db.isSQLite).toBe(true);
+    });
+  }
+});

package/tests/plugins.test.js

@@ -2,13 +2,8 @@ const request = require("supertest");
 const getApp = require("../app");
 const Table = require("@saltcorn/data/models/table");
 const Plugin = require("@saltcorn/data/models/plugin");
-const { getState, add_tenant } = require("@saltcorn/data/db/state");
-const { install_pack } = require("@saltcorn/admin-models/models/pack");
-const {
-  switchToTenant,
-  insertTenant,
-  create_tenant,
-} = require("@saltcorn/admin-models/models/tenant");
+const { getState } = require("@saltcorn/data/db/state");
+
 const {
   getAdminLoginCookie,
   itShouldRedirectUnauthToLogin,
@@ -331,98 +326,3 @@ describe("config endpoints", () => {
       .expect(toInclude(">FooSiteName<"));
   });
 });
-
-const plugin_pack = (plugin) => ({
-  tables: [],
-  views: [],
-  plugins: [
-    {
-      ...plugin,
-      configuration: null,
-    },
-  ],
-  pages: [],
-  roles: [],
-  library: [],
-  triggers: [],
-});
-
-describe("Tenant cannot install unsafe plugins", () => {
-  if (!db.isSQLite) {
-    it("creates a new tenant", async () => {
-      db.enable_multi_tenant();
-      const loadAndSaveNewPlugin = load_plugins.loadAndSaveNewPlugin;
-
-      await getState().setConfig("base_url", "http://example.com/");
-
-      add_tenant("test101");
-
-      await switchToTenant(
-        await insertTenant("test101", "foo@foo.com", ""),
-        "http://test101.example.com/"
-      );
-
-      await create_tenant({
-        t: "test101",
-        loadAndSaveNewPlugin,
-        plugin_loader() {},
-      });
-    });
-    it("can install safe plugins on tenant", async () => {
-      await db.runWithTenant("test101", async () => {
-        const loadAndSaveNewPlugin = load_plugins.loadAndSaveNewPlugin;
-
-        await install_pack(
-          plugin_pack({
-            name: "html",
-            source: "npm",
-            location: "@saltcorn/html",
-          }),
-          "Todo list",
-          loadAndSaveNewPlugin
-        );
-        const dbPlugin = await Plugin.findOne({ name: "html" });
-        expect(dbPlugin).not.toBe(null);
-      });
-    });
-    it("cannot install unsafe plugins on tenant", async () => {
-      await db.runWithTenant("test101", async () => {
-        const loadAndSaveNewPlugin = load_plugins.loadAndSaveNewPlugin;
-
-        await install_pack(
-          plugin_pack({
-            name: "sql-list",
-            source: "npm",
-            location: "@saltcorn/sql-list",
-          }),
-          "Todo list",
-          loadAndSaveNewPlugin
-        );
-        const dbPlugin = await Plugin.findOne({ name: "sql-list" });
-        expect(dbPlugin).toBe(null);
-      });
-    });
-    it("can install unsafe plugins on tenant when permitted", async () => {
-      await getState().setConfig("tenants_unsafe_plugins", true);
-      await db.runWithTenant("test101", async () => {
-        const loadAndSaveNewPlugin = load_plugins.loadAndSaveNewPlugin;
-
-        await install_pack(
-          plugin_pack({
-            name: "sql-list",
-            source: "npm",
-            location: "@saltcorn/sql-list",
-          }),
-          "Todo list",
-          loadAndSaveNewPlugin
-        );
-        const dbPlugin = await Plugin.findOne({ name: "sql-list" });
-        expect(dbPlugin).not.toBe(null);
-      });
-    });
-  } else {
-    it("does not support tenants on SQLite", async () => {
-      expect(db.isSQLite).toBe(true);
-    });
-  }
-});