@saltcorn/cli 0.8.8-beta.1 → 0.8.8-beta.3

package/src/commands/sync-upload-data.js

@@ -0,0 +1,246 @@
+const { Command, flags } = require("@oclif/command");
+const path = require("path");
+const { init_multi_tenant } = require("@saltcorn/data/db/state");
+const User = require("@saltcorn/data/models/user");
+const Table = require("@saltcorn/data/models/table");
+const fs = require("fs").promises;
+const { getState } = require("@saltcorn/data/db/state");
+const db = require("@saltcorn/data/db");
+const { loadAllPlugins } = require("@saltcorn/server/load_plugins");
+
+const pickFields = (table, pkName, row, keepId) => {
+  const result = {};
+  for (const { name, type, calculated } of table.getFields()) {
+    if ((!keepId && name === pkName) || calculated || !row[name]) continue;
+    if (type?.name === "Date") {
+      result[name] = row[name] ? new Date(row[name]) : undefined;
+    } else {
+      result[name] = row[name];
+    }
+  }
+  return result;
+};
+
+const translateInsertFks = async (allChanges, allTranslations) => {
+  const schema = db.getTenantSchemaPrefix();
+  const rowIds = (fk, targetTrans, tblName, pkName, changes) => {
+    if (Object.keys(targetTrans || {}).length > 0) {
+      const srcTrans = allTranslations[tblName] || {};
+      // ids with a fk where the target was translated
+      const insertIds = (changes.inserts || [])
+        .filter((row) => targetTrans[row[fk.name]] !== undefined)
+        .map((row) => srcTrans[row[pkName]] || row[pkName]);
+      return insertIds;
+    }
+    return null;
+  };
+
+  for (const [tblName, changes] of Object.entries(allChanges)) {
+    const table = Table.findOne({ name: tblName });
+    if (!table) throw new Error(`The table '${tblName}' does not exists`);
+    const pkName = table.pk_name;
+    for (const fk of table.getForeignKeys()) {
+      const targetTrans = allTranslations[fk.reftable_name];
+      const ids = rowIds(fk, targetTrans, table.name, pkName, changes);
+      if (ids?.length > 0) {
+        for (const [from, to] of Object.entries(targetTrans)) {
+          await db.query(
+            `update ${schema}"${db.sqlsanitize(tblName)}" set "${db.sqlsanitize(
+              fk.name
+            )}" = ${to}
+              where "${db.sqlsanitize(
+                fk.name
+              )}" = ${from} and "${db.sqlsanitize(pkName)}" in (${ids.join(
+              ","
+            )})`
+          );
+        }
+      }
+    }
+  }
+};
+
+const applyInserts = async (changes, syncTimestamp, user) => {
+  const schema = db.getTenantSchemaPrefix();
+  const allTranslations = {};
+  for (const [tblName, vals] of Object.entries(changes)) {
+    const table = Table.findOne({ name: tblName });
+    if (!table) throw new Error(`The table '${tblName}' does not exists`);
+    if (vals.inserts?.length > 0) {
+      const pkName = table.pk_name;
+      await db.query(
+        `alter table ${schema}"${db.sqlsanitize(tblName)}" disable trigger all`
+      );
+      const translations = {};
+      for (const insert of vals.inserts || []) {
+        const row = pickFields(table, pkName, insert);
+        const newId = await table.insertRow(
+          row,
+          user,
+          undefined,
+          true,
+          syncTimestamp
+        );
+        if (!newId) throw new Error(`Unable to insert into ${tblName}`);
+        else if (newId !== insert[pkName]) translations[insert[pkName]] = newId;
+      }
+      allTranslations[tblName] = translations;
+      await db.query(
+        `alter table ${schema}"${db.sqlsanitize(tblName)}" enable trigger all`
+      );
+    }
+  }
+  return allTranslations;
+};
+
+const applyUpdates = async (changes, allTranslations, syncTimestamp, user) => {
+  for (const [tblName, vals] of Object.entries(changes)) {
+    if (vals.updates?.length > 0) {
+      const table = Table.findOne({ name: tblName });
+      if (!table) throw new Error(`The table '${tblName}' does not exists`);
+      const pkName = table.pk_name;
+      const insertTranslations = allTranslations[tblName];
+      for (const update of vals.updates) {
+        const row = pickFields(table, pkName, update, true);
+        if (insertTranslations?.[row[pkName]])
+          row[pkName] = insertTranslations[row[pkName]];
+        for (const fk of table.getForeignKeys()) {
+          const oldVal = row[fk.name];
+          if (oldVal) {
+            const newVal = allTranslations[fk.reftable_name]?.[oldVal];
+            if (newVal) row[fk.name] = newVal;
+          }
+        }
+        const result = await table.updateRow(
+          row,
+          row[pkName],
+          user,
+          true,
+          undefined,
+          undefined,
+          syncTimestamp
+        );
+        if (result) throw new Error(`Unable to update ${tblName}: ${result}`);
+      }
+    }
+  }
+};
+
+const applyDeletes = async (changes, user) => {
+  for (const [tblName, vals] of Object.entries(changes)) {
+    const table = Table.findOne({ name: tblName });
+    if (!table) throw new Error(`The table '${tblName}' does not exists`);
+    const pkName = table.pk_name;
+    if (vals.deletes?.length > 0) {
+      const delIds = [];
+      const latestInfos = await table.latestSyncInfos(
+        vals.deletes.map((del) => del[pkName])
+      );
+      const refToInfo = {};
+      for (const info of latestInfos) {
+        refToInfo[info.ref] = info;
+      }
+      for (const del of vals.deletes) {
+        const appTimestamp = new Date(del.last_modified);
+        const info = refToInfo[del[pkName]];
+        if (!info || appTimestamp >= info.last_modified)
+          delIds.push(del[pkName]);
+      }
+      if (delIds.length > 0) {
+        await table.deleteRows({ [pkName]: { in: delIds } }, user, true);
+        if ((await table.countRows({ [pkName]: { in: delIds } })) !== 0)
+          throw new Error(
+            `Unable to delete in '${tblName}': Some rows were not deleted`
+          );
+      }
+    }
+  }
+};
+
+const writeTranslatedIds = async (translatedIds, directory) => {
+  const writeName = path.join(directory, "translated-ids.out");
+  await fs.writeFile(writeName, JSON.stringify(translatedIds));
+  await fs.rename(writeName, path.join(directory, "translated-ids.json"));
+};
+
+const writeErrorFile = async (message, directory) => {
+  const writeName = path.join(directory, "error.out");
+  await fs.writeFile(writeName, JSON.stringify({ message }));
+  await fs.rename(writeName, path.join(directory, "error.json"));
+};
+
+/**
+ *
+ */
+class SyncUploadData extends Command {
+  async run() {
+    let returnCode = 0,
+      inTransaction = false;
+    const { flags } = await this.parse(SyncUploadData);
+    if (db.is_it_multi_tenant() && flags.tenantAppName) {
+      await init_multi_tenant(loadAllPlugins, true, [flags.tenantAppName]);
+    }
+    const doSync = async () => {
+      try {
+        const changes = JSON.parse(
+          await fs.readFile(path.join(flags.directory, "changes.json"))
+        );
+        const syncTimestamp = flags.syncTimestamp;
+        const user = flags.userEmail
+          ? await User.findOne({ email: flags.userEmail })
+          : undefined;
+        await loadAllPlugins();
+        await db.begin();
+        inTransaction = true;
+        const translatedIds = await applyInserts(changes, syncTimestamp, user);
+        await translateInsertFks(changes, translatedIds);
+        await applyUpdates(changes, translatedIds, syncTimestamp, user);
+        await applyDeletes(changes, user);
+        await db.commit();
+        await writeTranslatedIds(translatedIds, flags.directory);
+      } catch (error) {
+        returnCode = 1;
+        getState().log(2, `Unable to sync: ${error.message}`);
+        await writeErrorFile(error.message, flags.directory);
+        if (inTransaction) await db.rollback();
+      } finally {
+        process.exit(returnCode);
+      }
+    };
+    if (
+      flags.tenantAppName &&
+      flags.tenantAppName !== db.connectObj.default_schema
+    ) {
+      await db.runWithTenant(flags.tenantAppName, doSync);
+    } else {
+      await doSync();
+    }
+  }
+}
+
+SyncUploadData.description = "Runs a sync for data supplied by the mobile app";
+
+SyncUploadData.flags = {
+  tenantAppName: flags.string({
+    name: "tenant",
+    string: "tenant",
+    description: "Optional name of a tenant application",
+  }),
+  userEmail: flags.string({
+    name: "user email",
+    string: "userEmail",
+    description: "email of the user running the sync",
+  }),
+  directory: flags.string({
+    name: "directory",
+    string: "directory",
+    description: "directory name for input output data",
+  }),
+  syncTimestamp: flags.integer({
+    name: "syncTimestamp",
+    string: "syncTimestamp",
+    description: "new timestamp for the sync_info rows",
+  }),
+};
+
+module.exports = SyncUploadData;

package/src/common.js

@@ -50,9 +50,29 @@ function sleep(ms) {
   return new Promise((resolve) => setTimeout(resolve, ms));
 }
 
+/**
+ * Read txt file (SyncMode)
+ * @param filename - absolute path to file
+ * @returns {null|string}
+ */
+function readFileSync(filename){
+  const path = require('path'), fs = require('fs');
+  try {
+    //let p = path.join(__dirname, filename);
+    let str = fs.readFileSync(filename, 'utf8');
+    // let str = fs.readFileSync(p, {encoding: 'utf8'});
+    console.log(str);
+    return str;
+  } catch (e) {
+    console.error(e.message);
+    return null;
+  }
+}
+
 module.exports = {
   maybe_as_tenant,
   parseJSONorString,
   sleep,
   init_some_tenants,
+  readFileSync,
 };