@saltcorn/data 0.8.8-beta.1 → 0.8.8-beta.3

package/dist/models/table.js

@@ -42,10 +42,10 @@ const csvtojson_1 = __importDefault(require("csvtojson"));
 const moment_1 = __importDefault(require("moment"));
 const fs_1 = require("fs");
 const promises_1 = require("fs/promises");
-const utils_1 = __importDefault(require("../utils"));
 //import { num_between } from "@saltcorn/types/generators";
 //import { devNull } from "os";
-const { prefixFieldsInWhere, InvalidConfiguration, InvalidAdminAction, satisfies, structuredClone, getLines, mergeIntoWhere, stringToJSON, } = utils_1.default;
+const utils_1 = __importDefault(require("../utils"));
+const { prefixFieldsInWhere, InvalidConfiguration, InvalidAdminAction, satisfies, structuredClone, getLines, mergeIntoWhere, stringToJSON, isNode, } = utils_1.default;
 const tags_1 = __importDefault(require("@saltcorn/markup/tags"));
 const { text } = tags_1.default;
 /**
@@ -111,6 +111,7 @@ class Table {
         this.ownership_field_id = o.ownership_field_id;
         this.ownership_formula = o.ownership_formula;
         this.versioned = !!o.versioned;
+        this.has_sync_info = !!o.has_sync_info;
         this.is_user_group = !!o.is_user_group;
         this.external = false;
         this.description = o.description;
@@ -233,6 +234,11 @@ class Table {
         }
         return [...dbs, ...externals];
     }
+    /**
+     * Get Models
+     * tbd why this function in this file - needs to models
+     * @param opts
+     */
     async get_models(opts) {
         const Model = require("./model");
         if (typeof opts === "string")
@@ -281,6 +287,10 @@ class Table {
             return user && user.id && row && `${row.id}` === `${user.id}`;
         return typeof field_name === "string" && row[field_name] === user.id;
     }
+    /**
+     * get Ownership options
+     * user interface...
+     */
     async ownership_options() {
         const fields = this.fields;
         //start with userfields
@@ -391,6 +401,9 @@ class Table {
         }
         return opts;
     }
+    /**
+     * get sanitized name of table
+     */
     get santized_name() {
         return (0, internal_1.sqlsanitize)(this.name);
     }
@@ -461,8 +474,10 @@ class Table {
     }
     /**
      * Drop current table
+     * @param only_forget boolean - if true that only
      * @returns {Promise<void>}
      */
+    // tbd check all other tables related to table description
     async delete(only_forget = false) {
         const schema = db_1.default.getTenantSchemaPrefix();
         const is_sqlite = db_1.default.isSQLite;
@@ -470,12 +485,16 @@ class Table {
         const client = is_sqlite ? db_1.default : await db_1.default.getClient();
         await client.query(`BEGIN`);
         try {
+            // drop table
             if (!only_forget)
                 await client.query(`drop table if exists ${schema}"${(0, internal_1.sqlsanitize)(this.name)}"`);
+            // delete fields
             await client.query(`delete FROM ${schema}_sc_fields WHERE table_id = $1`, [this.id]);
+            // delete table description
             await client.query(`delete FROM ${schema}_sc_tables WHERE id = $1`, [
                 this.id,
             ]);
+            // delete versioned table
             if (this.versioned)
                 await client.query(`drop table if exists ${schema}"${(0, internal_1.sqlsanitize)(this.name)}__history"`);
             await client.query(`COMMIT`);
@@ -491,12 +510,15 @@ class Table {
         await require("../db/state").getState().refresh_tables();
     }
     /***
-     * get Table SQL Name
+     * Get Table SQL Name
      * @type {string}
      */
     get sql_name() {
         return `${db_1.default.getTenantSchemaPrefix()}"${(0, internal_1.sqlsanitize)(this.name)}"`;
     }
+    /**
+     * Reset Sequence
+     */
     async resetSequence() {
         const fields = this.fields;
         const pk = fields.find((f) => f.primary_key);
@@ -508,6 +530,13 @@ class Table {
             pk.type.name === "Integer")
             await db_1.default.reset_sequence(this.name);
     }
+    /**
+     * update Where with Ownership
+     * @param where
+     * @param fields
+     * @param user
+     * @param forRead
+     */
     updateWhereWithOwnership(where, fields, user, forRead) {
         const role = user?.role_id;
         const min_role = forRead ? this.min_role_read : this.min_role_write;
@@ -524,13 +553,31 @@ class Table {
             });
         }
     }
+    async addDeleteSyncInfo(ids, timestamp) {
+        if (ids.length > 0) {
+            const schema = db_1.default.getTenantSchemaPrefix();
+            const pkName = this.pk_name;
+            if (isNode()) {
+                await db_1.default.query(`delete from ${schema}${db_1.default.sqlsanitize(this.name)}_sync_info where ref in (
+            ${ids.map((row) => row[pkName]).join(",")})`);
+                await db_1.default.query(`insert into ${schema}${db_1.default.sqlsanitize(this.name)}_sync_info values ${ids
+                    .map((row) => `(${row[pkName]}, date_trunc('milliseconds', to_timestamp( ${timestamp.valueOf() / 1000.0} ) ), true)`)
+                    .join(",")}`);
+            }
+            else {
+                await db_1.default.query(`update ${db_1.default.sqlsanitize(this.name)}_sync_info 
+           set deleted = true, modified_local = true
+           where ref in (${ids.map((row) => row[pkName]).join(",")})`);
+            }
+        }
+    }
     /**
      * Delete rows from table
      * @param where - condition
      * @param user - optional user, if null then no authorization will be checked
      * @returns
      */
-    async deleteRows(where, user) {
+    async deleteRows(where, user, noTrigger) {
         // get triggers on delete
         const triggers = await trigger_1.default.getTableTriggers("Delete", this);
         const fields = this.fields;
@@ -548,7 +595,7 @@ class Table {
         }
         const deleteFileFields = fields.filter((f) => f.type === "File" && f.attributes?.also_delete_file);
         const deleteFiles = [];
-        if (triggers.length > 0 || deleteFileFields.length > 0) {
+        if ((triggers.length > 0 || deleteFileFields.length > 0) && !noTrigger) {
             const File = require("./file");
             if (!rows)
                 rows = await this.getJoinedRows({
@@ -569,12 +616,28 @@ class Table {
                 }
             }
         }
-        if (rows)
+        if (rows) {
             await db_1.default.deleteWhere(this.name, {
                 [this.pk_name]: { in: rows.map((r) => r[this.pk_name]) },
             });
-        else
+            if (this.has_sync_info) {
+                const dbTime = await db_1.default.time();
+                const ids = rows.map((r) => r[this.pk_name || "id"]);
+                await this.addDeleteSyncInfo(ids, dbTime);
+            }
+        }
+        else {
+            const delIds = this.has_sync_info
+                ? await db_1.default.select(this.name, where, {
+                    fields: [this.pk_name],
+                })
+                : null;
             await db_1.default.deleteWhere(this.name, where);
+            if (this.has_sync_info) {
+                const dbTime = await db_1.default.time();
+                await this.addDeleteSyncInfo(delIds, dbTime);
+            }
+        }
         if (fields.find((f) => f.primary_key))
             await this.resetSequence();
         for (const file of deleteFiles) {
@@ -658,7 +721,7 @@ class Table {
             else
                 return []; //no ownership
         }
-        return apply_calculated_fields(rows.map((r) => this.readFromDB(r)), this.fields);
+        return apply_calculated_fields(rows.map((r) => this.readFromDB(r)), this.fields, !!selopts.ignore_errors);
     }
     /**
      * Count amount of rows in db table
@@ -685,6 +748,9 @@ class Table {
             return res.rows.map((r) => r[fieldnm]);
         }
     }
+    /**
+     *
+     */
     storedExpressionJoinFields() {
         let freeVars = new Set([]);
         for (const f of this.fields)
@@ -697,14 +763,14 @@ class Table {
     }
     /**
      * Update row
-     * @param v_in - colums with values to update
+     * @param v_in - columns with values to update
      * @param id - id value
      * @param _userid - user id
      * @param noTrigger
      * @param resultCollector
      * @returns
      */
-    async updateRow(v_in, id, user, noTrigger, resultCollector, restore_of_version) {
+    async updateRow(v_in, id, user, noTrigger, resultCollector, restore_of_version, syncTimestamp) {
         let existing;
         let v = { ...v_in };
         const fields = this.fields;
@@ -827,6 +893,13 @@ class Table {
                 });
         }
         await db_1.default.update(this.name, v, id, { pk_name });
+        if (this.has_sync_info) {
+            const oldInfo = await this.latestSyncInfo(id);
+            if (oldInfo && !oldInfo.deleted)
+                await this.updateSyncInfo(id, oldInfo.last_modified, syncTimestamp);
+            else
+                await this.insertSyncInfo(id, syncTimestamp);
+        }
         const newRow = { ...existing, ...v, [pk_name]: id };
         if (!noTrigger) {
             const trigPromise = trigger_1.default.runTableTriggers("Update", this, newRow, resultCollector, role === 100 ? undefined : user, { old_row: existing, updated_fields: v_in });
@@ -834,6 +907,39 @@ class Table {
                 await trigPromise;
         }
     }
+    async latestSyncInfo(id) {
+        const rows = await this.latestSyncInfos([id]);
+        return rows?.length === 1 ? rows[0] : null;
+    }
+    async latestSyncInfos(ids) {
+        const schema = db_1.default.getTenantSchemaPrefix();
+        const dbResult = await db_1.default.query(`select max(last_modified) "last_modified", ref
+       from ${schema}"${db_1.default.sqlsanitize(this.name)}_sync_info"
+       group by ref having ref in (${ids.join(",")})`);
+        return dbResult.rows;
+    }
+    async insertSyncInfo(id, syncTimestamp) {
+        const schema = db_1.default.getTenantSchemaPrefix();
+        if (isNode()) {
+            await db_1.default.query(`insert into ${schema}"${db_1.default.sqlsanitize(this.name)}_sync_info" values(${id},
+        date_trunc('milliseconds', to_timestamp(${(syncTimestamp ? syncTimestamp : await db_1.default.time()).valueOf() / 1000.0})))`);
+        }
+        else {
+            await db_1.default.query(`insert into "${db_1.default.sqlsanitize(this.name)}_sync_info"
+         (ref, modified_local, deleted) 
+         values(${id}, true, false)`);
+        }
+    }
+    async updateSyncInfo(id, oldLastModified, syncTimestamp) {
+        const schema = db_1.default.getTenantSchemaPrefix();
+        if (!db_1.default.isSQLite) {
+            await db_1.default.query(`update ${schema}"${db_1.default.sqlsanitize(this.name)}_sync_info" set last_modified=date_trunc('milliseconds', to_timestamp(${(syncTimestamp ? syncTimestamp : await db_1.default.time()).valueOf() / 1000.0})) where ref=${id} and last_modified = to_timestamp(${oldLastModified.valueOf() / 1000.0})`);
+        }
+        else {
+            await db_1.default.query(`update "${db_1.default.sqlsanitize(this.name)}_sync_info" set modified_local = true 
+         where ref = ${id} and last_modified = ${oldLastModified.valueOf()}`);
+        }
+    }
     /**
      * Try to Update row
      * @param v
@@ -876,7 +982,16 @@ class Table {
         }
         return pkField;
     }
-    check_table_constraints(row) {
+    /**
+     * Check table constraints
+     * @param row
+     */
+    check_table_constraints(row0) {
+        const row = { ...row0 };
+        this.fields.forEach((f) => {
+            if (typeof row[f.name] === "undefined")
+                row[f.name] = null;
+        });
         const fmls = this.constraints
             .filter((c) => c.type === "Formula")
             .map((c) => c.configuration);
@@ -886,6 +1001,11 @@ class Table {
         }
         return undefined;
     }
+    /**
+     *
+     * @param row
+     * @param user
+     */
     check_field_write_role(row, user) {
         for (const field of this.fields) {
             if (typeof row[field.name] !== "undefined" &&
@@ -902,7 +1022,7 @@ class Table {
      * @param resultCollector
      * @returns {Promise<*>}
      */
-    async insertRow(v_in, user, resultCollector) {
+    async insertRow(v_in, user, resultCollector, noTrigger, syncTimestamp) {
         const fields = this.fields;
         const pk_name = this.pk_name;
         const joinFields = this.storedExpressionJoinFields();
@@ -980,9 +1100,23 @@ class Table {
                 _userid: user?.id,
                 _time: new Date(),
             });
-        const trigPromise = trigger_1.default.runTableTriggers("Insert", this, { [pk_name]: id, ...v }, resultCollector, user);
-        if (resultCollector)
-            await trigPromise;
+        if (this.has_sync_info) {
+            if (isNode()) {
+                const schemaPrefix = db_1.default.getTenantSchemaPrefix();
+                await db_1.default.query(`insert into ${schemaPrefix}${db_1.default.sqlsanitize(this.name)}_sync_info 
+           values(${id}, date_trunc('milliseconds', to_timestamp(${(syncTimestamp ? syncTimestamp : await db_1.default.time()).valueOf() / 1000.0})))`);
+            }
+            else {
+                await db_1.default.query(`insert into ${db_1.default.sqlsanitize(this.name)}_sync_info 
+           (last_modified, ref, modified_local, deleted)
+           values(NULL, ${id}, true, false)`);
+            }
+        }
+        if (!noTrigger) {
+            const trigPromise = trigger_1.default.runTableTriggers("Insert", this, { [pk_name]: id, ...v }, resultCollector, user);
+            if (resultCollector)
+                await trigPromise;
+        }
         return id;
     }
     /**
@@ -1005,6 +1139,10 @@ class Table {
             return { error: this.normalise_error_message(e.message) };
         }
     }
+    /**
+     *
+     * @param msg
+     */
     normalise_error_message(msg) {
         let fieldnm = "";
         if (msg.toLowerCase().includes("unique constraint")) {
@@ -1118,6 +1256,28 @@ class Table {
           ,PRIMARY KEY("${pk}", _version)
           );`);
     }
+    async create_sync_info_table() {
+        const schemaPrefix = db_1.default.getTenantSchemaPrefix();
+        const fields = this.fields;
+        const pk = fields.find((f) => f.primary_key)?.name;
+        if (!pk) {
+            throw new Error("Unable to find a field with a primary key.");
+        }
+        await db_1.default.query(`create table ${schemaPrefix}${(0, internal_1.sqlsanitize)(this.name)}_sync_info (ref integer, last_modified timestamp, deleted boolean default false)`);
+        await db_1.default.query(`create index ${(0, internal_1.sqlsanitize)(this.name)}_sync_info_ref_index on ${schemaPrefix}${(0, internal_1.sqlsanitize)(this.name)}_sync_info(ref)`);
+        await db_1.default.query(`create index ${(0, internal_1.sqlsanitize)(this.name)}_sync_info_lm_index on ${schemaPrefix}${(0, internal_1.sqlsanitize)(this.name)}_sync_info(last_modified)`);
+    }
+    async drop_sync_table() {
+        const schemaPrefix = db_1.default.getTenantSchemaPrefix();
+        await db_1.default.query(`
+      drop table ${schemaPrefix}"${(0, internal_1.sqlsanitize)(this.name)}_sync_info";`);
+    }
+    /**
+     * Restore Row Version
+     * @param id
+     * @param version
+     * @param user
+     */
     async restore_row_version(id, version, user) {
         const row = await db_1.default.selectOne(`${db_1.default.sqlsanitize(this.name)}__history`, {
             id,
@@ -1131,28 +1291,38 @@ class Table {
         //console.log("restore_row_version", r);
         await this.updateRow(r, id, user, false, undefined, version);
     }
+    /**
+     * Undo row chnages
+     * @param id
+     * @param user
+     */
     async undo_row_changes(id, user) {
-        const current_verion_row = await db_1.default.selectMaybeOne(`${(0, internal_1.sqlsanitize)(this.name)}__history`, { id }, { orderBy: "_version", orderDesc: true, limit: 1 });
+        const current_version_row = await db_1.default.selectMaybeOne(`${(0, internal_1.sqlsanitize)(this.name)}__history`, { id }, { orderBy: "_version", orderDesc: true, limit: 1 });
         //get max that is not a restore
         const last_non_restore = await db_1.default.selectMaybeOne(`${(0, internal_1.sqlsanitize)(this.name)}__history`, {
             id,
             _version: {
-                lt: current_verion_row._restore_of_version
-                    ? current_verion_row._restore_of_version
-                    : current_verion_row._version,
+                lt: current_version_row._restore_of_version
+                    ? current_version_row._restore_of_version
+                    : current_version_row._version,
             },
         }, { orderBy: "_version", orderDesc: true, limit: 1 });
         if (last_non_restore) {
             await this.restore_row_version(id, last_non_restore._version, user);
         }
     }
+    /**
+     * Redo row changes
+     * @param id
+     * @param user
+     */
     async redo_row_changes(id, user) {
-        const current_verion_row = await db_1.default.selectMaybeOne(`${(0, internal_1.sqlsanitize)(this.name)}__history`, { id }, { orderBy: "_version", orderDesc: true, limit: 1 });
-        if (current_verion_row._restore_of_version) {
+        const current_version_row = await db_1.default.selectMaybeOne(`${(0, internal_1.sqlsanitize)(this.name)}__history`, { id }, { orderBy: "_version", orderDesc: true, limit: 1 });
+        if (current_version_row._restore_of_version) {
             const next_version = await db_1.default.selectMaybeOne(`${(0, internal_1.sqlsanitize)(this.name)}__history`, {
                 id,
                 _version: {
-                    gt: current_verion_row._restore_of_version,
+                    gt: current_version_row._restore_of_version,
                 },
             }, { orderBy: "_version", limit: 1 });
             if (next_version) {
@@ -1228,6 +1398,12 @@ class Table {
             else if (!new_table.versioned && existing.versioned) {
                 await new_table.drop_history_table();
             }
+            if (new_table.has_sync_info && !existing.has_sync_info) {
+                await this.create_sync_info_table();
+            }
+            else if (!new_table.has_sync_info && existing.has_sync_info) {
+                await new_table.drop_sync_table();
+            }
             Object.assign(this, new_table_rec);
         }
     }
@@ -1334,6 +1510,10 @@ class Table {
         await require("../db/state").getState().refresh_tables();
         return parse_res;
     }
+    /**
+     *
+     * @param state
+     */
     read_state_strict(state) {
         let errorString = "";
         this.fields.forEach((f) => {
@@ -2057,7 +2237,7 @@ class Table {
         const res = await db_1.default.query(sql, values);
         if (res.length === 0)
             return res; // check
-        let calcRow = apply_calculated_fields(res.rows, fields);
+        let calcRow = apply_calculated_fields(res.rows, fields, !!opts?.ignore_errors);
         //rename joinfields
         if (Object.values(joinFields || {}).some((jf) => jf.rename_object)) {
             let f = (x) => x;
@@ -2127,6 +2307,9 @@ class Table {
         }
         return calcRow;
     }
+    /**
+     *
+     */
     async slug_options() {
         const fields = this.fields;
         const unique_fields = fields.filter((f) => f.is_unique);
@@ -2151,18 +2334,27 @@ class Table {
         opts.unshift({ label: "", steps: [] });
         return opts;
     }
+    /**
+     *
+     */
     static async allSlugOptions() {
-        const tables = await Table.find({});
+        const tables = await Table.find({}, { cached: true });
         const options = {};
         for (const table of tables) {
             options[table.name] = await table.slug_options();
         }
         return options;
     }
+    /**
+     *
+     */
     async getTags() {
         const Tag = (await Promise.resolve().then(() => __importStar(require("./tag")))).default;
         return await Tag.findWithEntries({ table_id: this.id });
     }
+    /**
+     *
+     */
     async getForeignTables() {
         const result = new Array();
         if (this.fields) {