@budibase/server 2.1.43-alpha.6 → 2.1.44

package/dist/api/controllers/row/ExternalRequest.js

@@ -20,7 +20,6 @@ var __rest = (this && this.__rest) || function (s, e) {
     return t;
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.ExternalRequest = void 0;
 const types_1 = require("@budibase/types");
 const utils_1 = require("../../../integrations/utils");
 const utils_2 = require("./utils");
@@ -29,598 +28,602 @@ const utils_3 = require("../../../integrations/utils");
 const string_templates_1 = require("@budibase/string-templates");
 // @ts-ignore
 const fp_1 = require("lodash/fp");
-const rowProcessor_1 = require("../../../utilities/rowProcessor");
-const backend_core_1 = require("@budibase/backend-core");
-function buildFilters(id, filters, table) {
-    const primary = table.primary;
-    // if passed in array need to copy for shifting etc
-    let idCopy = (0, fp_1.cloneDeep)(id);
-    if (filters) {
-        // need to map over the filters and make sure the _id field isn't present
-        for (let filter of Object.values(filters)) {
-            if (filter._id && primary) {
-                const parts = (0, utils_1.breakRowIdField)(filter._id);
-                for (let field of primary) {
-                    filter[field] = parts.shift();
+const utils_4 = require("../../../utilities/rowProcessor/utils");
+// @ts-ignore
+const context_1 = require("@budibase/backend-core/context");
+var External;
+(function (External) {
+    function buildFilters(id, filters, table) {
+        const primary = table.primary;
+        // if passed in array need to copy for shifting etc
+        let idCopy = (0, fp_1.cloneDeep)(id);
+        if (filters) {
+            // need to map over the filters and make sure the _id field isn't present
+            for (let filter of Object.values(filters)) {
+                if (filter._id && primary) {
+                    const parts = (0, utils_1.breakRowIdField)(filter._id);
+                    for (let field of primary) {
+                        filter[field] = parts.shift();
+                    }
                 }
+                // make sure this field doesn't exist on any filter
+                delete filter._id;
             }
-            // make sure this field doesn't exist on any filter
-            delete filter._id;
         }
-    }
-    // there is no id, just use the user provided filters
-    if (!idCopy || !table) {
-        return filters;
-    }
-    // if used as URL parameter it will have been joined
-    if (!Array.isArray(idCopy)) {
-        idCopy = (0, utils_1.breakRowIdField)(idCopy);
-    }
-    const equal = {};
-    if (primary && idCopy) {
-        for (let field of primary) {
-            // work through the ID and get the parts
-            equal[field] = idCopy.shift();
+        // there is no id, just use the user provided filters
+        if (!idCopy || !table) {
+            return filters;
         }
-    }
-    return {
-        equal,
-    };
-}
-/**
- * This function checks the incoming parameters to make sure all the inputs are
- * valid based on on the table schema. The main thing this is looking for is when a
- * user has made use of the _id field of a row for a foreign key or a search parameter.
- * In these cases the key will be sent up as [1], rather than 1. In these cases we will
- * simplify it down to the requirements. This function is quite complex as we try to be
- * relatively restrictive over what types of columns we will perform this action for.
- */
-function cleanupConfig(config, table) {
-    const primaryOptions = [
-        constants_1.FieldTypes.STRING,
-        constants_1.FieldTypes.LONGFORM,
-        constants_1.FieldTypes.OPTIONS,
-        constants_1.FieldTypes.NUMBER,
-    ];
-    // filter out fields which cannot be keys
-    const fieldNames = Object.entries(table.schema)
-        .filter(schema => primaryOptions.find(val => val === schema[1].type))
-        .map(([fieldName]) => fieldName);
-    const iterateObject = (obj) => {
-        for (let [field, value] of Object.entries(obj)) {
-            if (fieldNames.find(name => name === field) && (0, utils_1.isRowId)(value)) {
-                obj[field] = (0, utils_1.convertRowId)(value);
-            }
+        // if used as URL parameter it will have been joined
+        if (!Array.isArray(idCopy)) {
+            idCopy = (0, utils_1.breakRowIdField)(idCopy);
         }
-    };
-    // check the row and filters to make sure they aren't a key of some sort
-    if (config.filters) {
-        for (let [key, filter] of Object.entries(config.filters)) {
-            // oneOf is an array, don't iterate it
-            if (typeof filter !== "object" ||
-                Object.keys(filter).length === 0 ||
-                key === types_1.FilterType.ONE_OF) {
-                continue;
+        const equal = {};
+        if (primary && idCopy) {
+            for (let field of primary) {
+                // work through the ID and get the parts
+                equal[field] = idCopy.shift();
             }
-            iterateObject(filter);
         }
+        return {
+            equal,
+        };
     }
-    if (config.row) {
-        iterateObject(config.row);
-    }
-    return config;
-}
-function generateIdForRow(row, table) {
-    const primary = table.primary;
-    if (!row || !primary) {
-        return "";
-    }
-    // build id array
-    let idParts = [];
-    for (let field of primary) {
-        // need to handle table name + field or just field, depending on if relationships used
-        const fieldValue = row[`${table.name}.${field}`] || row[field];
-        if (fieldValue) {
-            idParts.push(fieldValue);
+    /**
+     * This function checks the incoming parameters to make sure all the inputs are
+     * valid based on on the table schema. The main thing this is looking for is when a
+     * user has made use of the _id field of a row for a foreign key or a search parameter.
+     * In these cases the key will be sent up as [1], rather than 1. In these cases we will
+     * simplify it down to the requirements. This function is quite complex as we try to be
+     * relatively restrictive over what types of columns we will perform this action for.
+     */
+    function cleanupConfig(config, table) {
+        const primaryOptions = [
+            constants_1.FieldTypes.STRING,
+            constants_1.FieldTypes.LONGFORM,
+            constants_1.FieldTypes.OPTIONS,
+            constants_1.FieldTypes.NUMBER,
+        ];
+        // filter out fields which cannot be keys
+        const fieldNames = Object.entries(table.schema)
+            .filter(schema => primaryOptions.find(val => val === schema[1].type))
+            .map(([fieldName]) => fieldName);
+        const iterateObject = (obj) => {
+            for (let [field, value] of Object.entries(obj)) {
+                if (fieldNames.find(name => name === field) && (0, utils_1.isRowId)(value)) {
+                    obj[field] = (0, utils_1.convertRowId)(value);
+                }
+            }
+        };
+        // check the row and filters to make sure they aren't a key of some sort
+        if (config.filters) {
+            for (let [key, filter] of Object.entries(config.filters)) {
+                // oneOf is an array, don't iterate it
+                if (typeof filter !== "object" ||
+                    Object.keys(filter).length === 0 ||
+                    key === types_1.FilterType.ONE_OF) {
+                    continue;
+                }
+                iterateObject(filter);
+            }
         }
-    }
-    if (idParts.length === 0) {
-        return "";
-    }
-    return (0, utils_1.generateRowIdField)(idParts);
-}
-function getEndpoint(tableId, operation) {
-    if (!tableId) {
-        return {};
-    }
-    const { datasourceId, tableName } = (0, utils_3.breakExternalTableId)(tableId);
-    return {
-        datasourceId,
-        entityId: tableName,
-        operation,
-    };
-}
-function basicProcessing(row, table) {
-    const thisRow = {};
-    // filter the row down to what is actually the row (not joined)
-    for (let fieldName of Object.keys(table.schema)) {
-        const pathValue = row[`${table.name}.${fieldName}`];
-        const value = pathValue != null ? pathValue : row[fieldName];
-        // all responses include "select col as table.col" so that overlaps are handled
-        if (value != null) {
-            thisRow[fieldName] = value;
+        if (config.row) {
+            iterateObject(config.row);
         }
+        return config;
     }
-    thisRow._id = generateIdForRow(row, table);
-    thisRow.tableId = table._id;
-    thisRow._rev = "rev";
-    return (0, rowProcessor_1.processFormulas)(table, thisRow);
-}
-function fixArrayTypes(row, table) {
-    for (let [fieldName, schema] of Object.entries(table.schema)) {
-        if (schema.type === constants_1.FieldTypes.ARRAY &&
-            typeof row[fieldName] === "string") {
-            try {
-                row[fieldName] = JSON.parse(row[fieldName]);
-            }
-            catch (err) {
-                // couldn't convert back to array, ignore
-                delete row[fieldName];
+    function generateIdForRow(row, table) {
+        const primary = table.primary;
+        if (!row || !primary) {
+            return "";
+        }
+        // build id array
+        let idParts = [];
+        for (let field of primary) {
+            // need to handle table name + field or just field, depending on if relationships used
+            const fieldValue = row[`${table.name}.${field}`] || row[field];
+            if (fieldValue) {
+                idParts.push(fieldValue);
             }
         }
-    }
-    return row;
-}
-function isOneSide(field) {
-    return (field.relationshipType && field.relationshipType.split("-")[0] === "one");
-}
-class ExternalRequest {
-    constructor(operation, tableId, datasource) {
-        this.tables = {};
-        this.operation = operation;
-        this.tableId = tableId;
-        this.datasource = datasource;
-        if (datasource && datasource.entities) {
-            this.tables = datasource.entities;
+        if (idParts.length === 0) {
+            return "";
         }
+        return (0, utils_1.generateRowIdField)(idParts);
     }
-    getTable(tableId) {
+    function getEndpoint(tableId, operation) {
         if (!tableId) {
-            throw "Table ID is unknown, cannot find table";
-        }
-        const { tableName } = (0, utils_3.breakExternalTableId)(tableId);
-        if (tableName) {
-            return this.tables[tableName];
+            return {};
         }
+        const { datasourceId, tableName } = (0, utils_3.breakExternalTableId)(tableId);
+        return {
+            datasourceId,
+            entityId: tableName,
+            operation,
+        };
     }
-    inputProcessing(row, table) {
-        var _a;
-        if (!row) {
-            return { row, manyRelationships: [] };
-        }
-        // we don't really support composite keys for relationships, this is why [0] is used
-        // @ts-ignore
-        const tablePrimary = table.primary[0];
-        let newRow = {}, manyRelationships = [];
-        for (let [key, field] of Object.entries(table.schema)) {
-            // if set already, or not set just skip it
-            if (row[key] == null ||
-                newRow[key] ||
-                field.autocolumn ||
-                field.type === constants_1.FieldTypes.FORMULA) {
-                continue;
-            }
-            // if its an empty string then it means return the column to null (if possible)
-            if (row[key] === "") {
-                newRow[key] = null;
-                continue;
-            }
-            // parse floats/numbers
-            if (field.type === constants_1.FieldTypes.NUMBER && !isNaN(parseFloat(row[key]))) {
-                newRow[key] = parseFloat(row[key]);
-            }
-            // if its not a link then just copy it over
-            if (field.type !== constants_1.FieldTypes.LINK) {
-                newRow[key] = row[key];
-                continue;
-            }
-            const { tableName: linkTableName } = (0, utils_3.breakExternalTableId)(field === null || field === void 0 ? void 0 : field.tableId);
-            // table has to exist for many to many
-            if (!linkTableName || !this.tables[linkTableName]) {
-                continue;
-            }
-            const linkTable = this.tables[linkTableName];
-            // @ts-ignore
-            const linkTablePrimary = linkTable.primary[0];
-            // one to many
-            if (isOneSide(field)) {
-                let id = row[key][0];
-                if (typeof row[key] === "string") {
-                    id = (_a = decodeURIComponent(row[key]).match(/\[(.*?)\]/)) === null || _a === void 0 ? void 0 : _a[1];
-                }
-                newRow[field.foreignKey || linkTablePrimary] = (0, utils_1.breakRowIdField)(id)[0];
-            }
-            // many to many
-            else if (field.through) {
-                // we're not inserting a doc, will be a bunch of update calls
-                const otherKey = field.throughFrom || linkTablePrimary;
-                const thisKey = field.throughTo || tablePrimary;
-                row[key].map((relationship) => {
-                    manyRelationships.push({
-                        tableId: field.through || field.tableId,
-                        isUpdate: false,
-                        key: otherKey,
-                        [otherKey]: (0, utils_1.breakRowIdField)(relationship)[0],
-                        // leave the ID for enrichment later
-                        [thisKey]: `{{ literal ${tablePrimary} }}`,
-                    });
-                });
-            }
-            // many to one
-            else {
-                const thisKey = "id";
-                // @ts-ignore
-                const otherKey = field.fieldName;
-                row[key].map((relationship) => {
-                    manyRelationships.push({
-                        tableId: field.tableId,
-                        isUpdate: true,
-                        key: otherKey,
-                        [thisKey]: (0, utils_1.breakRowIdField)(relationship)[0],
-                        // leave the ID for enrichment later
-                        [otherKey]: `{{ literal ${tablePrimary} }}`,
-                    });
-                });
+    function basicProcessing(row, table) {
+        const thisRow = {};
+        // filter the row down to what is actually the row (not joined)
+        for (let fieldName of Object.keys(table.schema)) {
+            const pathValue = row[`${table.name}.${fieldName}`];
+            const value = pathValue != null ? pathValue : row[fieldName];
+            // all responses include "select col as table.col" so that overlaps are handled
+            if (value != null) {
+                thisRow[fieldName] = value;
             }
         }
-        // we return the relationships that may need to be created in the through table
-        // we do this so that if the ID is generated by the DB it can be inserted
-        // after the fact
-        return { row: newRow, manyRelationships };
+        thisRow._id = generateIdForRow(row, table);
+        thisRow.tableId = table._id;
+        thisRow._rev = "rev";
+        return (0, utils_4.processFormulas)(table, thisRow);
     }
-    squashRelationshipColumns(table, row, relationships) {
-        for (let relationship of relationships) {
-            const linkedTable = this.tables[relationship.tableName];
-            if (!linkedTable || !row[relationship.column]) {
-                continue;
-            }
-            const display = linkedTable.primaryDisplay;
-            for (let key of Object.keys(row[relationship.column])) {
-                const related = row[relationship.column][key];
-                row[relationship.column][key] = {
-                    primaryDisplay: display ? related[display] : undefined,
-                    _id: related._id,
-                };
+    function fixArrayTypes(row, table) {
+        for (let [fieldName, schema] of Object.entries(table.schema)) {
+            if (schema.type === constants_1.FieldTypes.ARRAY &&
+                typeof row[fieldName] === "string") {
+                try {
+                    row[fieldName] = JSON.parse(row[fieldName]);
+                }
+                catch (err) {
+                    // couldn't convert back to array, ignore
+                    delete row[fieldName];
+                }
             }
         }
         return row;
     }
-    /**
-     * This iterates through the returned rows and works out what elements of the rows
-     * actually match up to another row (based on primary keys) - this is pretty specific
-     * to SQL and the way that SQL relationships are returned based on joins.
-     * This is complicated, but the idea is that when a SQL query returns all the relations
-     * will be separate rows, with all of the data in each row. We have to decipher what comes
-     * from where (which tables) and how to convert that into budibase columns.
-     */
-    updateRelationshipColumns(table, row, rows, relationships) {
-        var _a, _b;
-        const columns = {};
-        for (let relationship of relationships) {
-            const linkedTable = this.tables[relationship.tableName];
-            if (!linkedTable) {
-                continue;
-            }
-            const fromColumn = `${table.name}.${relationship.from}`;
-            const toColumn = `${linkedTable.name}.${relationship.to}`;
-            // this is important when working with multiple relationships
-            // between the same tables, don't want to overlap/multiply the relations
-            if (!relationship.through &&
-                ((_a = row[fromColumn]) === null || _a === void 0 ? void 0 : _a.toString()) !== ((_b = row[toColumn]) === null || _b === void 0 ? void 0 : _b.toString())) {
-                continue;
-            }
-            let linked = basicProcessing(row, linkedTable);
-            if (!linked._id) {
-                continue;
+    function isOneSide(field) {
+        return (field.relationshipType && field.relationshipType.split("-")[0] === "one");
+    }
+    class ExternalRequest {
+        constructor(operation, tableId, datasource) {
+            this.tables = {};
+            this.operation = operation;
+            this.tableId = tableId;
+            this.datasource = datasource;
+            if (datasource && datasource.entities) {
+                this.tables = datasource.entities;
             }
-            columns[relationship.column] = linked;
         }
-        for (let [column, related] of Object.entries(columns)) {
-            if (!row._id) {
-                continue;
-            }
-            const rowId = row._id;
-            if (!Array.isArray(rows[rowId][column])) {
-                rows[rowId][column] = [];
+        getTable(tableId) {
+            if (!tableId) {
+                throw "Table ID is unknown, cannot find table";
             }
-            // make sure relationship hasn't been found already
-            if (!rows[rowId][column].find((relation) => relation._id === related._id)) {
-                rows[rowId][column].push(related);
+            const { tableName } = (0, utils_3.breakExternalTableId)(tableId);
+            if (tableName) {
+                return this.tables[tableName];
             }
         }
-        return rows;
-    }
-    outputProcessing(rows = [], table, relationships) {
-        if (!rows || rows.length === 0 || rows[0].read === true) {
-            return [];
-        }
-        let finalRows = {};
-        for (let row of rows) {
-            const rowId = generateIdForRow(row, table);
-            row._id = rowId;
-            // this is a relationship of some sort
-            if (finalRows[rowId]) {
-                finalRows = this.updateRelationshipColumns(table, row, finalRows, relationships);
-                continue;
+        inputProcessing(row, table) {
+            if (!row) {
+                return { row, manyRelationships: [] };
             }
-            const thisRow = fixArrayTypes(basicProcessing(row, table), table);
-            if (thisRow._id == null) {
-                throw "Unable to generate row ID for SQL rows";
+            // we don't really support composite keys for relationships, this is why [0] is used
+            // @ts-ignore
+            const tablePrimary = table.primary[0];
+            let newRow = {}, manyRelationships = [];
+            for (let [key, field] of Object.entries(table.schema)) {
+                // if set already, or not set just skip it
+                if (row[key] == null ||
+                    newRow[key] ||
+                    field.autocolumn ||
+                    field.type === constants_1.FieldTypes.FORMULA) {
+                    continue;
+                }
+                // if its an empty string then it means return the column to null (if possible)
+                if (row[key] === "") {
+                    newRow[key] = null;
+                    continue;
+                }
+                // parse floats/numbers
+                if (field.type === constants_1.FieldTypes.NUMBER && !isNaN(parseFloat(row[key]))) {
+                    newRow[key] = parseFloat(row[key]);
+                }
+                // if its not a link then just copy it over
+                if (field.type !== constants_1.FieldTypes.LINK) {
+                    newRow[key] = row[key];
+                    continue;
+                }
+                const { tableName: linkTableName } = (0, utils_3.breakExternalTableId)(field === null || field === void 0 ? void 0 : field.tableId);
+                // table has to exist for many to many
+                if (!linkTableName || !this.tables[linkTableName]) {
+                    continue;
+                }
+                const linkTable = this.tables[linkTableName];
+                // @ts-ignore
+                const linkTablePrimary = linkTable.primary[0];
+                // one to many
+                if (isOneSide(field)) {
+                    newRow[field.foreignKey || linkTablePrimary] = (0, utils_1.breakRowIdField)(row[key][0])[0];
+                }
+                // many to many
+                else if (field.through) {
+                    // we're not inserting a doc, will be a bunch of update calls
+                    const otherKey = field.throughFrom || linkTablePrimary;
+                    const thisKey = field.throughTo || tablePrimary;
+                    row[key].map((relationship) => {
+                        manyRelationships.push({
+                            tableId: field.through || field.tableId,
+                            isUpdate: false,
+                            key: otherKey,
+                            [otherKey]: (0, utils_1.breakRowIdField)(relationship)[0],
+                            // leave the ID for enrichment later
+                            [thisKey]: `{{ literal ${tablePrimary} }}`,
+                        });
+                    });
+                }
+                // many to one
+                else {
+                    const thisKey = "id";
+                    // @ts-ignore
+                    const otherKey = field.fieldName;
+                    row[key].map((relationship) => {
+                        manyRelationships.push({
+                            tableId: field.tableId,
+                            isUpdate: true,
+                            key: otherKey,
+                            [thisKey]: (0, utils_1.breakRowIdField)(relationship)[0],
+                            // leave the ID for enrichment later
+                            [otherKey]: `{{ literal ${tablePrimary} }}`,
+                        });
+                    });
+                }
             }
-            finalRows[thisRow._id] = thisRow;
-            // do this at end once its been added to the final rows
-            finalRows = this.updateRelationshipColumns(table, row, finalRows, relationships);
+            // we return the relationships that may need to be created in the through table
+            // we do this so that if the ID is generated by the DB it can be inserted
+            // after the fact
+            return { row: newRow, manyRelationships };
         }
-        // Process some additional data types
-        let finalRowArray = Object.values(finalRows);
-        finalRowArray = (0, rowProcessor_1.processDates)(table, finalRowArray);
-        finalRowArray = (0, rowProcessor_1.processFormulas)(table, finalRowArray);
-        return finalRowArray.map((row) => this.squashRelationshipColumns(table, row, relationships));
-    }
-    /**
-     * Gets the list of relationship JSON structures based on the columns in the table,
-     * this will be used by the underlying library to build whatever relationship mechanism
-     * it has (e.g. SQL joins).
-     */
-    buildRelationships(table) {
-        const relationships = [];
-        for (let [fieldName, field] of Object.entries(table.schema)) {
-            if (field.type !== constants_1.FieldTypes.LINK) {
-                continue;
-            }
-            const { tableName: linkTableName } = (0, utils_3.breakExternalTableId)(field.tableId);
-            // no table to link to, this is not a valid relationships
-            if (!linkTableName || !this.tables[linkTableName]) {
-                continue;
+        squashRelationshipColumns(table, row, relationships) {
+            for (let relationship of relationships) {
+                const linkedTable = this.tables[relationship.tableName];
+                if (!linkedTable || !row[relationship.column]) {
+                    continue;
+                }
+                const display = linkedTable.primaryDisplay;
+                for (let key of Object.keys(row[relationship.column])) {
+                    const related = row[relationship.column][key];
+                    row[relationship.column][key] = {
+                        primaryDisplay: display ? related[display] : undefined,
+                        _id: related._id,
+                    };
+                }
             }
-            const linkTable = this.tables[linkTableName];
-            if (!table.primary || !linkTable.primary) {
-                continue;
+            return row;
+        }
+        /**
+         * This iterates through the returned rows and works out what elements of the rows
+         * actually match up to another row (based on primary keys) - this is pretty specific
+         * to SQL and the way that SQL relationships are returned based on joins.
+         * This is complicated, but the idea is that when a SQL query returns all the relations
+         * will be separate rows, with all of the data in each row. We have to decipher what comes
+         * from where (which tables) and how to convert that into budibase columns.
+         */
+        updateRelationshipColumns(table, row, rows, relationships) {
+            var _a, _b;
+            const columns = {};
+            for (let relationship of relationships) {
+                const linkedTable = this.tables[relationship.tableName];
+                if (!linkedTable) {
+                    continue;
+                }
+                const fromColumn = `${table.name}.${relationship.from}`;
+                const toColumn = `${linkedTable.name}.${relationship.to}`;
+                // this is important when working with multiple relationships
+                // between the same tables, don't want to overlap/multiply the relations
+                if (!relationship.through &&
+                    ((_a = row[fromColumn]) === null || _a === void 0 ? void 0 : _a.toString()) !== ((_b = row[toColumn]) === null || _b === void 0 ? void 0 : _b.toString())) {
+                    continue;
+                }
+                let linked = basicProcessing(row, linkedTable);
+                if (!linked._id) {
+                    continue;
+                }
+                columns[relationship.column] = linked;
             }
-            const definition = {
-                // if no foreign key specified then use the name of the field in other table
-                from: field.foreignKey || table.primary[0],
-                to: field.fieldName,
-                tableName: linkTableName,
-                // need to specify where to put this back into
-                column: fieldName,
-            };
-            if (field.through) {
-                const { tableName: throughTableName } = (0, utils_3.breakExternalTableId)(field.through);
-                definition.through = throughTableName;
-                // don't support composite keys for relationships
-                definition.from = field.throughTo || table.primary[0];
-                definition.to = field.throughFrom || linkTable.primary[0];
-                definition.fromPrimary = table.primary[0];
-                definition.toPrimary = linkTable.primary[0];
+            for (let [column, related] of Object.entries(columns)) {
+                if (!row._id) {
+                    continue;
+                }
+                const rowId = row._id;
+                if (!Array.isArray(rows[rowId][column])) {
+                    rows[rowId][column] = [];
+                }
+                // make sure relationship hasn't been found already
+                if (!rows[rowId][column].find((relation) => relation._id === related._id)) {
+                    rows[rowId][column].push(related);
+                }
             }
-            relationships.push(definition);
+            return rows;
         }
-        return relationships;
-    }
-    /**
-     * This is a cached lookup, of relationship records, this is mainly for creating/deleting junction
-     * information.
-     */
-    lookupRelations(tableId, row) {
-        return __awaiter(this, void 0, void 0, function* () {
-            const related = {};
-            const { tableName } = (0, utils_3.breakExternalTableId)(tableId);
-            if (!tableName) {
-                return related;
+        outputProcessing(rows = [], table, relationships) {
+            if (!rows || rows.length === 0 || rows[0].read === true) {
+                return [];
+            }
+            let finalRows = {};
+            for (let row of rows) {
+                const rowId = generateIdForRow(row, table);
+                row._id = rowId;
+                // this is a relationship of some sort
+                if (finalRows[rowId]) {
+                    finalRows = this.updateRelationshipColumns(table, row, finalRows, relationships);
+                    continue;
+                }
+                const thisRow = fixArrayTypes(basicProcessing(row, table), table);
+                if (thisRow._id == null) {
+                    throw "Unable to generate row ID for SQL rows";
+                }
+                finalRows[thisRow._id] = thisRow;
+                // do this at end once its been added to the final rows
+                finalRows = this.updateRelationshipColumns(table, row, finalRows, relationships);
             }
-            const table = this.tables[tableName];
-            // @ts-ignore
-            const primaryKey = table.primary[0];
-            // make a new request to get the row with all its relationships
-            // we need this to work out if any relationships need removed
-            for (let field of Object.values(table.schema)) {
-                if (field.type !== constants_1.FieldTypes.LINK ||
-                    !field.fieldName ||
-                    isOneSide(field)) {
+            // Process some additional data types
+            let finalRowArray = Object.values(finalRows);
+            finalRowArray = (0, utils_4.processDates)(table, finalRowArray);
+            finalRowArray = (0, utils_4.processFormulas)(table, finalRowArray);
+            return finalRowArray.map((row) => this.squashRelationshipColumns(table, row, relationships));
+        }
+        /**
+         * Gets the list of relationship JSON structures based on the columns in the table,
+         * this will be used by the underlying library to build whatever relationship mechanism
+         * it has (e.g. SQL joins).
+         */
+        buildRelationships(table) {
+            const relationships = [];
+            for (let [fieldName, field] of Object.entries(table.schema)) {
+                if (field.type !== constants_1.FieldTypes.LINK) {
                     continue;
                 }
-                const isMany = field.relationshipType === constants_1.RelationshipTypes.MANY_TO_MANY;
-                const tableId = isMany ? field.through : field.tableId;
-                const { tableName: relatedTableName } = (0, utils_3.breakExternalTableId)(tableId);
-                // @ts-ignore
-                const linkPrimaryKey = this.tables[relatedTableName].primary[0];
-                const manyKey = field.throughTo || primaryKey;
-                const lookupField = isMany ? primaryKey : field.foreignKey;
-                const fieldName = isMany ? manyKey : field.fieldName;
-                if (!lookupField || !row[lookupField]) {
+                const { tableName: linkTableName } = (0, utils_3.breakExternalTableId)(field.tableId);
+                // no table to link to, this is not a valid relationships
+                if (!linkTableName || !this.tables[linkTableName]) {
                     continue;
                 }
-                const response = yield (0, utils_2.getDatasourceAndQuery)({
-                    endpoint: getEndpoint(tableId, types_1.Operation.READ),
-                    filters: {
-                        equal: {
-                            [fieldName]: row[lookupField],
-                        },
-                    },
-                });
-                // this is the response from knex if no rows found
-                const rows = !response[0].read ? response : [];
-                const storeTo = isMany ? field.throughFrom || linkPrimaryKey : fieldName;
-                related[storeTo] = { rows, isMany, tableId };
-            }
-            return related;
-        });
-    }
-    /**
-     * Once a row has been written we may need to update a many field, e.g. updating foreign keys
-     * in a bunch of rows in another table, or inserting/deleting rows from a junction table (many to many).
-     * This is quite a complex process and is handled by this function, there are a few things going on here:
-     * 1. If updating foreign keys its relatively simple, just create a filter for the row that needs updated
-     * and write the various components.
-     * 2. If junction table, then we lookup what exists already, write what doesn't exist, work out what
-     * isn't supposed to exist anymore and delete those. This is better than the usual method of delete them
-     * all and then re-create, as theres no chance of losing data (e.g. delete succeed, but write fail).
-     */
-    handleManyRelationships(mainTableId, row, relationships) {
-        var _a;
-        return __awaiter(this, void 0, void 0, function* () {
-            // if we're creating (in a through table) need to wipe the existing ones first
-            const promises = [];
-            const related = yield this.lookupRelations(mainTableId, row);
-            for (let relationship of relationships) {
-                const { key, tableId, isUpdate, id } = relationship, rest = __rest(relationship, ["key", "tableId", "isUpdate", "id"]);
-                const body = (0, string_templates_1.processObjectSync)(rest, row, {});
-                const linkTable = this.getTable(tableId);
-                // @ts-ignore
-                const linkPrimary = linkTable === null || linkTable === void 0 ? void 0 : linkTable.primary[0];
-                if (!linkTable || !linkPrimary) {
-                    return;
-                }
-                const rows = ((_a = related[key]) === null || _a === void 0 ? void 0 : _a.rows) || [];
-                const found = rows.find((row) => row[linkPrimary] === relationship.id ||
-                    row[linkPrimary] === (body === null || body === void 0 ? void 0 : body[linkPrimary]));
-                const operation = isUpdate ? types_1.Operation.UPDATE : types_1.Operation.CREATE;
-                if (!found) {
-                    promises.push((0, utils_2.getDatasourceAndQuery)({
-                        endpoint: getEndpoint(tableId, operation),
-                        // if we're doing many relationships then we're writing, only one response
-                        body,
-                        filters: buildFilters(id, {}, linkTable),
-                    }));
+                const linkTable = this.tables[linkTableName];
+                if (!table.primary || !linkTable.primary) {
+                    continue;
                 }
-                else {
-                    // remove the relationship from cache so it isn't adjusted again
-                    rows.splice(rows.indexOf(found), 1);
+                const definition = {
+                    // if no foreign key specified then use the name of the field in other table
+                    from: field.foreignKey || table.primary[0],
+                    to: field.fieldName,
+                    tableName: linkTableName,
+                    // need to specify where to put this back into
+                    column: fieldName,
+                };
+                if (field.through) {
+                    const { tableName: throughTableName } = (0, utils_3.breakExternalTableId)(field.through);
+                    definition.through = throughTableName;
+                    // don't support composite keys for relationships
+                    definition.from = field.throughTo || table.primary[0];
+                    definition.to = field.throughFrom || linkTable.primary[0];
+                    definition.fromPrimary = table.primary[0];
+                    definition.toPrimary = linkTable.primary[0];
                 }
+                relationships.push(definition);
             }
-            // finally cleanup anything that needs to be removed
-            for (let [colName, { isMany, rows, tableId }] of Object.entries(related)) {
-                const table = this.getTable(tableId);
-                // if its not the foreign key skip it, nothing to do
-                if (!table ||
-                    (!isMany && table.primary && table.primary.indexOf(colName) !== -1)) {
-                    continue;
+            return relationships;
+        }
+        /**
+         * This is a cached lookup, of relationship records, this is mainly for creating/deleting junction
+         * information.
+         */
+        lookupRelations(tableId, row) {
+            return __awaiter(this, void 0, void 0, function* () {
+                const related = {};
+                const { tableName } = (0, utils_3.breakExternalTableId)(tableId);
+                if (!tableName) {
+                    return related;
+                }
+                const table = this.tables[tableName];
+                // @ts-ignore
+                const primaryKey = table.primary[0];
+                // make a new request to get the row with all its relationships
+                // we need this to work out if any relationships need removed
+                for (let field of Object.values(table.schema)) {
+                    if (field.type !== constants_1.FieldTypes.LINK ||
+                        !field.fieldName ||
+                        isOneSide(field)) {
+                        continue;
+                    }
+                    const isMany = field.relationshipType === constants_1.RelationshipTypes.MANY_TO_MANY;
+                    const tableId = isMany ? field.through : field.tableId;
+                    const { tableName: relatedTableName } = (0, utils_3.breakExternalTableId)(tableId);
+                    // @ts-ignore
+                    const linkPrimaryKey = this.tables[relatedTableName].primary[0];
+                    const manyKey = field.throughTo || primaryKey;
+                    const lookupField = isMany ? primaryKey : field.foreignKey;
+                    const fieldName = isMany ? manyKey : field.fieldName;
+                    if (!lookupField || !row[lookupField]) {
+                        continue;
+                    }
+                    const response = yield (0, utils_2.getDatasourceAndQuery)({
+                        endpoint: getEndpoint(tableId, constants_1.DataSourceOperation.READ),
+                        filters: {
+                            equal: {
+                                [fieldName]: row[lookupField],
+                            },
+                        },
+                    });
+                    // this is the response from knex if no rows found
+                    const rows = !response[0].read ? response : [];
+                    const storeTo = isMany ? field.throughFrom || linkPrimaryKey : fieldName;
+                    related[storeTo] = { rows, isMany, tableId };
                 }
-                for (let row of rows) {
-                    const filters = buildFilters(generateIdForRow(row, table), {}, table);
-                    // safety check, if there are no filters on deletion bad things happen
-                    if (Object.keys(filters).length !== 0) {
-                        const op = isMany ? types_1.Operation.DELETE : types_1.Operation.UPDATE;
-                        const body = isMany ? null : { [colName]: null };
+                return related;
+            });
+        }
+        /**
+         * Once a row has been written we may need to update a many field, e.g. updating foreign keys
+         * in a bunch of rows in another table, or inserting/deleting rows from a junction table (many to many).
+         * This is quite a complex process and is handled by this function, there are a few things going on here:
+         * 1. If updating foreign keys its relatively simple, just create a filter for the row that needs updated
+         * and write the various components.
+         * 2. If junction table, then we lookup what exists already, write what doesn't exist, work out what
+         * isn't supposed to exist anymore and delete those. This is better than the usual method of delete them
+         * all and then re-create, as theres no chance of losing data (e.g. delete succeed, but write fail).
+         */
+        handleManyRelationships(mainTableId, row, relationships) {
+            var _a;
+            return __awaiter(this, void 0, void 0, function* () {
+                // if we're creating (in a through table) need to wipe the existing ones first
+                const promises = [];
+                const related = yield this.lookupRelations(mainTableId, row);
+                for (let relationship of relationships) {
+                    const { key, tableId, isUpdate, id } = relationship, rest = __rest(relationship, ["key", "tableId", "isUpdate", "id"]);
+                    const body = (0, string_templates_1.processObjectSync)(rest, row, {});
+                    const linkTable = this.getTable(tableId);
+                    // @ts-ignore
+                    const linkPrimary = linkTable === null || linkTable === void 0 ? void 0 : linkTable.primary[0];
+                    if (!linkTable || !linkPrimary) {
+                        return;
+                    }
+                    const rows = ((_a = related[key]) === null || _a === void 0 ? void 0 : _a.rows) || [];
+                    const found = rows.find((row) => row[linkPrimary] === relationship.id ||
+                        row[linkPrimary] === (body === null || body === void 0 ? void 0 : body[linkPrimary]));
+                    const operation = isUpdate
+                        ? constants_1.DataSourceOperation.UPDATE
+                        : constants_1.DataSourceOperation.CREATE;
+                    if (!found) {
                         promises.push((0, utils_2.getDatasourceAndQuery)({
-                            endpoint: getEndpoint(tableId, op),
+                            endpoint: getEndpoint(tableId, operation),
+                            // if we're doing many relationships then we're writing, only one response
                             body,
-                            filters,
+                            filters: buildFilters(id, {}, linkTable),
                         }));
                     }
+                    else {
+                        // remove the relationship from cache so it isn't adjusted again
+                        rows.splice(rows.indexOf(found), 1);
+                    }
                 }
-            }
-            yield Promise.all(promises);
-        });
-    }
-    /**
-     * This function is a bit crazy, but the exact purpose of it is to protect against the scenario in which
-     * you have column overlap in relationships, e.g. we join a few different tables and they all have the
-     * concept of an ID, but for some of them it will be null (if they say don't have a relationship).
-     * Creating the specific list of fields that we desire, and excluding the ones that are no use to us
-     * is more performant and has the added benefit of protecting against this scenario.
-     */
-    buildFields(table, includeRelations = types_1.IncludeRelationship.INCLUDE) {
-        function extractRealFields(table, existing = []) {
-            return Object.entries(table.schema)
-                .filter(column => column[1].type !== constants_1.FieldTypes.LINK &&
-                column[1].type !== constants_1.FieldTypes.FORMULA &&
-                !existing.find((field) => field === column[0]))
-                .map(column => `${table.name}.${column[0]}`);
+                // finally cleanup anything that needs to be removed
+                for (let [colName, { isMany, rows, tableId }] of Object.entries(related)) {
+                    const table = this.getTable(tableId);
+                    // if its not the foreign key skip it, nothing to do
+                    if (!table ||
+                        (table.primary && table.primary.indexOf(colName) !== -1)) {
+                        continue;
+                    }
+                    for (let row of rows) {
+                        const filters = buildFilters(generateIdForRow(row, table), {}, table);
+                        // safety check, if there are no filters on deletion bad things happen
+                        if (Object.keys(filters).length !== 0) {
+                            const op = isMany
+                                ? constants_1.DataSourceOperation.DELETE
+                                : constants_1.DataSourceOperation.UPDATE;
+                            const body = isMany ? null : { [colName]: null };
+                            promises.push((0, utils_2.getDatasourceAndQuery)({
+                                endpoint: getEndpoint(tableId, op),
+                                body,
+                                filters,
+                            }));
+                        }
+                    }
+                }
+                yield Promise.all(promises);
+            });
         }
-        let fields = extractRealFields(table);
-        for (let field of Object.values(table.schema)) {
-            if (field.type !== constants_1.FieldTypes.LINK || !includeRelations) {
-                continue;
-            }
-            const { tableName: linkTableName } = (0, utils_3.breakExternalTableId)(field.tableId);
-            if (linkTableName) {
-                const linkTable = this.tables[linkTableName];
-                if (linkTable) {
-                    const linkedFields = extractRealFields(linkTable, fields);
-                    fields = fields.concat(linkedFields);
+        /**
+         * This function is a bit crazy, but the exact purpose of it is to protect against the scenario in which
+         * you have column overlap in relationships, e.g. we join a few different tables and they all have the
+         * concept of an ID, but for some of them it will be null (if they say don't have a relationship).
+         * Creating the specific list of fields that we desire, and excluding the ones that are no use to us
+         * is more performant and has the added benefit of protecting against this scenario.
+         */
+        buildFields(table, includeRelations = types_1.IncludeRelationship.INCLUDE) {
+            function extractRealFields(table, existing = []) {
+                return Object.entries(table.schema)
+                    .filter(column => column[1].type !== constants_1.FieldTypes.LINK &&
+                    column[1].type !== constants_1.FieldTypes.FORMULA &&
+                    !existing.find((field) => field === column[0]))
+                    .map(column => `${table.name}.${column[0]}`);
+            }
+            let fields = extractRealFields(table);
+            for (let field of Object.values(table.schema)) {
+                if (field.type !== constants_1.FieldTypes.LINK || !includeRelations) {
+                    continue;
+                }
+                const { tableName: linkTableName } = (0, utils_3.breakExternalTableId)(field.tableId);
+                if (linkTableName) {
+                    const linkTable = this.tables[linkTableName];
+                    if (linkTable) {
+                        const linkedFields = extractRealFields(linkTable, fields);
+                        fields = fields.concat(linkedFields);
+                    }
                 }
             }
+            return fields;
         }
-        return fields;
-    }
-    run(config) {
-        return __awaiter(this, void 0, void 0, function* () {
-            const { operation, tableId } = this;
-            let { datasourceId, tableName } = (0, utils_3.breakExternalTableId)(tableId);
-            if (!tableName) {
-                throw "Unable to run without a table name";
-            }
-            if (!this.datasource) {
-                const db = backend_core_1.context.getAppDB();
-                this.datasource = yield db.get(datasourceId);
-                if (!this.datasource || !this.datasource.entities) {
-                    throw "No tables found, fetch tables before query.";
+        run(config) {
+            return __awaiter(this, void 0, void 0, function* () {
+                const { operation, tableId } = this;
+                let { datasourceId, tableName } = (0, utils_3.breakExternalTableId)(tableId);
+                if (!tableName) {
+                    throw "Unable to run without a table name";
                 }
-                this.tables = this.datasource.entities;
-            }
-            const table = this.tables[tableName];
-            let isSql = (0, utils_3.isSQL)(this.datasource);
-            if (!table) {
-                throw `Unable to process query, table "${tableName}" not defined.`;
-            }
-            // look for specific components of config which may not be considered acceptable
-            let { id, row, filters, sort, paginate, rows } = cleanupConfig(config, table);
-            filters = buildFilters(id, filters || {}, table);
-            const relationships = this.buildRelationships(table);
-            // clean up row on ingress using schema
-            const processed = this.inputProcessing(row, table);
-            row = processed.row;
-            if (operation === types_1.Operation.DELETE &&
-                (filters == null || Object.keys(filters).length === 0)) {
-                throw "Deletion must be filtered";
-            }
-            let json = {
-                endpoint: {
-                    datasourceId,
-                    entityId: tableName,
-                    operation,
-                },
-                resource: {
-                    // have to specify the fields to avoid column overlap (for SQL)
-                    fields: isSql ? this.buildFields(table) : [],
-                },
-                filters,
-                sort,
-                paginate,
-                relationships,
-                body: row || rows,
-                // pass an id filter into extra, purely for mysql/returning
-                extra: {
-                    idFilter: buildFilters(id || generateIdForRow(row, table), {}, table),
-                },
-                meta: {
-                    table,
-                },
-            };
-            // can't really use response right now
-            const response = yield (0, utils_2.getDatasourceAndQuery)(json);
-            // handle many to many relationships now if we know the ID (could be auto increment)
-            if (operation !== types_1.Operation.READ && processed.manyRelationships) {
-                yield this.handleManyRelationships(table._id || "", response[0], processed.manyRelationships);
-            }
-            const output = this.outputProcessing(response, table, relationships);
-            // if reading it'll just be an array of rows, return whole thing
-            return operation === types_1.Operation.READ && Array.isArray(response)
-                ? output
-                : { row: output[0], table };
-        });
+                if (!this.datasource) {
+                    const db = (0, context_1.getAppDB)();
+                    this.datasource = yield db.get(datasourceId);
+                    if (!this.datasource || !this.datasource.entities) {
+                        throw "No tables found, fetch tables before query.";
+                    }
+                    this.tables = this.datasource.entities;
+                }
+                const table = this.tables[tableName];
+                let isSql = (0, utils_3.isSQL)(this.datasource);
+                if (!table) {
+                    throw `Unable to process query, table "${tableName}" not defined.`;
+                }
+                // look for specific components of config which may not be considered acceptable
+                let { id, row, filters, sort, paginate, rows } = cleanupConfig(config, table);
+                filters = buildFilters(id, filters || {}, table);
+                const relationships = this.buildRelationships(table);
+                // clean up row on ingress using schema
+                const processed = this.inputProcessing(row, table);
+                row = processed.row;
+                if (operation === constants_1.DataSourceOperation.DELETE &&
+                    (filters == null || Object.keys(filters).length === 0)) {
+                    throw "Deletion must be filtered";
+                }
+                let json = {
+                    endpoint: {
+                        datasourceId,
+                        entityId: tableName,
+                        operation,
+                    },
+                    resource: {
+                        // have to specify the fields to avoid column overlap (for SQL)
+                        fields: isSql ? this.buildFields(table) : [],
+                    },
+                    filters,
+                    sort,
+                    paginate,
+                    relationships,
+                    body: row || rows,
+                    // pass an id filter into extra, purely for mysql/returning
+                    extra: {
+                        idFilter: buildFilters(id || generateIdForRow(row, table), {}, table),
+                    },
+                    meta: {
+                        table,
+                    },
+                };
+                // can't really use response right now
+                const response = yield (0, utils_2.getDatasourceAndQuery)(json);
+                // handle many to many relationships now if we know the ID (could be auto increment)
+                if (operation !== constants_1.DataSourceOperation.READ &&
+                    processed.manyRelationships) {
+                    yield this.handleManyRelationships(table._id || "", response[0], processed.manyRelationships);
+                }
+                const output = this.outputProcessing(response, table, relationships);
+                // if reading it'll just be an array of rows, return whole thing
+                return operation === constants_1.DataSourceOperation.READ && Array.isArray(response)
+                    ? output
+                    : { row: output[0], table };
+            });
+        }
     }
-}
-exports.ExternalRequest = ExternalRequest;
+    module.exports = ExternalRequest;
+})(External || (External = {}));