@budibase/backend-core 2.31.7 → 2.32.0

package/dist/src/sql/sql.js

@@ -43,12 +43,19 @@ const types_1 = require("@budibase/types");
 const environment_1 = __importDefault(require("../environment"));
 const shared_core_1 = require("@budibase/shared-core");
 const lodash_1 = require("lodash");
+const MAX_SQS_RELATIONSHIP_FIELDS = 63;
 function getBaseLimit() {
     const envLimit = environment_1.default.SQL_MAX_ROWS
         ? parseInt(environment_1.default.SQL_MAX_ROWS)
         : null;
     return envLimit || 5000;
 }
+function getRelationshipLimit() {
+    const envLimit = environment_1.default.SQL_MAX_RELATED_ROWS
+        ? parseInt(environment_1.default.SQL_MAX_RELATED_ROWS)
+        : null;
+    return envLimit || 500;
+}
 function getTableName(table) {
     // SQS uses the table ID rather than the table name
     if ((table === null || table === void 0 ? void 0 : table.sourceType) === types_1.TableSourceType.INTERNAL ||
@@ -77,6 +84,19 @@ function convertBooleans(query) {
 }
 class InternalBuilder {
     constructor(client, knex, query) {
+        // states the various situations in which we need a full mapped select statement
+        this.SPECIAL_SELECT_CASES = {
+            POSTGRES_MONEY: (field) => {
+                var _a;
+                return (this.client === types_1.SqlClient.POSTGRES &&
+                    ((_a = field === null || field === void 0 ? void 0 : field.externalType) === null || _a === void 0 ? void 0 : _a.includes("money")));
+            },
+            MSSQL_DATES: (field) => {
+                return (this.client === types_1.SqlClient.MS_SQL &&
+                    (field === null || field === void 0 ? void 0 : field.type) === types_1.FieldType.DATETIME &&
+                    field.timeOnly);
+            },
+        };
         this.client = client;
         this.query = query;
         this.knex = knex;
@@ -114,60 +134,58 @@ class InternalBuilder {
             .map(part => this.quote(part))
             .join(".");
     }
+    isFullSelectStatementRequired() {
+        const { meta } = this.query;
+        for (let column of Object.values(meta.table.schema)) {
+            if (this.SPECIAL_SELECT_CASES.POSTGRES_MONEY(column)) {
+                return true;
+            }
+            else if (this.SPECIAL_SELECT_CASES.MSSQL_DATES(column)) {
+                return true;
+            }
+        }
+        return false;
+    }
     generateSelectStatement() {
-        const { resource, meta } = this.query;
+        const { meta, endpoint, resource, tableAliases } = this.query;
         if (!resource || !resource.fields || resource.fields.length === 0) {
             return "*";
         }
+        const alias = (tableAliases === null || tableAliases === void 0 ? void 0 : tableAliases[endpoint.entityId])
+            ? tableAliases === null || tableAliases === void 0 ? void 0 : tableAliases[endpoint.entityId]
+            : endpoint.entityId;
         const schema = meta.table.schema;
-        return resource.fields.map(field => {
-            var _a;
+        if (!this.isFullSelectStatementRequired()) {
+            return [this.knex.raw(`${this.quote(alias)}.*`)];
+        }
+        // get just the fields for this table
+        return resource.fields
+            .map(field => {
             const parts = field.split(/\./g);
             let table = undefined;
-            let column = undefined;
+            let column = parts[0];
             // Just a column name, e.g.: "column"
-            if (parts.length === 1) {
-                column = parts[0];
-            }
-            // A table name and a column name, e.g.: "table.column"
-            if (parts.length === 2) {
-                table = parts[0];
-                column = parts[1];
-            }
-            // A link doc, e.g.: "table.doc1.fieldName"
-            if (parts.length > 2) {
+            if (parts.length > 1) {
                 table = parts[0];
                 column = parts.slice(1).join(".");
             }
-            if (!column) {
-                throw new Error(`Invalid field name: ${field}`);
-            }
+            return { table, column, field };
+        })
+            .filter(({ table }) => !table || table === alias)
+            .map(({ table, column, field }) => {
             const columnSchema = schema[column];
-            if (this.client === types_1.SqlClient.POSTGRES &&
-                ((_a = columnSchema === null || columnSchema === void 0 ? void 0 : columnSchema.externalType) === null || _a === void 0 ? void 0 : _a.includes("money"))) {
+            if (this.SPECIAL_SELECT_CASES.POSTGRES_MONEY(columnSchema)) {
                 return this.knex.raw(`${this.quotedIdentifier([table, column].join("."))}::money::numeric as ${this.quote(field)}`);
             }
-            if (this.client === types_1.SqlClient.MS_SQL &&
-                (columnSchema === null || columnSchema === void 0 ? void 0 : columnSchema.type) === types_1.FieldType.DATETIME &&
-                columnSchema.timeOnly) {
+            if (this.SPECIAL_SELECT_CASES.MSSQL_DATES(columnSchema)) {
                 // Time gets returned as timestamp from mssql, not matching the expected
                 // HH:mm format
                 return this.knex.raw(`CONVERT(varchar, ${field}, 108) as "${field}"`);
             }
-            // There's at least two edge cases being handled in the expression below.
-            //  1. The column name could start/end with a space, and in that case we
-            //     want to preseve that space.
-            //  2. Almost all column names are specified in the form table.column, except
-            //     in the case of relationships, where it's table.doc1.column. In that
-            //     case, we want to split it into `table`.`doc1.column` for reasons that
-            //     aren't actually clear to me, but `table`.`doc1` breaks things with the
-            //     sample data tests.
-            if (table) {
-                return this.knex.raw(`${this.quote(table)}.${this.quote(column)} as ${this.quote(field)}`);
-            }
-            else {
-                return this.knex.raw(`${this.quote(field)} as ${this.quote(field)}`);
-            }
+            const quoted = table
+                ? `${this.quote(table)}.${this.quote(column)}`
+                : this.quote(field);
+            return this.knex.raw(quoted);
         });
     }
     // OracleDB can't use character-large-objects (CLOBs) in WHERE clauses,
@@ -285,54 +303,118 @@ class InternalBuilder {
         }
         return filters;
     }
+    addJoinFieldCheck(query, relationship) {
+        var _a;
+        const document = ((_a = relationship.from) === null || _a === void 0 ? void 0 : _a.split(".")[0]) || "";
+        return query.andWhere(`${document}.fieldName`, "=", relationship.column);
+    }
+    addRelationshipForFilter(query, filterKey, whereCb) {
+        const mainKnex = this.knex;
+        const { relationships, endpoint, tableAliases: aliases } = this.query;
+        const tableName = endpoint.entityId;
+        const fromAlias = (aliases === null || aliases === void 0 ? void 0 : aliases[tableName]) || tableName;
+        const matches = (possibleTable) => filterKey.startsWith(`${possibleTable}`);
+        if (!relationships) {
+            return query;
+        }
+        for (const relationship of relationships) {
+            const relatedTableName = relationship.tableName;
+            const toAlias = (aliases === null || aliases === void 0 ? void 0 : aliases[relatedTableName]) || relatedTableName;
+            // this is the relationship which is being filtered
+            if ((matches(relatedTableName) || matches(toAlias)) &&
+                relationship.to &&
+                relationship.tableName) {
+                let subQuery = mainKnex
+                    .select(mainKnex.raw(1))
+                    .from({ [toAlias]: relatedTableName });
+                const manyToMany = (0, utils_1.validateManyToMany)(relationship);
+                if (manyToMany) {
+                    const throughAlias = (aliases === null || aliases === void 0 ? void 0 : aliases[manyToMany.through]) || relationship.through;
+                    let throughTable = this.tableNameWithSchema(manyToMany.through, {
+                        alias: throughAlias,
+                        schema: endpoint.schema,
+                    });
+                    subQuery = subQuery
+                        // add a join through the junction table
+                        .innerJoin(throughTable, function () {
+                        // @ts-ignore
+                        this.on(`${toAlias}.${manyToMany.toPrimary}`, "=", `${throughAlias}.${manyToMany.to}`);
+                    })
+                        // check the document in the junction table points to the main table
+                        .where(`${throughAlias}.${manyToMany.from}`, "=", mainKnex.raw(this.quotedIdentifier(`${fromAlias}.${manyToMany.fromPrimary}`)));
+                    // in SQS the same junction table is used for different many-to-many relationships between the
+                    // two same tables, this is needed to avoid rows ending up in all columns
+                    if (this.client === types_1.SqlClient.SQL_LITE) {
+                        subQuery = this.addJoinFieldCheck(subQuery, manyToMany);
+                    }
+                }
+                else {
+                    // "join" to the main table, making sure the ID matches that of the main
+                    subQuery = subQuery.where(`${toAlias}.${relationship.to}`, "=", mainKnex.raw(this.quotedIdentifier(`${fromAlias}.${relationship.from}`)));
+                }
+                query = query.whereExists(whereCb(subQuery));
+                break;
+            }
+        }
+        return query;
+    }
     // right now we only do filters on the specific table being queried
     addFilters(query, filters, opts) {
         if (!filters) {
             return query;
         }
+        const builder = this;
         filters = this.parseFilters(Object.assign({}, filters));
         const aliases = this.query.tableAliases;
         // if all or specified in filters, then everything is an or
         const allOr = filters.allOr;
-        const tableName = this.client === types_1.SqlClient.SQL_LITE ? this.table._id : this.table.name;
+        const isSqlite = this.client === types_1.SqlClient.SQL_LITE;
+        const tableName = isSqlite ? this.table._id : this.table.name;
         function getTableAlias(name) {
             const alias = aliases === null || aliases === void 0 ? void 0 : aliases[name];
             return alias || name;
         }
         function iterate(structure, fn, complexKeyFn) {
+            const handleRelationship = (q, key, value) => {
+                const [filterTableName, ...otherProperties] = key.split(".");
+                const property = otherProperties.join(".");
+                const alias = getTableAlias(filterTableName);
+                return fn(q, alias ? `${alias}.${property}` : property, value);
+            };
             for (const key in structure) {
                 const value = structure[key];
                 const updatedKey = dbCore.removeKeyNumbering(key);
                 const isRelationshipField = updatedKey.includes(".");
+                const shouldProcessRelationship = (opts === null || opts === void 0 ? void 0 : opts.relationship) && isRelationshipField;
                 let castedTypeValue;
                 if (key === types_1.InternalSearchFilterOperator.COMPLEX_ID_OPERATOR &&
                     (castedTypeValue = structure[key]) &&
                     complexKeyFn) {
                     const alias = getTableAlias(tableName);
-                    complexKeyFn(castedTypeValue.id.map((x) => alias ? `${alias}.${x}` : x), castedTypeValue.values);
+                    query = complexKeyFn(query, castedTypeValue.id.map((x) => alias ? `${alias}.${x}` : x), castedTypeValue.values);
                 }
                 else if (!isRelationshipField) {
                     const alias = getTableAlias(tableName);
-                    fn(alias ? `${alias}.${updatedKey}` : updatedKey, value);
+                    query = fn(query, alias ? `${alias}.${updatedKey}` : updatedKey, value);
                 }
-                if ((opts === null || opts === void 0 ? void 0 : opts.relationship) && isRelationshipField) {
-                    const [filterTableName, property] = updatedKey.split(".");
-                    const alias = getTableAlias(filterTableName);
-                    fn(alias ? `${alias}.${property}` : property, value);
+                else if (shouldProcessRelationship) {
+                    query = builder.addRelationshipForFilter(query, updatedKey, q => {
+                        return handleRelationship(q, updatedKey, value);
+                    });
                 }
             }
         }
-        const like = (key, value) => {
+        const like = (q, key, value) => {
             const fuzzyOr = filters === null || filters === void 0 ? void 0 : filters.fuzzyOr;
             const fnc = fuzzyOr || allOr ? "orWhere" : "where";
             // postgres supports ilike, nothing else does
             if (this.client === types_1.SqlClient.POSTGRES) {
-                query = query[fnc](key, "ilike", `%${value}%`);
+                return q[fnc](key, "ilike", `%${value}%`);
             }
             else {
                 const rawFnc = `${fnc}Raw`;
                 // @ts-ignore
-                query = query[rawFnc](`LOWER(${this.quotedIdentifier(key)}) LIKE ?`, [
+                return q[rawFnc](`LOWER(${this.quotedIdentifier(key)}) LIKE ?`, [
                     `%${value.toLowerCase()}%`,
                 ]);
             }
@@ -349,24 +431,24 @@ class InternalBuilder {
                 return `[${value.join(",")}]`;
             }
             if (this.client === types_1.SqlClient.POSTGRES) {
-                iterate(mode, (key, value) => {
+                iterate(mode, (q, key, value) => {
                     const wrap = any ? "" : "'";
                     const op = any ? "\\?| array" : "@>";
                     const fieldNames = key.split(/\./g);
                     const table = fieldNames[0];
                     const col = fieldNames[1];
-                    query = query[rawFnc](`${not}COALESCE("${table}"."${col}"::jsonb ${op} ${wrap}${stringifyArray(value, any ? "'" : '"')}${wrap}, FALSE)`);
+                    return q[rawFnc](`${not}COALESCE("${table}"."${col}"::jsonb ${op} ${wrap}${stringifyArray(value, any ? "'" : '"')}${wrap}, FALSE)`);
                 });
             }
             else if (this.client === types_1.SqlClient.MY_SQL) {
                 const jsonFnc = any ? "JSON_OVERLAPS" : "JSON_CONTAINS";
-                iterate(mode, (key, value) => {
-                    query = query[rawFnc](`${not}COALESCE(${jsonFnc}(${key}, '${stringifyArray(value)}'), FALSE)`);
+                iterate(mode, (q, key, value) => {
+                    return q[rawFnc](`${not}COALESCE(${jsonFnc}(${key}, '${stringifyArray(value)}'), FALSE)`);
                 });
             }
             else {
                 const andOr = mode === (filters === null || filters === void 0 ? void 0 : filters.containsAny) ? " OR " : " AND ";
-                iterate(mode, (key, value) => {
+                iterate(mode, (q, key, value) => {
                     let statement = "";
                     const identifier = this.quotedIdentifier(key);
                     for (let i in value) {
@@ -379,13 +461,13 @@ class InternalBuilder {
                         statement += `${statement ? andOr : ""}COALESCE(LOWER(${identifier}), '') LIKE ?`;
                     }
                     if (statement === "") {
-                        return;
+                        return q;
                     }
                     if (not) {
-                        query = query[rawFnc](`(NOT (${statement}) OR ${identifier} IS NULL)`, value);
+                        return q[rawFnc](`(NOT (${statement}) OR ${identifier} IS NULL)`, value);
                     }
                     else {
-                        query = query[rawFnc](statement, value);
+                        return q[rawFnc](statement, value);
                     }
                 });
             }
@@ -408,40 +490,40 @@ class InternalBuilder {
         }
         if (filters.oneOf) {
             const fnc = allOr ? "orWhereIn" : "whereIn";
-            iterate(filters.oneOf, (key, array) => {
+            iterate(filters.oneOf, (q, key, array) => {
                 if (this.client === types_1.SqlClient.ORACLE) {
                     key = this.convertClobs(key);
                     array = Array.isArray(array) ? array : [array];
                     const binding = new Array(array.length).fill("?").join(",");
-                    query = query.whereRaw(`${key} IN (${binding})`, array);
+                    return q.whereRaw(`${key} IN (${binding})`, array);
                 }
                 else {
-                    query = query[fnc](key, Array.isArray(array) ? array : [array]);
+                    return q[fnc](key, Array.isArray(array) ? array : [array]);
                 }
-            }, (key, array) => {
+            }, (q, key, array) => {
                 if (this.client === types_1.SqlClient.ORACLE) {
                     const keyStr = `(${key.map(k => this.convertClobs(k)).join(",")})`;
                     const binding = `(${array
                         .map((a) => `(${new Array(a.length).fill("?").join(",")})`)
                         .join(",")})`;
-                    query = query.whereRaw(`${keyStr} IN ${binding}`, array.flat());
+                    return q.whereRaw(`${keyStr} IN ${binding}`, array.flat());
                 }
                 else {
-                    query = query[fnc](key, Array.isArray(array) ? array : [array]);
+                    return q[fnc](key, Array.isArray(array) ? array : [array]);
                 }
             });
         }
         if (filters.string) {
-            iterate(filters.string, (key, value) => {
+            iterate(filters.string, (q, key, value) => {
                 const fnc = allOr ? "orWhere" : "where";
                 // postgres supports ilike, nothing else does
                 if (this.client === types_1.SqlClient.POSTGRES) {
-                    query = query[fnc](key, "ilike", `${value}%`);
+                    return q[fnc](key, "ilike", `${value}%`);
                 }
                 else {
                     const rawFnc = `${fnc}Raw`;
                     // @ts-ignore
-                    query = query[rawFnc](`LOWER(${this.quotedIdentifier(key)}) LIKE ?`, [
+                    return q[rawFnc](`LOWER(${this.quotedIdentifier(key)}) LIKE ?`, [
                         `${value.toLowerCase()}%`,
                     ]);
                 }
@@ -451,7 +533,7 @@ class InternalBuilder {
             iterate(filters.fuzzy, like);
         }
         if (filters.range) {
-            iterate(filters.range, (key, value) => {
+            iterate(filters.range, (q, key, value) => {
                 const isEmptyObject = (val) => {
                     return (val &&
                         Object.keys(val).length === 0 &&
@@ -472,75 +554,86 @@ class InternalBuilder {
                 if (lowValid && highValid) {
                     if ((schema === null || schema === void 0 ? void 0 : schema.type) === types_1.FieldType.BIGINT &&
                         this.client === types_1.SqlClient.SQL_LITE) {
-                        query = query.whereRaw(`CAST(${key} AS INTEGER) BETWEEN CAST(? AS INTEGER) AND CAST(? AS INTEGER)`, [value.low, value.high]);
+                        return q.whereRaw(`CAST(${key} AS INTEGER) BETWEEN CAST(? AS INTEGER) AND CAST(? AS INTEGER)`, [value.low, value.high]);
                     }
                     else {
                         const fnc = allOr ? "orWhereBetween" : "whereBetween";
-                        query = query[fnc](key, [value.low, value.high]);
+                        return q[fnc](key, [value.low, value.high]);
                     }
                 }
                 else if (lowValid) {
                     if ((schema === null || schema === void 0 ? void 0 : schema.type) === types_1.FieldType.BIGINT &&
                         this.client === types_1.SqlClient.SQL_LITE) {
-                        query = query.whereRaw(`CAST(${key} AS INTEGER) >= CAST(? AS INTEGER)`, [value.low]);
+                        return q.whereRaw(`CAST(${key} AS INTEGER) >= CAST(? AS INTEGER)`, [
+                            value.low,
+                        ]);
                     }
                     else {
                         const fnc = allOr ? "orWhere" : "where";
-                        query = query[fnc](key, ">=", value.low);
+                        return q[fnc](key, ">=", value.low);
                     }
                 }
                 else if (highValid) {
                     if ((schema === null || schema === void 0 ? void 0 : schema.type) === types_1.FieldType.BIGINT &&
                         this.client === types_1.SqlClient.SQL_LITE) {
-                        query = query.whereRaw(`CAST(${key} AS INTEGER) <= CAST(? AS INTEGER)`, [value.high]);
+                        return q.whereRaw(`CAST(${key} AS INTEGER) <= CAST(? AS INTEGER)`, [
+                            value.high,
+                        ]);
                     }
                     else {
                         const fnc = allOr ? "orWhere" : "where";
-                        query = query[fnc](key, "<=", value.high);
+                        return q[fnc](key, "<=", value.high);
                     }
                 }
+                return q;
             });
         }
         if (filters.equal) {
-            iterate(filters.equal, (key, value) => {
+            iterate(filters.equal, (q, key, value) => {
                 const fnc = allOr ? "orWhereRaw" : "whereRaw";
                 if (this.client === types_1.SqlClient.MS_SQL) {
-                    query = query[fnc](`CASE WHEN ${this.quotedIdentifier(key)} = ? THEN 1 ELSE 0 END = 1`, [value]);
+                    return q[fnc](`CASE WHEN ${this.quotedIdentifier(key)} = ? THEN 1 ELSE 0 END = 1`, [value]);
                 }
                 else if (this.client === types_1.SqlClient.ORACLE) {
                     const identifier = this.convertClobs(key);
-                    query = query[fnc](`(${identifier} IS NOT NULL AND ${identifier} = ?)`, [value]);
+                    return q[fnc](`(${identifier} IS NOT NULL AND ${identifier} = ?)`, [
+                        value,
+                    ]);
                 }
                 else {
-                    query = query[fnc](`COALESCE(${this.quotedIdentifier(key)} = ?, FALSE)`, [value]);
+                    return q[fnc](`COALESCE(${this.quotedIdentifier(key)} = ?, FALSE)`, [
+                        value,
+                    ]);
                 }
             });
         }
         if (filters.notEqual) {
-            iterate(filters.notEqual, (key, value) => {
+            iterate(filters.notEqual, (q, key, value) => {
                 const fnc = allOr ? "orWhereRaw" : "whereRaw";
                 if (this.client === types_1.SqlClient.MS_SQL) {
-                    query = query[fnc](`CASE WHEN ${this.quotedIdentifier(key)} = ? THEN 1 ELSE 0 END = 0`, [value]);
+                    return q[fnc](`CASE WHEN ${this.quotedIdentifier(key)} = ? THEN 1 ELSE 0 END = 0`, [value]);
                 }
                 else if (this.client === types_1.SqlClient.ORACLE) {
                     const identifier = this.convertClobs(key);
-                    query = query[fnc](`(${identifier} IS NOT NULL AND ${identifier} != ?) OR ${identifier} IS NULL`, [value]);
+                    return q[fnc](`(${identifier} IS NOT NULL AND ${identifier} != ?) OR ${identifier} IS NULL`, [value]);
                 }
                 else {
-                    query = query[fnc](`COALESCE(${this.quotedIdentifier(key)} != ?, TRUE)`, [value]);
+                    return q[fnc](`COALESCE(${this.quotedIdentifier(key)} != ?, TRUE)`, [
+                        value,
+                    ]);
                 }
             });
         }
         if (filters.empty) {
-            iterate(filters.empty, key => {
+            iterate(filters.empty, (q, key) => {
                 const fnc = allOr ? "orWhereNull" : "whereNull";
-                query = query[fnc](key);
+                return q[fnc](key);
             });
         }
         if (filters.notEmpty) {
-            iterate(filters.notEmpty, key => {
+            iterate(filters.notEmpty, (q, key) => {
                 const fnc = allOr ? "orWhereNotNull" : "whereNotNull";
-                query = query[fnc](key);
+                return q[fnc](key);
             });
         }
         if (filters.contains) {
@@ -614,10 +707,162 @@ class InternalBuilder {
         }
         return withSchema;
     }
-    addRelationships(query, fromTable, relationships, schema, aliases) {
-        if (!relationships) {
-            return query;
+    addJsonRelationships(query, fromTable, relationships) {
+        const sqlClient = this.client;
+        const knex = this.knex;
+        const { resource, tableAliases: aliases, endpoint } = this.query;
+        const fields = (resource === null || resource === void 0 ? void 0 : resource.fields) || [];
+        const jsonField = (field) => {
+            const parts = field.split(".");
+            let tableField, unaliased;
+            if (parts.length > 1) {
+                const alias = parts.shift();
+                unaliased = parts.join(".");
+                tableField = `${this.quote(alias)}.${this.quote(unaliased)}`;
+            }
+            else {
+                unaliased = parts.join(".");
+                tableField = this.quote(unaliased);
+            }
+            let separator = ",";
+            switch (sqlClient) {
+                case types_1.SqlClient.ORACLE:
+                    separator = " VALUE ";
+                    break;
+                case types_1.SqlClient.MS_SQL:
+                    separator = ":";
+            }
+            return `'${unaliased}'${separator}${tableField}`;
+        };
+        for (let relationship of relationships) {
+            const { tableName: toTable, through: throughTable, to: toKey, from: fromKey, fromPrimary, toPrimary, } = relationship;
+            // skip invalid relationships
+            if (!toTable || !fromTable) {
+                continue;
+            }
+            const toAlias = (aliases === null || aliases === void 0 ? void 0 : aliases[toTable]) || toTable, fromAlias = (aliases === null || aliases === void 0 ? void 0 : aliases[fromTable]) || fromTable;
+            let toTableWithSchema = this.tableNameWithSchema(toTable, {
+                alias: toAlias,
+                schema: endpoint.schema,
+            });
+            let relationshipFields = fields.filter(field => field.split(".")[0] === toAlias);
+            if (this.client === types_1.SqlClient.SQL_LITE) {
+                relationshipFields = relationshipFields.slice(0, MAX_SQS_RELATIONSHIP_FIELDS);
+            }
+            const fieldList = relationshipFields
+                .map(field => jsonField(field))
+                .join(",");
+            // SQL Server uses TOP - which performs a little differently to the normal LIMIT syntax
+            // it reduces the result set rather than limiting how much data it filters over
+            const primaryKey = `${toAlias}.${toPrimary || toKey}`;
+            let subQuery = knex
+                .from(toTableWithSchema)
+                .limit(getRelationshipLimit())
+                // add sorting to get consistent order
+                .orderBy(primaryKey);
+            // many-to-many relationship with junction table
+            if (throughTable && toPrimary && fromPrimary) {
+                const throughAlias = (aliases === null || aliases === void 0 ? void 0 : aliases[throughTable]) || throughTable;
+                let throughTableWithSchema = this.tableNameWithSchema(throughTable, {
+                    alias: throughAlias,
+                    schema: endpoint.schema,
+                });
+                subQuery = subQuery
+                    .join(throughTableWithSchema, function () {
+                    this.on(`${toAlias}.${toPrimary}`, "=", `${throughAlias}.${toKey}`);
+                })
+                    .where(`${throughAlias}.${fromKey}`, "=", knex.raw(this.quotedIdentifier(`${fromAlias}.${fromPrimary}`)));
+            }
+            // one-to-many relationship with foreign key
+            else {
+                subQuery = subQuery.where(`${toAlias}.${toKey}`, "=", knex.raw(this.quotedIdentifier(`${fromAlias}.${fromKey}`)));
+            }
+            const standardWrap = (select) => {
+                subQuery = subQuery.select(`${toAlias}.*`);
+                // @ts-ignore - the from alias syntax isn't in Knex typing
+                return knex.select(knex.raw(select)).from({
+                    [toAlias]: subQuery,
+                });
+            };
+            let wrapperQuery;
+            switch (sqlClient) {
+                case types_1.SqlClient.SQL_LITE:
+                    // need to check the junction table document is to the right column, this is just for SQS
+                    subQuery = this.addJoinFieldCheck(subQuery, relationship);
+                    wrapperQuery = standardWrap(`json_group_array(json_object(${fieldList}))`);
+                    break;
+                case types_1.SqlClient.POSTGRES:
+                    wrapperQuery = standardWrap(`json_agg(json_build_object(${fieldList}))`);
+                    break;
+                case types_1.SqlClient.MY_SQL:
+                    wrapperQuery = subQuery.select(knex.raw(`json_arrayagg(json_object(${fieldList}))`));
+                    break;
+                case types_1.SqlClient.ORACLE:
+                    wrapperQuery = standardWrap(`json_arrayagg(json_object(${fieldList}))`);
+                    break;
+                case types_1.SqlClient.MS_SQL:
+                    wrapperQuery = knex.raw(`(SELECT ${this.quote(toAlias)} = (${knex
+                        .select(`${fromAlias}.*`)
+                        // @ts-ignore - from alias syntax not TS supported
+                        .from({
+                        [fromAlias]: subQuery.select(`${toAlias}.*`),
+                    })} FOR JSON PATH))`);
+                    break;
+                default:
+                    throw new Error(`JSON relationships not implement for ${sqlClient}`);
+            }
+            query = query.select({ [relationship.column]: wrapperQuery });
         }
+        return query;
+    }
+    addJoin(query, tables, columns) {
+        const { tableAliases: aliases, endpoint } = this.query;
+        const schema = endpoint.schema;
+        const toTable = tables.to, fromTable = tables.from, throughTable = tables.through;
+        const toAlias = (aliases === null || aliases === void 0 ? void 0 : aliases[toTable]) || toTable, throughAlias = (throughTable && (aliases === null || aliases === void 0 ? void 0 : aliases[throughTable])) || throughTable, fromAlias = (aliases === null || aliases === void 0 ? void 0 : aliases[fromTable]) || fromTable;
+        let toTableWithSchema = this.tableNameWithSchema(toTable, {
+            alias: toAlias,
+            schema,
+        });
+        let throughTableWithSchema = throughTable
+            ? this.tableNameWithSchema(throughTable, {
+                alias: throughAlias,
+                schema,
+            })
+            : undefined;
+        if (!throughTable) {
+            // @ts-ignore
+            query = query.leftJoin(toTableWithSchema, function () {
+                for (let relationship of columns) {
+                    const from = relationship.from, to = relationship.to;
+                    // @ts-ignore
+                    this.orOn(`${fromAlias}.${from}`, "=", `${toAlias}.${to}`);
+                }
+            });
+        }
+        else {
+            query = query
+                // @ts-ignore
+                .leftJoin(throughTableWithSchema, function () {
+                for (let relationship of columns) {
+                    const fromPrimary = relationship.fromPrimary;
+                    const from = relationship.from;
+                    // @ts-ignore
+                    this.orOn(`${fromAlias}.${fromPrimary}`, "=", `${throughAlias}.${from}`);
+                }
+            })
+                .leftJoin(toTableWithSchema, function () {
+                for (let relationship of columns) {
+                    const toPrimary = relationship.toPrimary;
+                    const to = relationship.to;
+                    // @ts-ignore
+                    this.orOn(`${toAlias}.${toPrimary}`, `${throughAlias}.${to}`);
+                }
+            });
+        }
+        return query;
+    }
+    addRelationships(query, fromTable, relationships) {
         const tableSets = {};
         // aggregate into table sets (all the same to tables)
         for (let relationship of relationships) {
@@ -638,45 +883,11 @@ class InternalBuilder {
         }
         for (let [key, relationships] of Object.entries(tableSets)) {
             const { toTable, throughTable } = JSON.parse(key);
-            const toAlias = (aliases === null || aliases === void 0 ? void 0 : aliases[toTable]) || toTable, throughAlias = (aliases === null || aliases === void 0 ? void 0 : aliases[throughTable]) || throughTable, fromAlias = (aliases === null || aliases === void 0 ? void 0 : aliases[fromTable]) || fromTable;
-            let toTableWithSchema = this.tableNameWithSchema(toTable, {
-                alias: toAlias,
-                schema,
-            });
-            let throughTableWithSchema = this.tableNameWithSchema(throughTable, {
-                alias: throughAlias,
-                schema,
-            });
-            if (!throughTable) {
-                // @ts-ignore
-                query = query.leftJoin(toTableWithSchema, function () {
-                    for (let relationship of relationships) {
-                        const from = relationship.from, to = relationship.to;
-                        // @ts-ignore
-                        this.orOn(`${fromAlias}.${from}`, "=", `${toAlias}.${to}`);
-                    }
-                });
-            }
-            else {
-                query = query
-                    // @ts-ignore
-                    .leftJoin(throughTableWithSchema, function () {
-                    for (let relationship of relationships) {
-                        const fromPrimary = relationship.fromPrimary;
-                        const from = relationship.from;
-                        // @ts-ignore
-                        this.orOn(`${fromAlias}.${fromPrimary}`, "=", `${throughAlias}.${from}`);
-                    }
-                })
-                    .leftJoin(toTableWithSchema, function () {
-                    for (let relationship of relationships) {
-                        const toPrimary = relationship.toPrimary;
-                        const to = relationship.to;
-                        // @ts-ignore
-                        this.orOn(`${toAlias}.${toPrimary}`, `${throughAlias}.${to}`);
-                    }
-                });
-            }
+            query = this.addJoin(query, {
+                from: fromTable,
+                to: toTable,
+                through: throughTable,
+            }, relationships);
         }
         return query;
     }
@@ -766,7 +977,7 @@ class InternalBuilder {
         return query.upsert(parsedBody);
     }
     read(opts = {}) {
-        let { endpoint, filters, paginate, relationships, tableAliases } = this.query;
+        let { endpoint, filters, paginate, relationships } = this.query;
         const { limits } = opts;
         const counting = endpoint.operation === types_1.Operation.COUNT;
         const tableName = endpoint.entityId;
@@ -799,34 +1010,18 @@ class InternalBuilder {
             if (foundOffset != null) {
                 query = query.offset(foundOffset);
             }
-            // add sorting to pre-query
-            // no point in sorting when counting
-            query = this.addSorting(query);
         }
-        // add filters to the query (where)
-        query = this.addFilters(query, filters);
-        const alias = (tableAliases === null || tableAliases === void 0 ? void 0 : tableAliases[tableName]) || tableName;
-        let preQuery = this.knex({
-            // the typescript definition for the knex constructor doesn't support this
-            // syntax, but it is the only way to alias a pre-query result as part of
-            // a query - there is an alias dictionary type, but it assumes it can only
-            // be a table name, not a pre-query
-            [alias]: query,
-        });
         // if counting, use distinct count, else select
-        preQuery = !counting
-            ? preQuery.select(this.generateSelectStatement())
-            : this.addDistinctCount(preQuery);
+        query = !counting
+            ? query.select(this.generateSelectStatement())
+            : this.addDistinctCount(query);
         // have to add after as well (this breaks MS-SQL)
-        if (this.client !== types_1.SqlClient.MS_SQL && !counting) {
-            preQuery = this.addSorting(preQuery);
+        if (!counting) {
+            query = this.addSorting(query);
         }
         // handle joins
-        query = this.addRelationships(preQuery, tableName, relationships, endpoint.schema, tableAliases);
-        // add a base limit over the whole query
-        // if counting we can't set this limit
-        if (limits === null || limits === void 0 ? void 0 : limits.base) {
-            query = query.limit(limits.base);
+        if (relationships) {
+            query = this.addJsonRelationships(query, tableName, relationships);
         }
         return this.addFilters(query, filters, { relationship: true });
     }