@mikro-orm/knex 6.0.0-dev.11 → 6.0.0-dev.111

package/AbstractSqlDriver.js

@@ -5,9 +5,12 @@ const core_1 = require("@mikro-orm/core");
 const query_1 = require("./query");
 const SqlEntityManager_1 = require("./SqlEntityManager");
 class AbstractSqlDriver extends core_1.DatabaseDriver {
+    [core_1.EntityManagerType];
+    connection;
+    replicas = [];
+    platform;
     constructor(config, platform, connection, connector) {
         super(config, connector);
-        this.replicas = [];
         this.connection = new connection(this.config);
         this.replicas = this.createReplicas(conf => new connection(this.config, conf, 'read'));
         this.platform = platform;
@@ -26,19 +29,31 @@ class AbstractSqlDriver extends core_1.DatabaseDriver {
         }
         const populate = this.autoJoinOneToOneOwner(meta, options.populate, options.fields);
         const joinedProps = this.joinedProps(meta, populate);
-        const qb = this.createQueryBuilder(entityName, options.ctx, options.connectionType, false);
+        const qb = this.createQueryBuilder(entityName, options.ctx, options.connectionType, false, options.logging);
         const fields = this.buildFields(meta, populate, joinedProps, qb, options.fields);
         const joinedPropsOrderBy = this.buildJoinedPropsOrderBy(entityName, qb, meta, joinedProps);
+        const orderBy = [...core_1.Utils.asArray(options.orderBy), ...joinedPropsOrderBy];
         if (core_1.Utils.isPrimaryKey(where, meta.compositePK)) {
             where = { [core_1.Utils.getPrimaryKeyHash(meta.primaryKeys)]: where };
         }
+        const { first, last, before, after } = options;
+        const isCursorPagination = [first, last, before, after].some(v => v != null);
         qb.select(fields)
             .populate(populate, joinedProps.length > 0 ? options.populateWhere : undefined)
             .where(where)
-            .orderBy([...core_1.Utils.asArray(options.orderBy), ...joinedPropsOrderBy])
             .groupBy(options.groupBy)
             .having(options.having)
+            .indexHint(options.indexHint)
+            .comment(options.comments)
+            .hintComment(options.hintComments)
             .withSchema(this.getSchemaName(meta, options));
+        if (isCursorPagination) {
+            const { orderBy: newOrderBy, where } = this.processCursorOptions(meta, options, orderBy);
+            qb.andWhere(where).orderBy(newOrderBy);
+        }
+        else {
+            qb.orderBy(orderBy);
+        }
         if (options.limit !== undefined) {
             qb.limit(options.limit, options.offset);
         }
@@ -47,8 +62,11 @@ class AbstractSqlDriver extends core_1.DatabaseDriver {
         }
         core_1.Utils.asArray(options.flags).forEach(flag => qb.setFlag(flag));
         const result = await this.rethrow(qb.execute('all'));
+        if (isCursorPagination && !first && !!last) {
+            result.reverse();
+        }
         if (joinedProps.length > 0) {
-            return this.mergeJoinedResult(result, meta);
+            return this.mergeJoinedResult(result, meta, joinedProps);
         }
         return result;
     }
@@ -61,58 +79,57 @@ class AbstractSqlDriver extends core_1.DatabaseDriver {
             opts.limit = 1;
         }
         if (opts.limit > 0 && !opts.flags?.includes(core_1.QueryFlag.DISABLE_PAGINATE)) {
-            opts.flags ?? (opts.flags = []);
+            opts.flags ??= [];
             opts.flags.push(core_1.QueryFlag.DISABLE_PAGINATE);
         }
         const res = await this.find(entityName, where, opts);
         return res[0] || null;
     }
     async findVirtual(entityName, where, options) {
-        const meta = this.metadata.get(entityName);
-        /* istanbul ignore next */
-        if (!meta.expression) {
-            return [];
-        }
-        if (typeof meta.expression === 'string') {
-            return this.wrapVirtualExpressionInSubquery(meta, meta.expression, where, options);
-        }
-        const em = this.createEntityManager(false);
-        em.setTransactionContext(options.ctx);
-        const res = meta.expression(em, where, options);
-        if (res instanceof (query_1.QueryBuilder)) {
-            return this.wrapVirtualExpressionInSubquery(meta, res.getFormattedQuery(), where, options);
-        }
-        /* istanbul ignore next */
-        return res;
+        return this.findFromVirtual(entityName, where, options, query_1.QueryType.SELECT);
     }
     async countVirtual(entityName, where, options) {
+        return this.findFromVirtual(entityName, where, options, query_1.QueryType.COUNT);
+    }
+    async findFromVirtual(entityName, where, options, type) {
         const meta = this.metadata.get(entityName);
         /* istanbul ignore next */
         if (!meta.expression) {
-            return 0;
+            return type === query_1.QueryType.SELECT ? [] : 0;
         }
         if (typeof meta.expression === 'string') {
-            return this.wrapVirtualExpressionInSubquery(meta, meta.expression, where, options, query_1.QueryType.COUNT);
+            return this.wrapVirtualExpressionInSubquery(meta, meta.expression, where, options, type);
         }
         const em = this.createEntityManager(false);
         em.setTransactionContext(options.ctx);
         const res = meta.expression(em, where, options);
-        if (res instanceof (query_1.QueryBuilder)) {
-            return this.wrapVirtualExpressionInSubquery(meta, res.getFormattedQuery(), where, options, query_1.QueryType.COUNT);
+        if (typeof res === 'string') {
+            return this.wrapVirtualExpressionInSubquery(meta, res, where, options, type);
+        }
+        if (res instanceof query_1.QueryBuilder) {
+            return this.wrapVirtualExpressionInSubquery(meta, res.getFormattedQuery(), where, options, type);
+        }
+        if (core_1.Utils.isObject(res)) {
+            const { sql, bindings } = res.toSQL();
+            const query = this.platform.formatQuery(sql, bindings);
+            return this.wrapVirtualExpressionInSubquery(meta, query, where, options, type);
         }
         /* istanbul ignore next */
         return res;
     }
-    async wrapVirtualExpressionInSubquery(meta, expression, where, options, type = query_1.QueryType.SELECT) {
+    async wrapVirtualExpressionInSubquery(meta, expression, where, options, type) {
         const qb = this.createQueryBuilder(meta.className, options?.ctx, options.connectionType, options.convertCustomTypes)
-            .limit(options?.limit, options?.offset);
+            .limit(options?.limit, options?.offset)
+            .indexHint(options.indexHint)
+            .comment(options.comments)
+            .hintComment(options.hintComments);
         if (options.orderBy) {
             qb.orderBy(options.orderBy);
         }
         qb.where(where);
         const kqb = qb.getKnexQuery().clear('select');
         if (type === query_1.QueryType.COUNT) {
-            kqb.select(qb.raw('count(*) as count'));
+            kqb.select(this.connection.getKnex().raw('count(*) as count'));
         }
         else { // select
             kqb.select('*');
@@ -131,7 +148,10 @@ class AbstractSqlDriver extends core_1.DatabaseDriver {
             return null;
         }
         if (qb) {
+            // here we map the aliased results (cartesian product) to an object graph
             this.mapJoinedProps(ret, meta, populate, qb, ret, map);
+            // we need to remove the cycles from the mapped values
+            core_1.Utils.removeCycles(ret, meta);
         }
         return ret;
     }
@@ -146,19 +166,33 @@ class AbstractSqlDriver extends core_1.DatabaseDriver {
             const meta2 = this.metadata.find(relation.type);
             const path = parentJoinPath ? `${parentJoinPath}.${relation.name}` : `${meta.name}.${relation.name}`;
             const relationAlias = qb.getAliasForJoinPath(path);
-            const relationPojo = {};
+            let relationPojo = {};
             // If the primary key value for the relation is null, we know we haven't joined to anything
             // and therefore we don't return any record (since all values would be null)
             const hasPK = meta2.primaryKeys.every(pk => meta2.properties[pk].fieldNames.every(name => {
                 return root[`${relationAlias}__${name}`] != null;
             }));
             if (!hasPK) {
-                // initialize empty collections
-                if ([core_1.ReferenceType.MANY_TO_MANY, core_1.ReferenceType.ONE_TO_MANY].includes(relation.reference)) {
-                    result[relation.name] = result[relation.name] || [];
+                if ([core_1.ReferenceKind.MANY_TO_MANY, core_1.ReferenceKind.ONE_TO_MANY].includes(relation.kind)) {
+                    result[relation.name] ??= [];
+                }
+                if ([core_1.ReferenceKind.MANY_TO_ONE, core_1.ReferenceKind.ONE_TO_ONE].includes(relation.kind)) {
+                    result[relation.name] = null;
                 }
                 return;
             }
+            const tz = this.platform.getTimezone();
+            meta2.props
+                .filter(prop => prop.persist === false && prop.fieldNames)
+                .forEach(prop => {
+                if (prop.fieldNames.length > 1) { // composite keys
+                    relationPojo[prop.name] = prop.fieldNames.map(name => root[`${relationAlias}__${name}`]);
+                }
+                else {
+                    const alias = `${relationAlias}__${prop.fieldNames[0]}`;
+                    relationPojo[prop.name] = root[alias];
+                }
+            });
             meta2.props
                 .filter(prop => this.platform.shouldHaveColumn(prop, p.children || []))
                 .forEach(prop => {
@@ -166,21 +200,37 @@ class AbstractSqlDriver extends core_1.DatabaseDriver {
                     relationPojo[prop.name] = prop.fieldNames.map(name => root[`${relationAlias}__${name}`]);
                     prop.fieldNames.map(name => delete root[`${relationAlias}__${name}`]);
                 }
+                else if (prop.runtimeType === 'Date') {
+                    const alias = `${relationAlias}__${prop.fieldNames[0]}`;
+                    relationPojo[prop.name] = (typeof root[alias] === 'string' ? new Date(root[alias]) : root[alias]);
+                    delete root[alias];
+                }
                 else {
                     const alias = `${relationAlias}__${prop.fieldNames[0]}`;
                     relationPojo[prop.name] = root[alias];
                     delete root[alias];
                 }
             });
-            const key = `${meta.name}-${(core_1.Utils.getCompositeKeyHash(result, meta))}`;
+            const key = `${meta.name}-${core_1.Utils.getCompositeKeyHash(result, meta)}`;
+            const key2 = `${meta2.name}-${core_1.Utils.getCompositeKeyHash(relationPojo, meta2)}`;
+            if (map[key2]) {
+                const old = map[key2];
+                core_1.Utils.keys(relationPojo)
+                    .filter(k => relationPojo[k] != null)
+                    .forEach(k => old[k] = relationPojo[k]);
+                relationPojo = old;
+            }
+            else {
+                map[key2] = relationPojo;
+            }
             if (map[key]) {
                 result[relation.name] = map[key][relation.name];
             }
             else {
                 map[key] = result;
             }
-            if ([core_1.ReferenceType.MANY_TO_MANY, core_1.ReferenceType.ONE_TO_MANY].includes(relation.reference)) {
-                result[relation.name] = result[relation.name] || [];
+            if ([core_1.ReferenceKind.MANY_TO_MANY, core_1.ReferenceKind.ONE_TO_MANY].includes(relation.kind)) {
+                result[relation.name] ??= [];
                 this.appendToCollection(meta2, result[relation.name], relationPojo);
             }
             else {
@@ -207,6 +257,9 @@ class AbstractSqlDriver extends core_1.DatabaseDriver {
             return this.countVirtual(entityName, where, options);
         }
         const qb = this.createQueryBuilder(entityName, options.ctx, options.connectionType, false)
+            .indexHint(options.indexHint)
+            .comment(options.comments)
+            .hintComment(options.hintComments)
             .groupBy(options.groupBy)
             .having(options.having)
             .populate(options.populate ?? [])
@@ -215,7 +268,7 @@ class AbstractSqlDriver extends core_1.DatabaseDriver {
         return this.rethrow(qb.getCount());
     }
     async nativeInsert(entityName, data, options = {}) {
-        options.convertCustomTypes ?? (options.convertCustomTypes = true);
+        options.convertCustomTypes ??= true;
         const meta = this.metadata.find(entityName);
         const collections = this.extractManyToMany(entityName, data);
         const pks = meta?.primaryKeys ?? [this.config.getNamingStrategy().referenceColumnName()];
@@ -235,16 +288,20 @@ class AbstractSqlDriver extends core_1.DatabaseDriver {
         return res;
     }
     async nativeInsertMany(entityName, data, options = {}) {
-        options.processCollections ?? (options.processCollections = true);
-        options.convertCustomTypes ?? (options.convertCustomTypes = true);
-        const meta = this.metadata.find(entityName);
+        options.processCollections ??= true;
+        options.convertCustomTypes ??= true;
+        const meta = this.metadata.find(entityName)?.root;
         const collections = options.processCollections ? data.map(d => this.extractManyToMany(entityName, d)) : [];
         const pks = this.getPrimaryKeyFields(entityName);
         const set = new Set();
-        data.forEach(row => Object.keys(row).forEach(k => set.add(k)));
+        data.forEach(row => core_1.Utils.keys(row).forEach(k => set.add(k)));
         const props = [...set].map(name => meta?.properties[name] ?? { name, fieldNames: [name] });
-        const fields = core_1.Utils.flatten(props.map(prop => prop.fieldNames));
+        let fields = core_1.Utils.flatten(props.map(prop => prop.fieldNames));
+        const duplicates = core_1.Utils.findDuplicates(fields);
         const params = [];
+        if (duplicates.length) {
+            fields = core_1.Utils.unique(fields);
+        }
         /* istanbul ignore next */
         const tableName = meta ? this.getTableName(meta, options) : this.platform.quoteIdentifier(entityName);
         let sql = `insert into ${tableName} `;
@@ -255,29 +312,50 @@ class AbstractSqlDriver extends core_1.DatabaseDriver {
         else {
             sql += ' ' + data.map(() => `select null as ${this.platform.quoteIdentifier(pks[0])}`).join(' union all ');
         }
+        const addParams = (prop, row) => {
+            if (options.convertCustomTypes && prop.customType) {
+                params.push(prop.customType.convertToDatabaseValue(row[prop.name], this.platform, { key: prop.name, mode: 'query-data' }));
+                return;
+            }
+            params.push(row[prop.name]);
+        };
         if (fields.length > 0 || this.platform.usesDefaultKeyword()) {
             sql += data.map(row => {
                 const keys = [];
+                const usedDups = [];
                 props.forEach(prop => {
                     if (prop.fieldNames.length > 1) {
-                        params.push(...(row[prop.name] ?? prop.fieldNames.map(() => null)));
-                        keys.push(...prop.fieldNames.map(_ => '?'));
-                    }
-                    else if (prop.customType && 'convertToDatabaseValueSQL' in prop.customType && !this.platform.isRaw(row[prop.name])) {
-                        keys.push(prop.customType.convertToDatabaseValueSQL('?', this.platform));
-                        params.push(row[prop.name]);
+                        const param = core_1.Utils.flatten([...row[prop.name] ?? prop.fieldNames.map(() => null)]);
+                        const key = param.map(() => '?');
+                        prop.fieldNames.forEach((field, idx) => {
+                            if (!duplicates.includes(field) || !usedDups.includes(field)) {
+                                params.push(param[idx]);
+                                keys.push(key[idx]);
+                                usedDups.push(field);
+                            }
+                        });
                     }
                     else {
-                        params.push(row[prop.name]);
-                        keys.push('?');
+                        const field = prop.fieldNames[0];
+                        if (!duplicates.includes(field) || !usedDups.includes(field)) {
+                            if (prop.customType && 'convertToDatabaseValueSQL' in prop.customType && !this.platform.isRaw(row[prop.name])) {
+                                keys.push(prop.customType.convertToDatabaseValueSQL('?', this.platform));
+                            }
+                            else {
+                                keys.push('?');
+                            }
+                            addParams(prop, row);
+                            usedDups.push(field);
+                        }
                     }
                 });
                 return '(' + (keys.join(', ') || 'default') + ')';
             }).join(', ');
         }
-        if (this.platform.usesReturningStatement()) {
-            /* istanbul ignore next */
-            const returningProps = meta.hydrateProps.filter(prop => prop.persist !== false && (prop.primary || prop.defaultRaw));
+        if (meta && this.platform.usesReturningStatement()) {
+            const returningProps = meta.props
+                .filter(prop => prop.persist !== false && ((prop.primary && prop.autoincrement) || prop.defaultRaw))
+                .filter(prop => !(prop.name in data[0]) || core_1.Utils.isRawSql(data[0][prop.name]));
             const returningFields = core_1.Utils.flatten(returningProps.map(prop => prop.fieldNames));
             /* istanbul ignore next */
             sql += returningFields.length > 0 ? ` returning ${returningFields.map(field => this.platform.quoteIdentifier(field)).join(', ')}` : '';
@@ -289,8 +367,8 @@ class AbstractSqlDriver extends core_1.DatabaseDriver {
             pk = data.map(d => core_1.Utils.getPrimaryKeyCond(d, pks));
         }
         else {
-            res.row ?? (res.row = {});
-            res.rows ?? (res.rows = []);
+            res.row ??= {};
+            res.rows ??= [];
             pk = data.map((d, i) => d[pks[0]] ?? res.rows[i]?.[pks[0]]).map(d => [d]);
             res.insertId = res.insertId || res.row[pks[0]];
         }
@@ -300,7 +378,7 @@ class AbstractSqlDriver extends core_1.DatabaseDriver {
         return res;
     }
     async nativeUpdate(entityName, where, data, options = {}) {
-        options.convertCustomTypes ?? (options.convertCustomTypes = true);
+        options.convertCustomTypes ??= true;
         const meta = this.metadata.find(entityName);
         const pks = this.getPrimaryKeyFields(entityName);
         const collections = this.extractManyToMany(entityName, data);
@@ -314,11 +392,18 @@ class AbstractSqlDriver extends core_1.DatabaseDriver {
                 .withSchema(this.getSchemaName(meta, options));
             if (options.upsert) {
                 /* istanbul ignore next */
-                const uniqueFields = core_1.Utils.isPlainObject(where) ? Object.keys(where) : meta.primaryKeys;
-                /* istanbul ignore next */
+                const uniqueFields = options.onConflictFields ?? (core_1.Utils.isPlainObject(where) ? core_1.Utils.keys(where) : meta.primaryKeys);
+                const returning = (0, core_1.getOnConflictReturningFields)(meta, data, uniqueFields, options);
                 qb.insert(data)
-                    .onConflict(uniqueFields.map(p => meta?.properties[p]?.fieldNames[0] ?? p))
-                    .merge(Object.keys(data).filter(f => !uniqueFields.includes(f)));
+                    .onConflict(uniqueFields)
+                    .returning(returning);
+                if (!options.onConflictAction || options.onConflictAction === 'merge') {
+                    const fields = (0, core_1.getOnConflictFields)(data, uniqueFields, options);
+                    qb.merge(fields);
+                }
+                if (options.onConflictAction === 'ignore') {
+                    qb.ignore();
+                }
             }
             else {
                 qb.update(data).where(where);
@@ -331,20 +416,36 @@ class AbstractSqlDriver extends core_1.DatabaseDriver {
         return res;
     }
     async nativeUpdateMany(entityName, where, data, options = {}) {
-        options.processCollections ?? (options.processCollections = true);
-        options.convertCustomTypes ?? (options.convertCustomTypes = true);
+        options.processCollections ??= true;
+        options.convertCustomTypes ??= true;
         const meta = this.metadata.get(entityName);
         if (options.upsert) {
-            const uniqueFields = core_1.Utils.isPlainObject(where[0]) ? Object.keys(where[0]) : meta.primaryKeys;
+            const uniqueFields = options.onConflictFields ?? (core_1.Utils.isPlainObject(where[0]) ? Object.keys(where[0]) : meta.primaryKeys);
             const qb = this.createQueryBuilder(entityName, options.ctx, 'write', options.convertCustomTypes).withSchema(this.getSchemaName(meta, options));
+            const returning = (0, core_1.getOnConflictReturningFields)(meta, data[0], uniqueFields, options);
             qb.insert(data)
-                .onConflict(uniqueFields.map(p => meta?.properties[p]?.fieldNames[0] ?? p))
-                .merge(Object.keys(data[0]).filter(f => !uniqueFields.includes(f)));
+                .onConflict(uniqueFields)
+                .returning(returning);
+            if (!options.onConflictAction || options.onConflictAction === 'merge') {
+                const fields = (0, core_1.getOnConflictFields)(data[0], uniqueFields, options);
+                qb.merge(fields);
+            }
+            if (options.onConflictAction === 'ignore') {
+                qb.ignore();
+            }
             return qb.execute('run', false);
         }
         const collections = options.processCollections ? data.map(d => this.extractManyToMany(entityName, d)) : [];
         const keys = new Set();
-        data.forEach(row => Object.keys(row).forEach(k => keys.add(k)));
+        const returning = new Set();
+        data.forEach(row => {
+            core_1.Utils.keys(row).forEach(k => {
+                keys.add(k);
+                if (core_1.Utils.isRawSql(row[k])) {
+                    returning.add(k);
+                }
+            });
+        });
         const pkCond = core_1.Utils.flatten(meta.primaryKeys.map(pk => meta.properties[pk].fieldNames)).map(pk => `${this.platform.quoteIdentifier(pk)} = ?`).join(' and ');
         const params = [];
         let sql = `update ${this.getTableName(meta, options)} set `;
@@ -373,7 +474,7 @@ class AbstractSqlDriver extends core_1.DatabaseDriver {
             const versionProperty = meta.properties[meta.versionProperty];
             const quotedFieldName = this.platform.quoteIdentifier(versionProperty.fieldNames[0]);
             sql += `${quotedFieldName} = `;
-            if (versionProperty.type.toLowerCase() === 'date') {
+            if (versionProperty.runtimeType === 'Date') {
                 sql += this.platform.getCurrentTimestampSQL(versionProperty.length);
             }
             else {
@@ -404,6 +505,11 @@ class AbstractSqlDriver extends core_1.DatabaseDriver {
             return '?';
         });
         sql += ` in (${conds.join(', ')})`;
+        if (this.platform.usesReturningStatement() && returning.size > 0) {
+            const returningFields = core_1.Utils.flatten([...returning].map(prop => meta.properties[prop].fieldNames));
+            /* istanbul ignore next */
+            sql += returningFields.length > 0 ? ` returning ${returningFields.map(field => this.platform.quoteIdentifier(field)).join(', ')}` : '';
+        }
         const res = await this.rethrow(this.execute(sql, params, 'run', options.ctx));
         for (let i = 0; i < collections.length; i++) {
             await this.processManyToMany(meta, where[i], collections[i], false, options);
@@ -438,30 +544,53 @@ class AbstractSqlDriver extends core_1.DatabaseDriver {
             deleteDiff.push(...snapshot);
             insertDiff.push(...current);
         }
-        if (coll.property.reference === core_1.ReferenceType.ONE_TO_MANY) {
+        if (coll.property.kind === core_1.ReferenceKind.ONE_TO_MANY) {
             const cols = coll.property.referencedColumnNames;
             const qb = this.createQueryBuilder(coll.property.type, ctx, 'write')
-                .withSchema(this.getSchemaName(meta, options))
-                .update({ [coll.property.mappedBy]: pks })
+                .withSchema(this.getSchemaName(meta, options));
+            if (coll.getSnapshot() === undefined) {
+                if (coll.property.orphanRemoval) {
+                    const kqb = qb.delete({ [coll.property.mappedBy]: pks })
+                        .getKnexQuery()
+                        .whereNotIn(cols, insertDiff);
+                    return this.rethrow(this.execute(kqb));
+                }
+                const kqb = qb.update({ [coll.property.mappedBy]: null })
+                    .getKnexQuery()
+                    .whereNotIn(cols, insertDiff);
+                return this.rethrow(this.execute(kqb));
+            }
+            const kqb = qb.update({ [coll.property.mappedBy]: pks })
                 .getKnexQuery()
                 .whereIn(cols, insertDiff);
-            return this.rethrow(this.execute(qb));
+            return this.rethrow(this.execute(kqb));
         }
         /* istanbul ignore next */
         const ownerSchema = wrapped.getSchema() === '*' ? this.config.get('schema') : wrapped.getSchema();
         const pivotMeta = this.metadata.find(coll.property.pivotEntity);
         if (pivotMeta.schema === '*') {
             /* istanbul ignore next */
-            options ?? (options = {});
+            options ??= {};
             options.schema = ownerSchema;
         }
         return this.rethrow(this.updateCollectionDiff(meta, coll.property, pks, deleteDiff, insertDiff, options));
     }
     async loadFromPivotTable(prop, owners, where = {}, orderBy, ctx, options) {
         const pivotProp2 = this.getPivotInverseProperty(prop);
+        const prop2 = prop.targetMeta.properties[prop.mappedBy ?? prop.inversedBy];
         const ownerMeta = this.metadata.find(pivotProp2.type);
         const pivotMeta = this.metadata.find(prop.pivotEntity);
-        const cond = { [`${prop.pivotEntity}.${pivotProp2.name}`]: { $in: ownerMeta.compositePK ? owners : owners.map(o => o[0]) } };
+        options = { ...options };
+        const qb = this.createQueryBuilder(prop.type, ctx, options.connectionType)
+            .withSchema(this.getSchemaName(prop.targetMeta, options))
+            .indexHint(options.indexHint)
+            .comment(options.comments)
+            .hintComment(options.hintComments);
+        const pivotAlias = qb.getNextAlias(pivotMeta.tableName);
+        const pivotKey = pivotProp2.joinColumns.map(column => `${pivotAlias}.${column}`).join(core_1.Utils.PK_SEPARATOR);
+        const cond = {
+            [pivotKey]: { $in: ownerMeta.compositePK ? owners : owners.map(o => o[0]) },
+        };
         /* istanbul ignore if */
         if (!core_1.Utils.isEmpty(where) && Object.keys(where).every(k => core_1.Utils.isOperator(k, false))) {
             where = cond;
@@ -469,22 +598,32 @@ class AbstractSqlDriver extends core_1.DatabaseDriver {
         else {
             where = { ...where, ...cond };
         }
-        orderBy = this.getPivotOrderBy(prop, orderBy);
-        const qb = this.createQueryBuilder(prop.type, ctx, options?.connectionType)
-            .unsetFlag(core_1.QueryFlag.CONVERT_CUSTOM_TYPES)
-            .withSchema(this.getSchemaName(prop.targetMeta, options));
-        const populate = this.autoJoinOneToOneOwner(prop.targetMeta, [{ field: prop.pivotEntity }]);
-        const fields = this.buildFields(prop.targetMeta, (options?.populate ?? []), [], qb, options?.fields);
-        qb.select(fields).populate(populate).where(where).orderBy(orderBy).setLockMode(options?.lockMode, options?.lockTableAliases);
-        if (owners.length === 1 && (options?.offset != null || options?.limit != null)) {
+        orderBy = this.getPivotOrderBy(prop, pivotAlias, orderBy);
+        const populate = this.autoJoinOneToOneOwner(prop.targetMeta, []);
+        const fields = this.buildFields(prop.targetMeta, (options.populate ?? []), [], qb, options.fields);
+        const k1 = !prop.owner ? 'joinColumns' : 'inverseJoinColumns';
+        const k2 = prop.owner ? 'joinColumns' : 'inverseJoinColumns';
+        const cols = [
+            ...prop[k1].map(col => `${pivotAlias}.${col} as fk__${col}`),
+            ...prop[k2].map(col => `${pivotAlias}.${col} as fk__${col}`),
+        ];
+        fields.push(...cols);
+        qb.select(fields)
+            .join(prop2.name, pivotAlias, {}, 'pivotJoin', undefined, options.schema)
+            .populate(populate)
+            .where(where)
+            .orderBy(orderBy)
+            .setLockMode(options.lockMode, options.lockTableAliases);
+        if (owners.length === 1 && (options.offset != null || options.limit != null)) {
             qb.limit(options.limit, options.offset);
         }
-        if (prop.targetMeta.schema !== '*' && pivotMeta.schema === '*' && options?.schema) {
+        const schema = this.config.get('schema');
+        if (prop.targetMeta.schema !== '*' && pivotMeta.schema === '*' && options.schema) {
             // eslint-disable-next-line dot-notation
             qb['finalize']();
             // eslint-disable-next-line dot-notation
             Object.values(qb['_joins']).forEach(join => {
-                join.schema = options.schema;
+                join.schema = schema;
             });
         }
         const items = owners.length ? await this.rethrow(qb.execute('all')) : [];
@@ -508,6 +647,18 @@ class AbstractSqlDriver extends core_1.DatabaseDriver {
         });
         return map;
     }
+    getPivotOrderBy(prop, pivotAlias, orderBy) {
+        if (!core_1.Utils.isEmpty(orderBy)) {
+            return orderBy;
+        }
+        if (!core_1.Utils.isEmpty(prop.orderBy)) {
+            return core_1.Utils.asArray(prop.orderBy);
+        }
+        if (prop.fixedOrder) {
+            return [{ [`${pivotAlias}.${prop.fixedOrderColumn}`]: core_1.QueryOrder.ASC }];
+        }
+        return [];
+    }
     async execute(queryOrKnex, params = [], method = 'all', ctx) {
         return this.rethrow(this.connection.execute(queryOrKnex, params, method, ctx));
     }
@@ -520,7 +671,7 @@ class AbstractSqlDriver extends core_1.DatabaseDriver {
         }
         const relationsToPopulate = populate.map(({ field }) => field);
         const toPopulate = meta.relations
-            .filter(prop => prop.reference === core_1.ReferenceType.ONE_TO_ONE && !prop.owner && !relationsToPopulate.includes(prop.name))
+            .filter(prop => prop.kind === core_1.ReferenceKind.ONE_TO_ONE && !prop.owner && !relationsToPopulate.includes(prop.name))
             .filter(prop => fields.length === 0 || fields.some(f => prop.name === f || prop.name.startsWith(`${String(f)}.`)))
             .map(prop => ({ field: prop.name, strategy: prop.strategy }));
         return [...populate, ...toPopulate];
@@ -528,23 +679,30 @@ class AbstractSqlDriver extends core_1.DatabaseDriver {
     joinedProps(meta, populate) {
         return populate.filter(p => {
             const prop = meta.properties[p.field] || {};
-            return (p.strategy || prop.strategy || this.config.get('loadStrategy')) === core_1.LoadStrategy.JOINED && prop.reference !== core_1.ReferenceType.SCALAR;
+            return (p.strategy || prop.strategy || this.config.get('loadStrategy')) === core_1.LoadStrategy.JOINED && prop.kind !== core_1.ReferenceKind.SCALAR;
         });
     }
     /**
      * @internal
      */
-    mergeJoinedResult(rawResults, meta) {
+    mergeJoinedResult(rawResults, meta, joinedProps) {
         // group by the root entity primary key first
         const map = {};
         const res = [];
         rawResults.forEach(item => {
             const pk = core_1.Utils.getCompositeKeyHash(item, meta);
             if (map[pk]) {
-                map[pk].push(item);
+                joinedProps.forEach(hint => {
+                    // Sometimes we might join a M:N relation with additional filter on the target entity, and as a result, we get
+                    // the first result with `null` for all target values, which is mapped as empty array. When we see that happen,
+                    // we need to merge the results of the next item.
+                    if (Array.isArray(map[pk][hint.field]) && Array.isArray(item[hint.field]) && map[pk][hint.field].length === 0) {
+                        item[hint.field].forEach((el) => map[pk][hint.field].push(el));
+                    }
+                });
             }
             else {
-                map[pk] = [item];
+                map[pk] = item;
                 res.push(item);
             }
         });
@@ -553,14 +711,17 @@ class AbstractSqlDriver extends core_1.DatabaseDriver {
     getFieldsForJoinedLoad(qb, meta, explicitFields, populate = [], parentTableAlias, parentJoinPath) {
         const fields = [];
         const joinedProps = this.joinedProps(meta, populate);
+        if (explicitFields?.includes('*')) {
+            fields.push('*');
+        }
         const shouldHaveColumn = (prop, populate, fields) => {
             if (!this.platform.shouldHaveColumn(prop, populate)) {
                 return false;
             }
             if (!fields || prop.primary) {
-                return true;
+                return !fields?.includes('*');
             }
-            return fields.includes(prop.name);
+            return fields.some(f => f === prop.name || f.toString().startsWith(prop.name + '.'));
         };
         // alias all fields in the primary table
         meta.props
@@ -587,24 +748,25 @@ class AbstractSqlDriver extends core_1.DatabaseDriver {
      * @internal
      */
     mapPropToFieldNames(qb, prop, tableAlias) {
-        const aliased = qb.ref(tableAlias ? `${tableAlias}__${prop.fieldNames[0]}` : prop.fieldNames[0]).toString();
+        const knex = this.connection.getKnex();
+        const aliased = knex.ref(tableAlias ? `${tableAlias}__${prop.fieldNames[0]}` : prop.fieldNames[0]).toString();
         if (prop.customType?.convertToJSValueSQL && tableAlias) {
-            const prefixed = qb.ref(prop.fieldNames[0]).withSchema(tableAlias).toString();
-            return [qb.raw(prop.customType.convertToJSValueSQL(prefixed, this.platform) + ' as ' + aliased).toString()];
+            const prefixed = knex.ref(prop.fieldNames[0]).withSchema(tableAlias).toString();
+            return [(0, core_1.raw)(`${prop.customType.convertToJSValueSQL(prefixed, this.platform)} as ${aliased}`)];
         }
         if (prop.formula) {
-            const alias = qb.ref(tableAlias ?? qb.alias).toString();
-            return [`${prop.formula(alias)} as ${aliased}`];
+            const alias = knex.ref(tableAlias ?? qb.alias).toString();
+            return [(0, core_1.raw)(`${prop.formula(alias)} as ${aliased}`)];
         }
         if (tableAlias) {
-            return prop.fieldNames.map(fieldName => qb.ref(fieldName).withSchema(tableAlias).as(`${tableAlias}__${fieldName}`));
+            return prop.fieldNames.map(fieldName => knex.ref(fieldName).withSchema(tableAlias).as(`${tableAlias}__${fieldName}`));
         }
         return prop.fieldNames;
     }
     /** @internal */
-    createQueryBuilder(entityName, ctx, preferredConnectionType, convertCustomTypes) {
+    createQueryBuilder(entityName, ctx, preferredConnectionType, convertCustomTypes, logging) {
         const connectionType = this.resolveConnectionType({ ctx, connectionType: preferredConnectionType });
-        const qb = new query_1.QueryBuilder(entityName, this.metadata, this, ctx, undefined, connectionType);
+        const qb = new query_1.QueryBuilder(entityName, this.metadata, this, ctx, undefined, connectionType, undefined, logging);
         if (!convertCustomTypes) {
             qb.unsetFlag(core_1.QueryFlag.CONVERT_CUSTOM_TYPES);
         }
@@ -628,7 +790,7 @@ class AbstractSqlDriver extends core_1.DatabaseDriver {
         }
         const ret = {};
         this.metadata.find(entityName).relations.forEach(prop => {
-            if (prop.reference === core_1.ReferenceType.MANY_TO_MANY && data[prop.name]) {
+            if (prop.kind === core_1.ReferenceKind.MANY_TO_MANY && data[prop.name]) {
                 ret[prop.name] = data[prop.name].map((item) => core_1.Utils.asArray(item));
                 delete data[prop.name];
             }
@@ -649,10 +811,9 @@ class AbstractSqlDriver extends core_1.DatabaseDriver {
         if (!deleteDiff) {
             deleteDiff = [];
         }
+        const pivotMeta = this.metadata.find(prop.pivotEntity);
         if (deleteDiff === true || deleteDiff.length > 0) {
-            const qb1 = this.createQueryBuilder(prop.pivotEntity, options?.ctx, 'write')
-                .withSchema(this.getSchemaName(meta, options))
-                .unsetFlag(core_1.QueryFlag.CONVERT_CUSTOM_TYPES);
+            const qb1 = this.createQueryBuilder(prop.pivotEntity, options?.ctx, 'write').withSchema(this.getSchemaName(pivotMeta, options));
             const knex = qb1.getKnex();
             if (Array.isArray(deleteDiff)) {
                 knex.whereIn(prop.inverseJoinColumns, deleteDiff);
@@ -671,13 +832,16 @@ class AbstractSqlDriver extends core_1.DatabaseDriver {
         });
         /* istanbul ignore else */
         if (this.platform.allowsMultiInsert()) {
-            await this.nativeInsertMany(prop.pivotEntity, items, { ...options, convertCustomTypes: false, processCollections: false });
+            await this.nativeInsertMany(prop.pivotEntity, items, {
+                ...options,
+                convertCustomTypes: false,
+                processCollections: false,
+            });
         }
         else {
             await core_1.Utils.runSerial(items, item => {
                 return this.createQueryBuilder(prop.pivotEntity, options?.ctx, 'write')
-                    .unsetFlag(core_1.QueryFlag.CONVERT_CUSTOM_TYPES)
-                    .withSchema(this.getSchemaName(meta, options))
+                    .withSchema(this.getSchemaName(pivotMeta, options))
                     .insert(item)
                     .execute('run', false);
             });
@@ -685,9 +849,9 @@ class AbstractSqlDriver extends core_1.DatabaseDriver {
     }
     async lockPessimistic(entity, options) {
         const meta = (0, core_1.helper)(entity).__meta;
-        const qb = this.createQueryBuilder(entity.constructor.name, options.ctx).unsetFlag(core_1.QueryFlag.CONVERT_CUSTOM_TYPES).withSchema(options.schema ?? meta.schema);
+        const qb = this.createQueryBuilder(entity.constructor.name, options.ctx).withSchema(options.schema ?? meta.schema);
         const cond = core_1.Utils.getPrimaryKeyCond(entity, meta.primaryKeys);
-        qb.select('1').where(cond).setLockMode(options.lockMode, options.lockTableAliases);
+        qb.select((0, core_1.raw)('1')).where(cond).setLockMode(options.lockMode, options.lockTableAliases);
         await this.rethrow(qb.execute());
     }
     buildJoinedPropsOrderBy(entityName, qb, meta, populate, parentPath) {
@@ -699,7 +863,7 @@ class AbstractSqlDriver extends core_1.DatabaseDriver {
             const path = `${parentPath ? parentPath : entityName}.${relation.field}`;
             const propAlias = qb.getAliasForJoinPath(path);
             if (propOrderBy) {
-                Object.keys(propOrderBy).forEach(field => {
+                core_1.Utils.keys(propOrderBy).forEach(field => {
                     orderBy.push({ [`${propAlias}.${field}`]: propOrderBy[field] });
                 });
             }
@@ -726,12 +890,12 @@ class AbstractSqlDriver extends core_1.DatabaseDriver {
         return ret;
     }
     processField(meta, prop, field, ret, populate, joinedProps, qb) {
-        if (!prop || (prop.reference === core_1.ReferenceType.ONE_TO_ONE && !prop.owner)) {
+        if (!prop || (prop.kind === core_1.ReferenceKind.ONE_TO_ONE && !prop.owner)) {
             return;
         }
-        if (prop.reference === core_1.ReferenceType.EMBEDDED) {
+        if (prop.kind === core_1.ReferenceKind.EMBEDDED) {
             if (prop.object) {
-                ret.push(prop.fieldNames[0]);
+                ret.push(prop.name);
                 return;
             }
             const parts = field.split('.');
@@ -743,9 +907,7 @@ class AbstractSqlDriver extends core_1.DatabaseDriver {
             }
             return;
         }
-        if (prop.fieldNames) {
-            ret.push(prop.fieldNames[0]);
-        }
+        ret.push(prop.name);
     }
     buildFields(meta, populate, joinedProps, qb, fields) {
         const lazyProps = meta.props.filter(prop => prop.lazy && !populate.some(p => p.field === prop.name || p.all));
@@ -753,6 +915,7 @@ class AbstractSqlDriver extends core_1.DatabaseDriver {
         const requiresSQLConversion = meta.props.some(p => p.customType?.convertToJSValueSQL);
         const hasExplicitFields = !!fields;
         const ret = [];
+        let addFormulas = false;
         if (joinedProps.length > 0) {
             ret.push(...this.getFieldsForJoinedLoad(qb, meta, fields, populate));
         }
@@ -778,20 +941,22 @@ class AbstractSqlDriver extends core_1.DatabaseDriver {
         else if (lazyProps.filter(p => !p.formula).length > 0) {
             const props = meta.props.filter(prop => this.platform.shouldHaveColumn(prop, populate, false));
             ret.push(...core_1.Utils.flatten(props.filter(p => !lazyProps.includes(p)).map(p => p.fieldNames)));
+            addFormulas = true;
         }
         else if (hasLazyFormulas || requiresSQLConversion) {
             ret.push('*');
+            addFormulas = true;
         }
-        if (ret.length > 0 && !hasExplicitFields) {
+        if (ret.length > 0 && !hasExplicitFields && addFormulas) {
             meta.props
                 .filter(prop => prop.formula && !lazyProps.includes(prop))
                 .forEach(prop => {
-                const alias = qb.ref(qb.alias).toString();
-                const aliased = qb.ref(prop.fieldNames[0]).toString();
-                ret.push(`${prop.formula(alias)} as ${aliased}`);
+                const alias = this.connection.getKnex().ref(qb.alias).toString();
+                const aliased = this.connection.getKnex().ref(prop.fieldNames[0]).toString();
+                ret.push((0, core_1.raw)(`${prop.formula(alias)} as ${aliased}`));
             });
             meta.props
-                .filter(prop => prop.customType?.convertToDatabaseValueSQL || prop.customType?.convertToJSValueSQL)
+                .filter(prop => prop.hasConvertToDatabaseValueSQL || prop.hasConvertToJSValueSQL)
                 .forEach(prop => ret.push(prop.name));
         }
         return ret.length > 0 ? core_1.Utils.unique(ret) : ['*'];