@mikro-orm/knex 6.0.0-dev.26 → 6.0.0-dev.261

package/AbstractSqlDriver.js

@@ -4,10 +4,14 @@ exports.AbstractSqlDriver = void 0;
 const core_1 = require("@mikro-orm/core");
 const query_1 = require("./query");
 const SqlEntityManager_1 = require("./SqlEntityManager");
+const PivotCollectionPersister_1 = require("./PivotCollectionPersister");
 class AbstractSqlDriver extends core_1.DatabaseDriver {
+    [core_1.EntityManagerType];
+    connection;
+    replicas = [];
+    platform;
     constructor(config, platform, connection, connector) {
         super(config, connector);
-        this.replicas = [];
         this.connection = new connection(this.config);
         this.replicas = this.createReplicas(conf => new connection(this.config, conf, 'read'));
         this.platform = platform;
@@ -25,21 +29,26 @@ class AbstractSqlDriver extends core_1.DatabaseDriver {
             return this.findVirtual(entityName, where, options);
         }
         const populate = this.autoJoinOneToOneOwner(meta, options.populate, options.fields);
-        const joinedProps = this.joinedProps(meta, populate);
-        const qb = this.createQueryBuilder(entityName, options.ctx, options.connectionType, false);
-        const fields = this.buildFields(meta, populate, joinedProps, qb, options.fields);
-        const joinedPropsOrderBy = this.buildJoinedPropsOrderBy(entityName, qb, meta, joinedProps);
-        const orderBy = [...core_1.Utils.asArray(options.orderBy), ...joinedPropsOrderBy];
+        const joinedProps = this.joinedProps(meta, populate, options);
+        const qb = this.createQueryBuilder(entityName, options.ctx, options.connectionType, false, options.logging);
+        const fields = this.buildFields(meta, populate, joinedProps, qb, qb.alias, options.fields, options);
+        const orderBy = this.buildOrderBy(qb, meta, populate, options);
+        core_1.Utils.asArray(options.flags).forEach(flag => qb.setFlag(flag));
         if (core_1.Utils.isPrimaryKey(where, meta.compositePK)) {
             where = { [core_1.Utils.getPrimaryKeyHash(meta.primaryKeys)]: where };
         }
         const { first, last, before, after } = options;
         const isCursorPagination = [first, last, before, after].some(v => v != null);
+        qb.__populateWhere = options._populateWhere;
         qb.select(fields)
+            // only add populateWhere if we are populate-joining, as this will be used to add `on` conditions
             .populate(populate, joinedProps.length > 0 ? options.populateWhere : undefined)
             .where(where)
             .groupBy(options.groupBy)
             .having(options.having)
+            .indexHint(options.indexHint)
+            .comment(options.comments)
+            .hintComment(options.hintComments)
             .withSchema(this.getSchemaName(meta, options));
         if (isCursorPagination) {
             const { orderBy: newOrderBy, where } = this.processCursorOptions(meta, options, orderBy);
@@ -48,83 +57,92 @@ class AbstractSqlDriver extends core_1.DatabaseDriver {
         else {
             qb.orderBy(orderBy);
         }
-        if (options.limit !== undefined) {
+        if (options.limit != null || options.offset != null) {
             qb.limit(options.limit, options.offset);
         }
         if (options.lockMode) {
             qb.setLockMode(options.lockMode, options.lockTableAliases);
         }
-        core_1.Utils.asArray(options.flags).forEach(flag => qb.setFlag(flag));
         const result = await this.rethrow(qb.execute('all'));
         if (isCursorPagination && !first && !!last) {
             result.reverse();
         }
-        if (joinedProps.length > 0) {
-            return this.mergeJoinedResult(result, meta);
-        }
         return result;
     }
     async findOne(entityName, where, options) {
         const opts = { populate: [], ...(options || {}) };
         const meta = this.metadata.find(entityName);
         const populate = this.autoJoinOneToOneOwner(meta, opts.populate, opts.fields);
-        const joinedProps = this.joinedProps(meta, populate);
-        if (joinedProps.length === 0) {
+        const joinedProps = this.joinedProps(meta, populate, options);
+        const hasToManyJoins = joinedProps.some(hint => this.hasToManyJoins(hint, meta));
+        if (joinedProps.length === 0 || !hasToManyJoins) {
             opts.limit = 1;
         }
         if (opts.limit > 0 && !opts.flags?.includes(core_1.QueryFlag.DISABLE_PAGINATE)) {
-            opts.flags ?? (opts.flags = []);
+            opts.flags ??= [];
             opts.flags.push(core_1.QueryFlag.DISABLE_PAGINATE);
         }
         const res = await this.find(entityName, where, opts);
         return res[0] || null;
     }
-    async findVirtual(entityName, where, options) {
-        const meta = this.metadata.get(entityName);
-        /* istanbul ignore next */
-        if (!meta.expression) {
-            return [];
+    hasToManyJoins(hint, meta) {
+        const [propName] = hint.field.split(':', 2);
+        const prop = meta.properties[propName];
+        if (prop && [core_1.ReferenceKind.ONE_TO_MANY, core_1.ReferenceKind.MANY_TO_MANY].includes(prop.kind)) {
+            return true;
         }
-        if (typeof meta.expression === 'string') {
-            return this.wrapVirtualExpressionInSubquery(meta, meta.expression, where, options);
+        if (hint.children && prop.targetMeta) {
+            return hint.children.some(hint => this.hasToManyJoins(hint, prop.targetMeta));
         }
-        const em = this.createEntityManager(false);
-        em.setTransactionContext(options.ctx);
-        const res = meta.expression(em, where, options);
-        if (res instanceof (query_1.QueryBuilder)) {
-            return this.wrapVirtualExpressionInSubquery(meta, res.getFormattedQuery(), where, options);
-        }
-        /* istanbul ignore next */
-        return res;
+        return false;
+    }
+    async findVirtual(entityName, where, options) {
+        return this.findFromVirtual(entityName, where, options, query_1.QueryType.SELECT);
     }
     async countVirtual(entityName, where, options) {
+        return this.findFromVirtual(entityName, where, options, query_1.QueryType.COUNT);
+    }
+    async findFromVirtual(entityName, where, options, type) {
         const meta = this.metadata.get(entityName);
         /* istanbul ignore next */
         if (!meta.expression) {
-            return 0;
+            return type === query_1.QueryType.SELECT ? [] : 0;
         }
         if (typeof meta.expression === 'string') {
-            return this.wrapVirtualExpressionInSubquery(meta, meta.expression, where, options, query_1.QueryType.COUNT);
+            return this.wrapVirtualExpressionInSubquery(meta, meta.expression, where, options, type);
         }
-        const em = this.createEntityManager(false);
+        const em = this.createEntityManager();
         em.setTransactionContext(options.ctx);
         const res = meta.expression(em, where, options);
-        if (res instanceof (query_1.QueryBuilder)) {
-            return this.wrapVirtualExpressionInSubquery(meta, res.getFormattedQuery(), where, options, query_1.QueryType.COUNT);
+        if (typeof res === 'string') {
+            return this.wrapVirtualExpressionInSubquery(meta, res, where, options, type);
+        }
+        if (res instanceof query_1.QueryBuilder) {
+            return this.wrapVirtualExpressionInSubquery(meta, res.getFormattedQuery(), where, options, type);
+        }
+        if (core_1.Utils.isObject(res)) {
+            const { sql, bindings } = res.toSQL();
+            const query = this.platform.formatQuery(sql, bindings);
+            return this.wrapVirtualExpressionInSubquery(meta, query, where, options, type);
         }
         /* istanbul ignore next */
         return res;
     }
-    async wrapVirtualExpressionInSubquery(meta, expression, where, options, type = query_1.QueryType.SELECT) {
+    async wrapVirtualExpressionInSubquery(meta, expression, where, options, type) {
         const qb = this.createQueryBuilder(meta.className, options?.ctx, options.connectionType, options.convertCustomTypes)
-            .limit(options?.limit, options?.offset);
-        if (options.orderBy) {
-            qb.orderBy(options.orderBy);
+            .indexHint(options.indexHint)
+            .comment(options.comments)
+            .hintComment(options.hintComments);
+        if (type !== query_1.QueryType.COUNT) {
+            qb.limit(options?.limit, options?.offset);
+            if (options.orderBy) {
+                qb.orderBy(options.orderBy);
+            }
         }
         qb.where(where);
         const kqb = qb.getKnexQuery().clear('select');
         if (type === query_1.QueryType.COUNT) {
-            kqb.select(qb.raw('count(*) as count'));
+            kqb.select(this.connection.getKnex().raw('count(*) as count'));
         }
         else { // select
             kqb.select('*');
@@ -143,40 +161,88 @@ class AbstractSqlDriver extends core_1.DatabaseDriver {
             return null;
         }
         if (qb) {
+            // here we map the aliased results (cartesian product) to an object graph
             this.mapJoinedProps(ret, meta, populate, qb, ret, map);
         }
         return ret;
     }
     mapJoinedProps(result, meta, populate, qb, root, map, parentJoinPath) {
         const joinedProps = this.joinedProps(meta, populate);
-        joinedProps.forEach(p => {
-            const relation = meta.properties[p.field];
+        joinedProps.forEach(hint => {
+            const [propName, ref] = hint.field.split(':', 2);
+            const prop = meta.properties[propName];
             /* istanbul ignore next */
-            if (!relation) {
+            if (!prop) {
+                return;
+            }
+            const pivotRefJoin = prop.kind === core_1.ReferenceKind.MANY_TO_MANY && ref;
+            const meta2 = this.metadata.find(prop.type);
+            let path = parentJoinPath ? `${parentJoinPath}.${prop.name}` : `${meta.name}.${prop.name}`;
+            if (!parentJoinPath) {
+                path = '[populate]' + path;
+            }
+            if (pivotRefJoin) {
+                path += '[pivot]';
+            }
+            const relationAlias = qb.getAliasForJoinPath(path, { matchPopulateJoins: true });
+            // pivot ref joins via joined strategy need to be handled separately here, as they dont join the target entity
+            if (pivotRefJoin) {
+                let item;
+                if (prop.inverseJoinColumns.length > 1) { // composite keys
+                    item = prop.inverseJoinColumns.map(name => root[`${relationAlias}__${name}`]);
+                }
+                else {
+                    const alias = `${relationAlias}__${prop.inverseJoinColumns[0]}`;
+                    item = root[alias];
+                }
+                prop.joinColumns.forEach(name => delete root[`${relationAlias}__${name}`]);
+                prop.inverseJoinColumns.forEach(name => delete root[`${relationAlias}__${name}`]);
+                result[prop.name] ??= [];
+                if (item) {
+                    result[prop.name].push(item);
+                }
                 return;
             }
-            const meta2 = this.metadata.find(relation.type);
-            const path = parentJoinPath ? `${parentJoinPath}.${relation.name}` : `${meta.name}.${relation.name}`;
-            const relationAlias = qb.getAliasForJoinPath(path);
-            const relationPojo = {};
             // If the primary key value for the relation is null, we know we haven't joined to anything
             // and therefore we don't return any record (since all values would be null)
             const hasPK = meta2.primaryKeys.every(pk => meta2.properties[pk].fieldNames.every(name => {
                 return root[`${relationAlias}__${name}`] != null;
             }));
             if (!hasPK) {
-                // initialize empty collections
-                if ([core_1.ReferenceType.MANY_TO_MANY, core_1.ReferenceType.ONE_TO_MANY].includes(relation.reference)) {
-                    result[relation.name] = result[relation.name] || [];
+                if ([core_1.ReferenceKind.MANY_TO_MANY, core_1.ReferenceKind.ONE_TO_MANY].includes(prop.kind)) {
+                    result[prop.name] ??= [];
+                }
+                if ([core_1.ReferenceKind.MANY_TO_ONE, core_1.ReferenceKind.ONE_TO_ONE].includes(prop.kind)) {
+                    result[prop.name] ??= null;
                 }
                 return;
             }
+            const relationPojo = {};
             meta2.props
-                .filter(prop => this.platform.shouldHaveColumn(prop, p.children || []))
+                .filter(prop => prop.persist === false && prop.fieldNames)
+                .filter(prop => !prop.lazy || populate.some(p => p.field === prop.name || p.all))
                 .forEach(prop => {
+                /* istanbul ignore if */
                 if (prop.fieldNames.length > 1) { // composite keys
                     relationPojo[prop.name] = prop.fieldNames.map(name => root[`${relationAlias}__${name}`]);
+                }
+                else {
+                    const alias = `${relationAlias}__${prop.fieldNames[0]}`;
+                    relationPojo[prop.name] = root[alias];
+                }
+            });
+            meta2.props
+                .filter(prop => this.platform.shouldHaveColumn(prop, hint.children || []))
+                .forEach(prop => {
+                if (prop.fieldNames.length > 1) { // composite keys
+                    const fk = prop.fieldNames.map(name => root[`${relationAlias}__${name}`]);
                     prop.fieldNames.map(name => delete root[`${relationAlias}__${name}`]);
+                    relationPojo[prop.name] = core_1.Utils.mapFlatCompositePrimaryKey(fk, prop);
+                }
+                else if (prop.runtimeType === 'Date') {
+                    const alias = `${relationAlias}__${prop.fieldNames[0]}`;
+                    relationPojo[prop.name] = (typeof root[alias] === 'string' ? new Date(root[alias]) : root[alias]);
+                    delete root[alias];
                 }
                 else {
                     const alias = `${relationAlias}__${prop.fieldNames[0]}`;
@@ -184,41 +250,26 @@ class AbstractSqlDriver extends core_1.DatabaseDriver {
                     delete root[alias];
                 }
             });
-            const key = `${meta.name}-${(core_1.Utils.getCompositeKeyHash(result, meta))}`;
-            if (map[key]) {
-                result[relation.name] = map[key][relation.name];
-            }
-            else {
-                map[key] = result;
-            }
-            if ([core_1.ReferenceType.MANY_TO_MANY, core_1.ReferenceType.ONE_TO_MANY].includes(relation.reference)) {
-                result[relation.name] = result[relation.name] || [];
-                this.appendToCollection(meta2, result[relation.name], relationPojo);
+            if ([core_1.ReferenceKind.MANY_TO_MANY, core_1.ReferenceKind.ONE_TO_MANY].includes(prop.kind)) {
+                result[prop.name] ??= [];
+                result[prop.name].push(relationPojo);
             }
             else {
-                result[relation.name] = relationPojo;
+                result[prop.name] = relationPojo;
             }
-            const populateChildren = p.children || [];
+            const populateChildren = hint.children || [];
             this.mapJoinedProps(relationPojo, meta2, populateChildren, qb, root, map, path);
         });
     }
-    appendToCollection(meta, collection, relationPojo) {
-        if (collection.length === 0) {
-            return void collection.push(relationPojo);
-        }
-        const last = collection[collection.length - 1];
-        const pk1 = core_1.Utils.getCompositeKeyHash(last, meta);
-        const pk2 = core_1.Utils.getCompositeKeyHash(relationPojo, meta);
-        if (pk1 !== pk2) {
-            collection.push(relationPojo);
-        }
-    }
     async count(entityName, where, options = {}) {
         const meta = this.metadata.find(entityName);
         if (meta?.virtual) {
             return this.countVirtual(entityName, where, options);
         }
         const qb = this.createQueryBuilder(entityName, options.ctx, options.connectionType, false)
+            .indexHint(options.indexHint)
+            .comment(options.comments)
+            .hintComment(options.hintComments)
             .groupBy(options.groupBy)
             .having(options.having)
             .populate(options.populate ?? [])
@@ -227,7 +278,7 @@ class AbstractSqlDriver extends core_1.DatabaseDriver {
         return this.rethrow(qb.getCount());
     }
     async nativeInsert(entityName, data, options = {}) {
-        options.convertCustomTypes ?? (options.convertCustomTypes = true);
+        options.convertCustomTypes ??= true;
         const meta = this.metadata.find(entityName);
         const collections = this.extractManyToMany(entityName, data);
         const pks = meta?.primaryKeys ?? [this.config.getNamingStrategy().referenceColumnName()];
@@ -247,16 +298,20 @@ class AbstractSqlDriver extends core_1.DatabaseDriver {
         return res;
     }
     async nativeInsertMany(entityName, data, options = {}) {
-        options.processCollections ?? (options.processCollections = true);
-        options.convertCustomTypes ?? (options.convertCustomTypes = true);
-        const meta = this.metadata.find(entityName);
+        options.processCollections ??= true;
+        options.convertCustomTypes ??= true;
+        const meta = this.metadata.find(entityName)?.root;
         const collections = options.processCollections ? data.map(d => this.extractManyToMany(entityName, d)) : [];
         const pks = this.getPrimaryKeyFields(entityName);
         const set = new Set();
-        data.forEach(row => Object.keys(row).forEach(k => set.add(k)));
+        data.forEach(row => core_1.Utils.keys(row).forEach(k => set.add(k)));
         const props = [...set].map(name => meta?.properties[name] ?? { name, fieldNames: [name] });
-        const fields = core_1.Utils.flatten(props.map(prop => prop.fieldNames));
+        let fields = core_1.Utils.flatten(props.map(prop => prop.fieldNames));
+        const duplicates = core_1.Utils.findDuplicates(fields);
         const params = [];
+        if (duplicates.length) {
+            fields = core_1.Utils.unique(fields);
+        }
         /* istanbul ignore next */
         const tableName = meta ? this.getTableName(meta, options) : this.platform.quoteIdentifier(entityName);
         let sql = `insert into ${tableName} `;
@@ -267,29 +322,62 @@ class AbstractSqlDriver extends core_1.DatabaseDriver {
         else {
             sql += ' ' + data.map(() => `select null as ${this.platform.quoteIdentifier(pks[0])}`).join(' union all ');
         }
+        const addParams = (prop, row) => {
+            let value = row[prop.name];
+            if (prop.kind === core_1.ReferenceKind.EMBEDDED && prop.object) {
+                if (prop.array) {
+                    for (let i = 0; i < value.length; i++) {
+                        const item = value[i];
+                        value[i] = this.mapDataToFieldNames(item, false, prop.embeddedProps, options.convertCustomTypes);
+                    }
+                }
+                else {
+                    value = this.mapDataToFieldNames(value, false, prop.embeddedProps, options.convertCustomTypes);
+                }
+            }
+            if (options.convertCustomTypes && prop.customType) {
+                params.push(prop.customType.convertToDatabaseValue(value, this.platform, { key: prop.name, mode: 'query-data' }));
+                return;
+            }
+            params.push(value);
+        };
         if (fields.length > 0 || this.platform.usesDefaultKeyword()) {
             sql += data.map(row => {
                 const keys = [];
+                const usedDups = [];
                 props.forEach(prop => {
                     if (prop.fieldNames.length > 1) {
-                        params.push(...(row[prop.name] ?? prop.fieldNames.map(() => null)));
-                        keys.push(...prop.fieldNames.map(_ => '?'));
-                    }
-                    else if (prop.customType && 'convertToDatabaseValueSQL' in prop.customType && !this.platform.isRaw(row[prop.name])) {
-                        keys.push(prop.customType.convertToDatabaseValueSQL('?', this.platform));
-                        params.push(row[prop.name]);
+                        const param = core_1.Utils.flatten([...row[prop.name] ?? prop.fieldNames.map(() => null)]);
+                        const key = param.map(() => '?');
+                        prop.fieldNames.forEach((field, idx) => {
+                            if (!duplicates.includes(field) || !usedDups.includes(field)) {
+                                params.push(param[idx]);
+                                keys.push(key[idx]);
+                                usedDups.push(field);
+                            }
+                        });
                     }
                     else {
-                        params.push(row[prop.name]);
-                        keys.push('?');
+                        const field = prop.fieldNames[0];
+                        if (!duplicates.includes(field) || !usedDups.includes(field)) {
+                            if (prop.customType && 'convertToDatabaseValueSQL' in prop.customType && !this.platform.isRaw(row[prop.name])) {
+                                keys.push(prop.customType.convertToDatabaseValueSQL('?', this.platform));
+                            }
+                            else {
+                                keys.push('?');
+                            }
+                            addParams(prop, row);
+                            usedDups.push(field);
+                        }
                     }
                 });
                 return '(' + (keys.join(', ') || 'default') + ')';
             }).join(', ');
         }
-        if (this.platform.usesReturningStatement()) {
-            /* istanbul ignore next */
-            const returningProps = meta.hydrateProps.filter(prop => prop.persist !== false && (prop.primary || prop.defaultRaw));
+        if (meta && this.platform.usesReturningStatement()) {
+            const returningProps = meta.props
+                .filter(prop => prop.persist !== false && prop.defaultRaw || prop.autoincrement || prop.generated)
+                .filter(prop => !(prop.name in data[0]) || core_1.Utils.isRawSql(data[0][prop.name]));
             const returningFields = core_1.Utils.flatten(returningProps.map(prop => prop.fieldNames));
             /* istanbul ignore next */
             sql += returningFields.length > 0 ? ` returning ${returningFields.map(field => this.platform.quoteIdentifier(field)).join(', ')}` : '';
@@ -301,8 +389,8 @@ class AbstractSqlDriver extends core_1.DatabaseDriver {
             pk = data.map(d => core_1.Utils.getPrimaryKeyCond(d, pks));
         }
         else {
-            res.row ?? (res.row = {});
-            res.rows ?? (res.rows = []);
+            res.row ??= {};
+            res.rows ??= [];
             pk = data.map((d, i) => d[pks[0]] ?? res.rows[i]?.[pks[0]]).map(d => [d]);
             res.insertId = res.insertId || res.row[pks[0]];
         }
@@ -312,7 +400,7 @@ class AbstractSqlDriver extends core_1.DatabaseDriver {
         return res;
     }
     async nativeUpdate(entityName, where, data, options = {}) {
-        options.convertCustomTypes ?? (options.convertCustomTypes = true);
+        options.convertCustomTypes ??= true;
         const meta = this.metadata.find(entityName);
         const pks = this.getPrimaryKeyFields(entityName);
         const collections = this.extractManyToMany(entityName, data);
@@ -326,14 +414,27 @@ class AbstractSqlDriver extends core_1.DatabaseDriver {
                 .withSchema(this.getSchemaName(meta, options));
             if (options.upsert) {
                 /* istanbul ignore next */
-                const uniqueFields = core_1.Utils.isPlainObject(where) ? Object.keys(where) : meta.primaryKeys;
-                /* istanbul ignore next */
+                const uniqueFields = options.onConflictFields ?? (core_1.Utils.isPlainObject(where) ? core_1.Utils.keys(where) : meta.primaryKeys);
+                const returning = (0, core_1.getOnConflictReturningFields)(meta, data, uniqueFields, options);
                 qb.insert(data)
-                    .onConflict(uniqueFields.map(p => meta?.properties[p]?.fieldNames[0] ?? p))
-                    .merge(Object.keys(data).filter(f => !uniqueFields.includes(f)));
+                    .onConflict(uniqueFields)
+                    .returning(returning);
+                if (!options.onConflictAction || options.onConflictAction === 'merge') {
+                    const fields = (0, core_1.getOnConflictFields)(data, uniqueFields, options);
+                    qb.merge(fields);
+                }
+                if (options.onConflictAction === 'ignore') {
+                    qb.ignore();
+                }
             }
             else {
                 qb.update(data).where(where);
+                // reload generated columns and version fields
+                const returning = [];
+                meta?.props
+                    .filter(prop => (prop.generated && !prop.primary) || prop.version)
+                    .forEach(prop => returning.push(prop.name));
+                qb.returning(returning);
             }
             res = await this.rethrow(qb.execute('run', false));
         }
@@ -343,26 +444,65 @@ class AbstractSqlDriver extends core_1.DatabaseDriver {
         return res;
     }
     async nativeUpdateMany(entityName, where, data, options = {}) {
-        options.processCollections ?? (options.processCollections = true);
-        options.convertCustomTypes ?? (options.convertCustomTypes = true);
+        options.processCollections ??= true;
+        options.convertCustomTypes ??= true;
         const meta = this.metadata.get(entityName);
         if (options.upsert) {
-            const uniqueFields = core_1.Utils.isPlainObject(where[0]) ? Object.keys(where[0]) : meta.primaryKeys;
+            const uniqueFields = options.onConflictFields ?? (core_1.Utils.isPlainObject(where[0]) ? Object.keys(where[0]).flatMap(key => core_1.Utils.splitPrimaryKeys(key)) : meta.primaryKeys);
             const qb = this.createQueryBuilder(entityName, options.ctx, 'write', options.convertCustomTypes).withSchema(this.getSchemaName(meta, options));
+            const returning = (0, core_1.getOnConflictReturningFields)(meta, data[0], uniqueFields, options);
             qb.insert(data)
-                .onConflict(uniqueFields.map(p => meta?.properties[p]?.fieldNames[0] ?? p))
-                .merge(Object.keys(data[0]).filter(f => !uniqueFields.includes(f)));
-            return qb.execute('run', false);
+                .onConflict(uniqueFields)
+                .returning(returning);
+            if (!options.onConflictAction || options.onConflictAction === 'merge') {
+                const fields = (0, core_1.getOnConflictFields)(data[0], uniqueFields, options);
+                qb.merge(fields);
+            }
+            if (options.onConflictAction === 'ignore') {
+                qb.ignore();
+            }
+            return this.rethrow(qb.execute('run', false));
         }
         const collections = options.processCollections ? data.map(d => this.extractManyToMany(entityName, d)) : [];
         const keys = new Set();
-        data.forEach(row => Object.keys(row).forEach(k => keys.add(k)));
+        const fields = new Set();
+        const returning = new Set();
+        data.forEach(row => {
+            core_1.Utils.keys(row).forEach(k => {
+                keys.add(k);
+                if (core_1.Utils.isRawSql(row[k])) {
+                    returning.add(k);
+                }
+            });
+        });
+        // reload generated columns and version fields
+        meta?.props
+            .filter(prop => (prop.generated && !prop.primary) || prop.version)
+            .forEach(prop => returning.add(prop.name));
         const pkCond = core_1.Utils.flatten(meta.primaryKeys.map(pk => meta.properties[pk].fieldNames)).map(pk => `${this.platform.quoteIdentifier(pk)} = ?`).join(' and ');
         const params = [];
         let sql = `update ${this.getTableName(meta, options)} set `;
+        const addParams = (prop, value) => {
+            if (prop.kind === core_1.ReferenceKind.EMBEDDED && prop.object) {
+                if (prop.array) {
+                    for (let i = 0; i < value.length; i++) {
+                        const item = value[i];
+                        value[i] = this.mapDataToFieldNames(item, false, prop.embeddedProps, options.convertCustomTypes);
+                    }
+                }
+                else {
+                    value = this.mapDataToFieldNames(value, false, prop.embeddedProps, options.convertCustomTypes);
+                }
+            }
+            params.push(value);
+        };
         keys.forEach(key => {
             const prop = meta.properties[key];
             prop.fieldNames.forEach((fieldName, fieldNameIdx) => {
+                if (fields.has(fieldName)) {
+                    return;
+                }
+                fields.add(fieldName);
                 sql += `${this.platform.quoteIdentifier(fieldName)} = case`;
                 where.forEach((cond, idx) => {
                     if (key in data[idx]) {
@@ -374,7 +514,8 @@ class AbstractSqlDriver extends core_1.DatabaseDriver {
                         else {
                             sql += '?';
                         }
-                        params.push(...pks, prop.fieldNames.length > 1 ? data[idx][key]?.[fieldNameIdx] : data[idx][key]);
+                        params.push(...pks);
+                        addParams(prop, prop.fieldNames.length > 1 ? data[idx][key]?.[fieldNameIdx] : data[idx][key]);
                     }
                 });
                 sql += ` else ${this.platform.quoteIdentifier(fieldName)} end, `;
@@ -385,7 +526,7 @@ class AbstractSqlDriver extends core_1.DatabaseDriver {
             const versionProperty = meta.properties[meta.versionProperty];
             const quotedFieldName = this.platform.quoteIdentifier(versionProperty.fieldNames[0]);
             sql += `${quotedFieldName} = `;
-            if (versionProperty.type.toLowerCase() === 'date') {
+            if (versionProperty.runtimeType === 'Date') {
                 sql += this.platform.getCurrentTimestampSQL(versionProperty.length);
             }
             else {
@@ -416,6 +557,11 @@ class AbstractSqlDriver extends core_1.DatabaseDriver {
             return '?';
         });
         sql += ` in (${conds.join(', ')})`;
+        if (this.platform.usesReturningStatement() && returning.size > 0) {
+            const returningFields = core_1.Utils.flatten([...returning].map(prop => meta.properties[prop].fieldNames));
+            /* istanbul ignore next */
+            sql += returningFields.length > 0 ? ` returning ${returningFields.map(field => this.platform.quoteIdentifier(field)).join(', ')}` : '';
+        }
         const res = await this.rethrow(this.execute(sql, params, 'run', options.ctx));
         for (let i = 0; i < collections.length; i++) {
             await this.processManyToMany(meta, where[i], collections[i], false, options);
@@ -431,49 +577,81 @@ class AbstractSqlDriver extends core_1.DatabaseDriver {
         const qb = this.createQueryBuilder(entityName, options.ctx, 'write', false).delete(where).withSchema(this.getSchemaName(meta, options));
         return this.rethrow(qb.execute('run', false));
     }
-    async syncCollection(coll, options) {
-        const wrapped = (0, core_1.helper)(coll.owner);
-        const meta = wrapped.__meta;
-        const pks = wrapped.getPrimaryKeys(true);
-        const snap = coll.getSnapshot();
-        const includes = (arr, item) => !!arr.find(i => core_1.Utils.equals(i, item));
-        const snapshot = snap ? snap.map(item => (0, core_1.helper)(item).getPrimaryKeys(true)) : [];
-        const current = coll.getItems(false).map(item => (0, core_1.helper)(item).getPrimaryKeys(true));
-        const deleteDiff = snap ? snapshot.filter(item => !includes(current, item)) : true;
-        const insertDiff = current.filter(item => !includes(snapshot, item));
-        const target = snapshot.filter(item => includes(current, item)).concat(...insertDiff);
-        const equals = core_1.Utils.equals(current, target);
-        const ctx = options?.ctx;
-        // wrong order if we just delete and insert to the end (only owning sides can have fixed order)
-        if (coll.property.owner && coll.property.fixedOrder && !equals && Array.isArray(deleteDiff)) {
-            deleteDiff.length = insertDiff.length = 0;
-            deleteDiff.push(...snapshot);
-            insertDiff.push(...current);
-        }
-        if (coll.property.reference === core_1.ReferenceType.ONE_TO_MANY) {
-            const cols = coll.property.referencedColumnNames;
-            const qb = this.createQueryBuilder(coll.property.type, ctx, 'write')
-                .withSchema(this.getSchemaName(meta, options))
-                .update({ [coll.property.mappedBy]: pks })
-                .getKnexQuery()
-                .whereIn(cols, insertDiff);
-            return this.rethrow(this.execute(qb));
-        }
-        /* istanbul ignore next */
-        const ownerSchema = wrapped.getSchema() === '*' ? this.config.get('schema') : wrapped.getSchema();
-        const pivotMeta = this.metadata.find(coll.property.pivotEntity);
-        if (pivotMeta.schema === '*') {
+    async syncCollections(collections, options) {
+        const groups = {};
+        for (const coll of collections) {
+            const wrapped = (0, core_1.helper)(coll.owner);
+            const meta = wrapped.__meta;
+            const pks = wrapped.getPrimaryKeys(true);
+            const snap = coll.getSnapshot();
+            const includes = (arr, item) => !!arr.find(i => core_1.Utils.equals(i, item));
+            const snapshot = snap ? snap.map(item => (0, core_1.helper)(item).getPrimaryKeys(true)) : [];
+            const current = coll.getItems(false).map(item => (0, core_1.helper)(item).getPrimaryKeys(true));
+            const deleteDiff = snap ? snapshot.filter(item => !includes(current, item)) : true;
+            const insertDiff = current.filter(item => !includes(snapshot, item));
+            const target = snapshot.filter(item => includes(current, item)).concat(...insertDiff);
+            const equals = core_1.Utils.equals(current, target);
+            // wrong order if we just delete and insert to the end (only owning sides can have fixed order)
+            if (coll.property.owner && coll.property.fixedOrder && !equals && Array.isArray(deleteDiff)) {
+                deleteDiff.length = insertDiff.length = 0;
+                deleteDiff.push(...snapshot);
+                insertDiff.push(...current);
+            }
+            if (coll.property.kind === core_1.ReferenceKind.ONE_TO_MANY) {
+                const cols = coll.property.referencedColumnNames;
+                const qb = this.createQueryBuilder(coll.property.type, options?.ctx, 'write')
+                    .withSchema(this.getSchemaName(meta, options));
+                if (coll.getSnapshot() === undefined) {
+                    if (coll.property.orphanRemoval) {
+                        const kqb = qb.delete({ [coll.property.mappedBy]: pks })
+                            .getKnexQuery()
+                            .whereNotIn(cols, insertDiff);
+                        return this.rethrow(this.execute(kqb));
+                    }
+                    const kqb = qb.update({ [coll.property.mappedBy]: null })
+                        .getKnexQuery()
+                        .whereNotIn(cols, insertDiff);
+                    return this.rethrow(this.execute(kqb));
+                }
+                const kqb = qb.update({ [coll.property.mappedBy]: pks })
+                    .getKnexQuery()
+                    .whereIn(cols, insertDiff);
+                return this.rethrow(this.execute(kqb));
+            }
             /* istanbul ignore next */
-            options ?? (options = {});
-            options.schema = ownerSchema;
+            const pivotMeta = this.metadata.find(coll.property.pivotEntity);
+            let schema = pivotMeta.schema;
+            if (schema === '*') {
+                const ownerSchema = wrapped.getSchema() === '*' ? this.config.get('schema') : wrapped.getSchema();
+                schema = coll.property.owner ? ownerSchema : this.config.get('schema');
+            }
+            else if (schema == null) {
+                schema = this.config.get('schema');
+            }
+            const tableName = `${schema ?? '_'}.${pivotMeta.tableName}`;
+            const persister = groups[tableName] ??= new PivotCollectionPersister_1.PivotCollectionPersister(pivotMeta, this, options?.ctx, schema);
+            persister.enqueueUpdate(coll.property, insertDiff, deleteDiff, pks);
+        }
+        for (const persister of core_1.Utils.values(groups)) {
+            await this.rethrow(persister.execute());
         }
-        return this.rethrow(this.updateCollectionDiff(meta, coll.property, pks, deleteDiff, insertDiff, options));
     }
-    async loadFromPivotTable(prop, owners, where = {}, orderBy, ctx, options) {
-        const pivotProp2 = this.getPivotInverseProperty(prop);
-        const ownerMeta = this.metadata.find(pivotProp2.type);
+    async loadFromPivotTable(prop, owners, where = {}, orderBy, ctx, options, pivotJoin) {
         const pivotMeta = this.metadata.find(prop.pivotEntity);
-        const cond = { [`${prop.pivotEntity}.${pivotProp2.name}`]: { $in: ownerMeta.compositePK ? owners : owners.map(o => o[0]) } };
+        const pivotProp1 = pivotMeta.relations[prop.owner ? 1 : 0];
+        const pivotProp2 = pivotMeta.relations[prop.owner ? 0 : 1];
+        const ownerMeta = this.metadata.find(pivotProp2.type);
+        options = { ...options };
+        const qb = this.createQueryBuilder(prop.pivotEntity, ctx, options.connectionType)
+            .withSchema(this.getSchemaName(pivotMeta, options))
+            .indexHint(options.indexHint)
+            .comment(options.comments)
+            .hintComment(options.hintComments);
+        const pivotAlias = qb.alias;
+        const pivotKey = pivotProp2.joinColumns.map(column => `${pivotAlias}.${column}`).join(core_1.Utils.PK_SEPARATOR);
+        const cond = {
+            [pivotKey]: { $in: ownerMeta.compositePK ? owners : owners.map(o => o[0]) },
+        };
         /* istanbul ignore if */
         if (!core_1.Utils.isEmpty(where) && Object.keys(where).every(k => core_1.Utils.isOperator(k, false))) {
             where = cond;
@@ -481,29 +659,51 @@ class AbstractSqlDriver extends core_1.DatabaseDriver {
         else {
             where = { ...where, ...cond };
         }
-        orderBy = this.getPivotOrderBy(prop, orderBy);
-        const qb = this.createQueryBuilder(prop.type, ctx, options?.connectionType)
-            .unsetFlag(core_1.QueryFlag.CONVERT_CUSTOM_TYPES)
-            .withSchema(this.getSchemaName(prop.targetMeta, options));
-        const populate = this.autoJoinOneToOneOwner(prop.targetMeta, [{ field: prop.pivotEntity }]);
-        const fields = this.buildFields(prop.targetMeta, (options?.populate ?? []), [], qb, options?.fields);
-        qb.select(fields).populate(populate).where(where).orderBy(orderBy).setLockMode(options?.lockMode, options?.lockTableAliases);
-        if (owners.length === 1 && (options?.offset != null || options?.limit != null)) {
-            qb.limit(options.limit, options.offset);
-        }
-        if (prop.targetMeta.schema !== '*' && pivotMeta.schema === '*' && options?.schema) {
-            // eslint-disable-next-line dot-notation
-            qb['finalize']();
-            // eslint-disable-next-line dot-notation
-            Object.values(qb['_joins']).forEach(join => {
-                join.schema = options.schema;
+        orderBy = this.getPivotOrderBy(prop, pivotProp1, pivotAlias, orderBy);
+        const populate = this.autoJoinOneToOneOwner(prop.targetMeta, []);
+        const fields = [];
+        const k1 = !prop.owner ? 'joinColumns' : 'inverseJoinColumns';
+        const k2 = prop.owner ? 'joinColumns' : 'inverseJoinColumns';
+        const cols = [
+            ...prop[k1].map(col => `${pivotAlias}.${col} as fk__${col}`),
+            ...prop[k2].map(col => `${pivotAlias}.${col} as fk__${col}`),
+        ];
+        fields.push(...cols);
+        if (!pivotJoin) {
+            const targetAlias = qb.getNextAlias(prop.targetMeta.tableName);
+            const targetSchema = this.getSchemaName(prop.targetMeta, options) ?? this.platform.getDefaultSchemaName();
+            qb.innerJoin(pivotProp1.name, targetAlias, {}, targetSchema);
+            const targetFields = this.buildFields(prop.targetMeta, (options.populate ?? []), [], qb, targetAlias, options.fields, options);
+            for (const field of targetFields) {
+                const f = field.toString();
+                fields.unshift(f.includes('.') ? field : `${targetAlias}.${f}`);
+            }
+            // we need to handle 1:1 owner auto-joins explicitly, as the QB type is the pivot table, not the target
+            populate.forEach(hint => {
+                const alias = qb.getNextAlias(prop.targetMeta.tableName);
+                qb.leftJoin(`${targetAlias}.${hint.field}`, alias);
+                // eslint-disable-next-line dot-notation
+                Object.values(qb['_joins']).forEach(join => {
+                    const [propName] = hint.field.split(':', 2);
+                    if (join.alias === alias && join.prop.name === propName) {
+                        fields.push(...qb.helper.mapJoinColumns(qb.type, join));
+                    }
+                });
             });
         }
-        const items = owners.length ? await this.rethrow(qb.execute('all')) : [];
+        qb.select(fields)
+            .where({ [pivotProp1.name]: where })
+            .orderBy(orderBy)
+            .setLockMode(options.lockMode, options.lockTableAliases);
+        if (owners.length === 1 && (options.offset != null || options.limit != null)) {
+            qb.limit(options.limit, options.offset);
+        }
+        const res = owners.length ? await this.rethrow(qb.execute('all', { mergeResults: false, mapResults: false })) : [];
+        const items = res.map((row) => super.mapResult(row, prop.targetMeta));
         const map = {};
         const pkProps = ownerMeta.getPrimaryProps();
         owners.forEach(owner => {
-            const key = core_1.Utils.getPrimaryKeyHash(prop.joinColumns.map((col, idx) => {
+            const key = core_1.Utils.getPrimaryKeyHash(prop.joinColumns.map((_col, idx) => {
                 const pkProp = pkProps[idx];
                 return pkProp.customType ? pkProp.customType.convertToJSValue(owner[idx], this.platform) : owner[idx];
             }));
@@ -516,10 +716,25 @@ class AbstractSqlDriver extends core_1.DatabaseDriver {
             }));
             map[key].push(item);
             prop.joinColumns.forEach(col => delete item[`fk__${col}`]);
-            prop.inverseJoinColumns.forEach(col => delete item[`fk__${col}`]);
+            prop.inverseJoinColumns.forEach((col, idx) => {
+                core_1.Utils.renameKey(item, `fk__${col}`, prop.targetMeta.primaryKeys[idx]);
+            });
         });
         return map;
     }
+    getPivotOrderBy(prop, pivotProp, pivotAlias, orderBy) {
+        // FIXME this is ignoring the rest of the array items
+        if (!core_1.Utils.isEmpty(orderBy)) {
+            return [{ [pivotProp.name]: core_1.Utils.asArray(orderBy)[0] }];
+        }
+        if (!core_1.Utils.isEmpty(prop.orderBy)) {
+            return [{ [pivotProp.name]: core_1.Utils.asArray(prop.orderBy)[0] }];
+        }
+        if (prop.fixedOrder) {
+            return [{ [`${pivotAlias}.${prop.fixedOrderColumn}`]: core_1.QueryOrder.ASC }];
+        }
+        return [];
+    }
     async execute(queryOrKnex, params = [], method = 'all', ctx) {
         return this.rethrow(this.connection.execute(queryOrKnex, params, method, ctx));
     }
@@ -532,39 +747,69 @@ class AbstractSqlDriver extends core_1.DatabaseDriver {
         }
         const relationsToPopulate = populate.map(({ field }) => field);
         const toPopulate = meta.relations
-            .filter(prop => prop.reference === core_1.ReferenceType.ONE_TO_ONE && !prop.owner && !relationsToPopulate.includes(prop.name))
+            .filter(prop => prop.kind === core_1.ReferenceKind.ONE_TO_ONE && !prop.owner && !relationsToPopulate.includes(prop.name))
             .filter(prop => fields.length === 0 || fields.some(f => prop.name === f || prop.name.startsWith(`${String(f)}.`)))
-            .map(prop => ({ field: prop.name, strategy: prop.strategy }));
+            .map(prop => ({ field: `${prop.name}:ref`, strategy: prop.strategy }));
         return [...populate, ...toPopulate];
     }
-    joinedProps(meta, populate) {
-        return populate.filter(p => {
-            const prop = meta.properties[p.field] || {};
-            return (p.strategy || prop.strategy || this.config.get('loadStrategy')) === core_1.LoadStrategy.JOINED && prop.reference !== core_1.ReferenceType.SCALAR;
+    /**
+     * @internal
+     */
+    joinedProps(meta, populate, options) {
+        return populate.filter(hint => {
+            const [propName, ref] = hint.field.split(':', 2);
+            const prop = meta.properties[propName] || {};
+            if ((options?.strategy || hint.strategy || prop.strategy || this.config.get('loadStrategy')) !== core_1.LoadStrategy.JOINED) {
+                return false;
+            }
+            if (ref) {
+                // keep only pivot ref joins here, as that only triggers actual join
+                return prop.kind === core_1.ReferenceKind.MANY_TO_MANY;
+            }
+            return ![core_1.ReferenceKind.SCALAR, core_1.ReferenceKind.EMBEDDED].includes(prop.kind);
         });
     }
     /**
      * @internal
      */
-    mergeJoinedResult(rawResults, meta) {
-        // group by the root entity primary key first
-        const map = {};
+    mergeJoinedResult(rawResults, meta, joinedProps) {
         const res = [];
-        rawResults.forEach(item => {
+        const map = {};
+        for (const item of rawResults) {
             const pk = core_1.Utils.getCompositeKeyHash(item, meta);
             if (map[pk]) {
-                map[pk].push(item);
+                for (const hint of joinedProps) {
+                    const [propName, ref] = hint.field.split(':', 2);
+                    const prop = meta.properties[propName];
+                    if (!item[propName]) {
+                        continue;
+                    }
+                    if (prop.kind === core_1.ReferenceKind.MANY_TO_MANY && ref) {
+                        map[pk][propName] = [...map[pk][propName], ...item[propName]];
+                        continue;
+                    }
+                    switch (prop.kind) {
+                        case core_1.ReferenceKind.ONE_TO_MANY:
+                        case core_1.ReferenceKind.MANY_TO_MANY:
+                            map[pk][propName] = this.mergeJoinedResult([...map[pk][propName], ...item[propName]], prop.targetMeta, hint.children ?? []);
+                            break;
+                        case core_1.ReferenceKind.MANY_TO_ONE:
+                        case core_1.ReferenceKind.ONE_TO_ONE:
+                            map[pk][propName] = this.mergeJoinedResult([map[pk][propName], item[propName]], prop.targetMeta, hint.children ?? [])[0];
+                            break;
+                    }
+                }
             }
             else {
-                map[pk] = [item];
+                map[pk] = item;
                 res.push(item);
             }
-        });
+        }
         return res;
     }
-    getFieldsForJoinedLoad(qb, meta, explicitFields, populate = [], parentTableAlias, parentJoinPath) {
+    getFieldsForJoinedLoad(qb, meta, explicitFields, populate = [], options, parentTableAlias, parentJoinPath) {
         const fields = [];
-        const joinedProps = this.joinedProps(meta, populate);
+        const joinedProps = this.joinedProps(meta, populate, options);
         if (explicitFields?.includes('*')) {
             fields.push('*');
         }
@@ -575,26 +820,41 @@ class AbstractSqlDriver extends core_1.DatabaseDriver {
             if (!fields || prop.primary) {
                 return !fields?.includes('*');
             }
-            return fields.includes(prop.name);
+            return fields.some(f => f === prop.name || f.toString().startsWith(prop.name + '.'));
         };
-        // alias all fields in the primary table
-        meta.props
-            .filter(prop => shouldHaveColumn(prop, populate, explicitFields))
-            .forEach(prop => fields.push(...this.mapPropToFieldNames(qb, prop, parentTableAlias)));
-        joinedProps.forEach(relation => {
-            const prop = meta.properties[relation.field];
+        const populateWhereAll = options?._populateWhere === 'all' || core_1.Utils.isEmpty(options?._populateWhere);
+        // root entity is already handled, skip that
+        if (parentJoinPath) {
+            // alias all fields in the primary table
+            meta.props
+                .filter(prop => shouldHaveColumn(prop, populate, explicitFields))
+                .forEach(prop => fields.push(...this.mapPropToFieldNames(qb, prop, parentTableAlias)));
+        }
+        joinedProps.forEach(hint => {
+            const [propName, ref] = hint.field.split(':', 2);
+            const prop = meta.properties[propName];
             const meta2 = this.metadata.find(prop.type);
+            const pivotRefJoin = prop.kind === core_1.ReferenceKind.MANY_TO_MANY && ref;
             const tableAlias = qb.getNextAlias(prop.name);
             const field = parentTableAlias ? `${parentTableAlias}.${prop.name}` : prop.name;
-            const path = parentJoinPath ? `${parentJoinPath}.${prop.name}` : `${meta.name}.${prop.name}`;
-            qb.join(field, tableAlias, {}, 'leftJoin', path);
+            let path = parentJoinPath ? `${parentJoinPath}.${prop.name}` : `${meta.name}.${prop.name}`;
+            if (!parentJoinPath && populateWhereAll && !path.startsWith('[populate]')) {
+                path = '[populate]' + path;
+            }
+            const joinType = pivotRefJoin ? query_1.JoinType.pivotJoin : query_1.JoinType.leftJoin;
+            qb.join(field, tableAlias, {}, joinType, path);
+            if (pivotRefJoin) {
+                fields.push(...prop.joinColumns.map(col => qb.helper.mapper(`${tableAlias}.${col}`, qb.type, undefined, `${tableAlias}__${col}`)), ...prop.inverseJoinColumns.map(col => qb.helper.mapper(`${tableAlias}.${col}`, qb.type, undefined, `${tableAlias}__${col}`)));
+            }
             const childExplicitFields = explicitFields?.filter(f => core_1.Utils.isPlainObject(f)).map(o => o[prop.name])[0] || [];
             explicitFields?.forEach(f => {
                 if (typeof f === 'string' && f.startsWith(`${prop.name}.`)) {
                     childExplicitFields.push(f.substring(prop.name.length + 1));
                 }
             });
-            fields.push(...this.getFieldsForJoinedLoad(qb, meta2, childExplicitFields.length === 0 ? undefined : childExplicitFields, relation.children, tableAlias, path));
+            if (!ref) {
+                fields.push(...this.getFieldsForJoinedLoad(qb, meta2, childExplicitFields.length === 0 ? undefined : childExplicitFields, hint.children, options, tableAlias, path));
+            }
         });
         return fields;
     }
@@ -602,24 +862,35 @@ class AbstractSqlDriver extends core_1.DatabaseDriver {
      * @internal
      */
     mapPropToFieldNames(qb, prop, tableAlias) {
-        const aliased = qb.ref(tableAlias ? `${tableAlias}__${prop.fieldNames[0]}` : prop.fieldNames[0]).toString();
-        if (prop.customType?.convertToJSValueSQL && tableAlias) {
-            const prefixed = qb.ref(prop.fieldNames[0]).withSchema(tableAlias).toString();
-            return [qb.raw(prop.customType.convertToJSValueSQL(prefixed, this.platform) + ' as ' + aliased).toString()];
+        const knex = this.connection.getKnex();
+        const aliased = knex.ref(tableAlias ? `${tableAlias}__${prop.fieldNames[0]}` : prop.fieldNames[0]).toString();
+        if (tableAlias && prop.customTypes?.some(type => type?.convertToJSValueSQL)) {
+            return prop.fieldNames.map((col, idx) => {
+                if (!prop.customTypes[idx]?.convertToJSValueSQL) {
+                    return col;
+                }
+                const prefixed = knex.ref(col).withSchema(tableAlias).toString();
+                const aliased = knex.ref(`${tableAlias}__${col}`).toString();
+                return (0, core_1.raw)(`${prop.customTypes[idx].convertToJSValueSQL(prefixed, this.platform)} as ${aliased}`);
+            });
+        }
+        if (tableAlias && prop.customType?.convertToJSValueSQL) {
+            const prefixed = knex.ref(prop.fieldNames[0]).withSchema(tableAlias).toString();
+            return [(0, core_1.raw)(`${prop.customType.convertToJSValueSQL(prefixed, this.platform)} as ${aliased}`)];
         }
         if (prop.formula) {
-            const alias = qb.ref(tableAlias ?? qb.alias).toString();
-            return [`${prop.formula(alias)} as ${aliased}`];
+            const alias = knex.ref(tableAlias ?? qb.alias).toString();
+            return [(0, core_1.raw)(`${prop.formula(alias)} as ${aliased}`)];
         }
         if (tableAlias) {
-            return prop.fieldNames.map(fieldName => qb.ref(fieldName).withSchema(tableAlias).as(`${tableAlias}__${fieldName}`));
+            return prop.fieldNames.map(fieldName => knex.ref(fieldName).withSchema(tableAlias).as(`${tableAlias}__${fieldName}`));
         }
         return prop.fieldNames;
     }
     /** @internal */
-    createQueryBuilder(entityName, ctx, preferredConnectionType, convertCustomTypes) {
+    createQueryBuilder(entityName, ctx, preferredConnectionType, convertCustomTypes, logging) {
         const connectionType = this.resolveConnectionType({ ctx, connectionType: preferredConnectionType });
-        const qb = new query_1.QueryBuilder(entityName, this.metadata, this, ctx, undefined, connectionType);
+        const qb = new query_1.QueryBuilder(entityName, this.metadata, this, ctx, undefined, connectionType, undefined, logging);
         if (!convertCustomTypes) {
             qb.unsetFlag(core_1.QueryFlag.CONVERT_CUSTOM_TYPES);
         }
@@ -643,7 +914,7 @@ class AbstractSqlDriver extends core_1.DatabaseDriver {
         }
         const ret = {};
         this.metadata.find(entityName).relations.forEach(prop => {
-            if (prop.reference === core_1.ReferenceType.MANY_TO_MANY && data[prop.name]) {
+            if (prop.kind === core_1.ReferenceKind.MANY_TO_MANY && data[prop.name]) {
                 ret[prop.name] = data[prop.name].map((item) => core_1.Utils.asArray(item));
                 delete data[prop.name];
             }
@@ -656,77 +927,97 @@ class AbstractSqlDriver extends core_1.DatabaseDriver {
         }
         for (const prop of meta.relations) {
             if (collections[prop.name]) {
-                await this.rethrow(this.updateCollectionDiff(meta, prop, pks, clear, collections[prop.name], options));
+                const pivotMeta = this.metadata.find(prop.pivotEntity);
+                const persister = new PivotCollectionPersister_1.PivotCollectionPersister(pivotMeta, this, options?.ctx, options?.schema);
+                persister.enqueueUpdate(prop, collections[prop.name], clear, pks);
+                await this.rethrow(persister.execute());
             }
         }
     }
-    async updateCollectionDiff(meta, prop, pks, deleteDiff, insertDiff, options) {
-        if (!deleteDiff) {
-            deleteDiff = [];
-        }
-        if (deleteDiff === true || deleteDiff.length > 0) {
-            const qb1 = this.createQueryBuilder(prop.pivotEntity, options?.ctx, 'write')
-                .withSchema(this.getSchemaName(meta, options))
-                .unsetFlag(core_1.QueryFlag.CONVERT_CUSTOM_TYPES);
-            const knex = qb1.getKnex();
-            if (Array.isArray(deleteDiff)) {
-                knex.whereIn(prop.inverseJoinColumns, deleteDiff);
-            }
-            prop.joinColumns.forEach((joinColumn, idx) => knex.andWhere(joinColumn, pks[idx]));
-            await this.execute(knex.delete());
-        }
-        if (insertDiff.length === 0) {
-            return;
-        }
-        const items = insertDiff.map(item => {
-            const cond = {};
-            prop.joinColumns.forEach((joinColumn, idx) => cond[joinColumn] = pks[idx]);
-            prop.inverseJoinColumns.forEach((inverseJoinColumn, idx) => cond[inverseJoinColumn] = item[idx]);
-            return cond;
-        });
-        /* istanbul ignore else */
-        if (this.platform.allowsMultiInsert()) {
-            await this.nativeInsertMany(prop.pivotEntity, items, {
-                ...options,
-                convertCustomTypes: false,
-                processCollections: false,
-            });
-        }
-        else {
-            await core_1.Utils.runSerial(items, item => {
-                return this.createQueryBuilder(prop.pivotEntity, options?.ctx, 'write')
-                    .unsetFlag(core_1.QueryFlag.CONVERT_CUSTOM_TYPES)
-                    .withSchema(this.getSchemaName(meta, options))
-                    .insert(item)
-                    .execute('run', false);
-            });
-        }
-    }
     async lockPessimistic(entity, options) {
         const meta = (0, core_1.helper)(entity).__meta;
-        const qb = this.createQueryBuilder(entity.constructor.name, options.ctx).unsetFlag(core_1.QueryFlag.CONVERT_CUSTOM_TYPES).withSchema(options.schema ?? meta.schema);
+        const qb = this.createQueryBuilder(entity.constructor.name, options.ctx).withSchema(options.schema ?? meta.schema);
         const cond = core_1.Utils.getPrimaryKeyCond(entity, meta.primaryKeys);
-        qb.select('1').where(cond).setLockMode(options.lockMode, options.lockTableAliases);
+        qb.select((0, core_1.raw)('1')).where(cond).setLockMode(options.lockMode, options.lockTableAliases);
         await this.rethrow(qb.execute());
     }
-    buildJoinedPropsOrderBy(entityName, qb, meta, populate, parentPath) {
+    buildOrderBy(qb, meta, populate, options) {
+        const joinedProps = this.joinedProps(meta, populate, options);
+        // `options._populateWhere` is a copy of the value provided by user with a fallback to the global config option
+        // as `options.populateWhere` will be always recomputed to respect filters
+        const populateWhereAll = options._populateWhere !== 'infer' && !core_1.Utils.isEmpty(options._populateWhere);
+        const path = (populateWhereAll ? '[populate]' : '') + meta.className;
+        const populateOrderBy = this.buildPopulateOrderBy(qb, meta, core_1.Utils.asArray(options.populateOrderBy ?? options.orderBy), path);
+        const joinedPropsOrderBy = this.buildJoinedPropsOrderBy(qb, meta, joinedProps, options, path);
+        return [...core_1.Utils.asArray(options.orderBy), ...populateOrderBy, ...joinedPropsOrderBy];
+    }
+    buildPopulateOrderBy(qb, meta, populateOrderBy, parentPath, parentAlias = qb.alias) {
         const orderBy = [];
-        const joinedProps = this.joinedProps(meta, populate);
-        joinedProps.forEach(relation => {
-            const prop = meta.properties[relation.field];
+        for (let i = 0; i < populateOrderBy.length; i++) {
+            const orderHint = populateOrderBy[i];
+            for (const propName of core_1.Utils.keys(orderHint)) {
+                const prop = meta.properties[propName];
+                const meta2 = this.metadata.find(prop.type);
+                const childOrder = orderHint[prop.name];
+                let path = `${parentPath}.${propName}`;
+                if (prop.kind === core_1.ReferenceKind.MANY_TO_MANY && typeof childOrder !== 'object') {
+                    path += '[pivot]';
+                }
+                const join = qb.getJoinForPath(path, { matchPopulateJoins: true });
+                const propAlias = qb.getAliasForJoinPath(join ?? path, { matchPopulateJoins: true }) ?? parentAlias;
+                if (!join && parentAlias === qb.alias) {
+                    continue;
+                }
+                if (![core_1.ReferenceKind.SCALAR, core_1.ReferenceKind.EMBEDDED].includes(prop.kind) && typeof childOrder === 'object') {
+                    const children = this.buildPopulateOrderBy(qb, meta2, core_1.Utils.asArray(childOrder), path, propAlias);
+                    orderBy.push(...children);
+                    continue;
+                }
+                if (prop.kind === core_1.ReferenceKind.MANY_TO_MANY && join) {
+                    if (prop.fixedOrderColumn) {
+                        orderBy.push({ [`${join.alias}.${prop.fixedOrderColumn}`]: childOrder });
+                    }
+                    else {
+                        for (const col of prop.inverseJoinColumns) {
+                            orderBy.push({ [`${join.ownerAlias}.${col}`]: childOrder });
+                        }
+                    }
+                    continue;
+                }
+                const order = typeof childOrder === 'object' ? childOrder[propName] : childOrder;
+                orderBy.push({ [`${propAlias}.${propName}`]: order });
+            }
+        }
+        return orderBy;
+    }
+    buildJoinedPropsOrderBy(qb, meta, populate, options, parentPath) {
+        const orderBy = [];
+        const joinedProps = this.joinedProps(meta, populate, options);
+        for (const hint of joinedProps) {
+            const [propName, ref] = hint.field.split(':', 2);
+            const prop = meta.properties[propName];
             const propOrderBy = prop.orderBy;
-            const path = `${parentPath ? parentPath : entityName}.${relation.field}`;
-            const propAlias = qb.getAliasForJoinPath(path);
+            let path = `${parentPath}.${propName}`;
+            if (prop.kind === core_1.ReferenceKind.MANY_TO_MANY && ref) {
+                path += '[pivot]';
+            }
+            const join = qb.getJoinForPath(path, { matchPopulateJoins: true });
+            const propAlias = qb.getAliasForJoinPath(join ?? path, { matchPopulateJoins: true });
+            const meta2 = this.metadata.find(prop.type);
+            if (prop.kind === core_1.ReferenceKind.MANY_TO_MANY && prop.fixedOrder && join) {
+                const alias = ref ? propAlias : join.ownerAlias;
+                orderBy.push({ [`${alias}.${prop.fixedOrderColumn}`]: core_1.QueryOrder.ASC });
+            }
             if (propOrderBy) {
-                Object.keys(propOrderBy).forEach(field => {
+                core_1.Utils.keys(propOrderBy).forEach(field => {
                     orderBy.push({ [`${propAlias}.${field}`]: propOrderBy[field] });
                 });
             }
-            if (relation.children) {
-                const meta2 = this.metadata.find(prop.type);
-                orderBy.push(...this.buildJoinedPropsOrderBy(prop.name, qb, meta2, relation.children, path));
+            if (hint.children) {
+                const buildJoinedPropsOrderBy = this.buildJoinedPropsOrderBy(qb, meta2, hint.children, options, path);
+                orderBy.push(...buildJoinedPropsOrderBy);
             }
-        });
+        }
         return orderBy;
     }
     normalizeFields(fields, prefix = '') {
@@ -745,12 +1036,12 @@ class AbstractSqlDriver extends core_1.DatabaseDriver {
         return ret;
     }
     processField(meta, prop, field, ret, populate, joinedProps, qb) {
-        if (!prop || (prop.reference === core_1.ReferenceType.ONE_TO_ONE && !prop.owner)) {
+        if (!prop || (prop.kind === core_1.ReferenceKind.ONE_TO_ONE && !prop.owner)) {
             return;
         }
-        if (prop.reference === core_1.ReferenceType.EMBEDDED) {
+        if (prop.kind === core_1.ReferenceKind.EMBEDDED) {
             if (prop.object) {
-                ret.push(prop.fieldNames[0]);
+                ret.push(prop.name);
                 return;
             }
             const parts = field.split('.');
@@ -762,20 +1053,17 @@ class AbstractSqlDriver extends core_1.DatabaseDriver {
             }
             return;
         }
-        if (prop.fieldNames) {
-            ret.push(...prop.fieldNames);
-        }
+        ret.push(prop.name);
     }
-    buildFields(meta, populate, joinedProps, qb, fields) {
+    buildFields(meta, populate, joinedProps, qb, alias, fields, options) {
         const lazyProps = meta.props.filter(prop => prop.lazy && !populate.some(p => p.field === prop.name || p.all));
         const hasLazyFormulas = meta.props.some(p => p.lazy && p.formula);
-        const requiresSQLConversion = meta.props.some(p => p.customType?.convertToJSValueSQL);
+        const requiresSQLConversion = meta.props.some(p => p.customType?.convertToJSValueSQL && p.persist !== false);
         const hasExplicitFields = !!fields;
         const ret = [];
-        if (joinedProps.length > 0) {
-            ret.push(...this.getFieldsForJoinedLoad(qb, meta, fields, populate));
-        }
-        else if (fields) {
+        let addFormulas = false;
+        // handle root entity properties first, this is used for both strategies in the same way
+        if (fields) {
             for (const field of this.normalizeFields(fields)) {
                 if (field === '*') {
                     ret.push('*');
@@ -792,28 +1080,39 @@ class AbstractSqlDriver extends core_1.DatabaseDriver {
                 });
                 this.processField(meta, prop, parts.join('.'), ret, populate, joinedProps, qb);
             }
-            ret.unshift(...meta.primaryKeys.filter(pk => !fields.includes(pk)));
+            if (!fields.includes('*') && !fields.includes(`${qb.alias}.*`)) {
+                ret.unshift(...meta.primaryKeys.filter(pk => !fields.includes(pk)));
+            }
         }
         else if (lazyProps.filter(p => !p.formula).length > 0) {
             const props = meta.props.filter(prop => this.platform.shouldHaveColumn(prop, populate, false));
             ret.push(...core_1.Utils.flatten(props.filter(p => !lazyProps.includes(p)).map(p => p.fieldNames)));
+            addFormulas = true;
         }
         else if (hasLazyFormulas || requiresSQLConversion) {
             ret.push('*');
+            addFormulas = true;
         }
-        if (ret.length > 0 && !hasExplicitFields) {
+        else {
+            ret.push('*');
+        }
+        if (ret.length > 0 && !hasExplicitFields && addFormulas) {
             meta.props
                 .filter(prop => prop.formula && !lazyProps.includes(prop))
                 .forEach(prop => {
-                const alias = qb.ref(qb.alias).toString();
-                const aliased = qb.ref(prop.fieldNames[0]).toString();
-                ret.push(`${prop.formula(alias)} as ${aliased}`);
+                const a = this.connection.getKnex().ref(alias).toString();
+                const aliased = this.connection.getKnex().ref(prop.fieldNames[0]).toString();
+                ret.push((0, core_1.raw)(`${prop.formula(a)} as ${aliased}`));
             });
             meta.props
-                .filter(prop => prop.customType?.convertToDatabaseValueSQL || prop.customType?.convertToJSValueSQL)
+                .filter(prop => prop.hasConvertToDatabaseValueSQL || prop.hasConvertToJSValueSQL)
                 .forEach(prop => ret.push(prop.name));
         }
-        return ret.length > 0 ? core_1.Utils.unique(ret) : ['*'];
+        // add joined relations after the root entity fields
+        if (joinedProps.length > 0) {
+            ret.push(...this.getFieldsForJoinedLoad(qb, meta, fields, populate, options, alias));
+        }
+        return core_1.Utils.unique(ret);
     }
 }
 exports.AbstractSqlDriver = AbstractSqlDriver;