@mikro-orm/knex 7.0.0-dev.9 → 7.0.0-dev.90

package/AbstractSqlDriver.js

@@ -1,4 +1,4 @@
-import { ALIAS_REPLACEMENT_RE, DatabaseDriver, EntityManagerType, getOnConflictFields, getOnConflictReturningFields, helper, isRaw, LoadStrategy, parseJsonSafe, QueryFlag, QueryHelper, QueryOrder, raw, RawQueryFragment, ReferenceKind, Utils, } from '@mikro-orm/core';
+import { ALIAS_REPLACEMENT_RE, DatabaseDriver, EntityManagerType, getLoadingStrategy, getOnConflictFields, getOnConflictReturningFields, helper, isRaw, LoadStrategy, parseJsonSafe, QueryFlag, QueryHelper, QueryOrder, raw, RawQueryFragment, ReferenceKind, Utils, } from '@mikro-orm/core';
 import { QueryBuilder } from './query/QueryBuilder.js';
 import { JoinType, QueryType } from './query/enums.js';
 import { SqlEntityManager } from './SqlEntityManager.js';
@@ -21,15 +21,11 @@ export class AbstractSqlDriver extends DatabaseDriver {
         const EntityManagerClass = this.config.get('entityManager', SqlEntityManager);
         return new EntityManagerClass(this.config, this, this.metadata, useContext);
     }
-    async find(entityName, where, options = {}) {
-        options = { populate: [], orderBy: [], ...options };
-        const meta = this.metadata.find(entityName);
-        if (meta?.virtual) {
-            return this.findVirtual(entityName, where, options);
-        }
+    async createQueryBuilderFromOptions(meta, where, options = {}) {
+        const connectionType = this.resolveConnectionType({ ctx: options.ctx, connectionType: options.connectionType });
         const populate = this.autoJoinOneToOneOwner(meta, options.populate, options.fields);
         const joinedProps = this.joinedProps(meta, populate, options);
-        const qb = this.createQueryBuilder(entityName, options.ctx, options.connectionType, false, options.logging);
+        const qb = this.createQueryBuilder(meta.className, options.ctx, connectionType, false, options.logging, undefined, options.em);
         const fields = this.buildFields(meta, populate, joinedProps, qb, qb.alias, options);
         const orderBy = this.buildOrderBy(qb, meta, populate, options);
         const populateWhere = this.buildPopulateWhere(meta, joinedProps, options);
@@ -66,8 +62,17 @@ export class AbstractSqlDriver extends DatabaseDriver {
         if (options.em) {
             await qb.applyJoinedFilters(options.em, options.filters);
         }
+        return qb;
+    }
+    async find(entityName, where, options = {}) {
+        options = { populate: [], orderBy: [], ...options };
+        const meta = this.metadata.find(entityName);
+        if (meta?.virtual) {
+            return this.findVirtual(entityName, where, options);
+        }
+        const qb = await this.createQueryBuilderFromOptions(meta, where, options);
         const result = await this.rethrow(qb.execute('all'));
-        if (isCursorPagination && !first && !!last) {
+        if (options.last && !options.first) {
             result.reverse();
         }
         return result;
@@ -107,7 +112,7 @@ export class AbstractSqlDriver extends DatabaseDriver {
     }
     async findFromVirtual(entityName, where, options, type) {
         const meta = this.metadata.get(entityName);
-        /* v8 ignore next 3 */
+        /* v8 ignore next */
         if (!meta.expression) {
             return type === QueryType.SELECT ? [] : 0;
         }
@@ -130,32 +135,46 @@ export class AbstractSqlDriver extends DatabaseDriver {
         /* v8 ignore next */
         return res;
     }
-    async wrapVirtualExpressionInSubquery(meta, expression, where, options, type) {
-        const qb = this.createQueryBuilder(meta.className, options?.ctx, options.connectionType, options.convertCustomTypes, options.logging)
-            .indexHint(options.indexHint)
-            .comment(options.comments)
-            .hintComment(options.hintComments);
-        qb.where(where);
-        const { first, last, before, after } = options;
-        const isCursorPagination = [first, last, before, after].some(v => v != null);
-        if (type !== QueryType.COUNT) {
-            if (options.orderBy) {
-                if (isCursorPagination) {
-                    const { orderBy: newOrderBy, where } = this.processCursorOptions(meta, options, options.orderBy);
-                    qb.andWhere(where).orderBy(newOrderBy);
-                }
-                else {
-                    qb.orderBy(options.orderBy);
-                }
-            }
-            qb.limit(options?.limit, options?.offset);
+    async *streamFromVirtual(entityName, where, options) {
+        const meta = this.metadata.get(entityName);
+        /* v8 ignore next */
+        if (!meta.expression) {
+            return;
         }
-        const native = qb.getNativeQuery(false).clear('select');
-        if (type === QueryType.COUNT) {
-            native.count();
+        if (typeof meta.expression === 'string') {
+            yield* this.wrapVirtualExpressionInSubqueryStream(meta, meta.expression, where, options, QueryType.SELECT);
+            return;
         }
-        else { // select
-            native.select('*');
+        const em = this.createEntityManager();
+        em.setTransactionContext(options.ctx);
+        const res = meta.expression(em, where, options, true);
+        if (typeof res === 'string') {
+            yield* this.wrapVirtualExpressionInSubqueryStream(meta, res, where, options, QueryType.SELECT);
+            return;
+        }
+        if (res instanceof QueryBuilder) {
+            yield* this.wrapVirtualExpressionInSubqueryStream(meta, res.getFormattedQuery(), where, options, QueryType.SELECT);
+            return;
+        }
+        if (res instanceof RawQueryFragment) {
+            const expr = this.platform.formatQuery(res.sql, res.params);
+            yield* this.wrapVirtualExpressionInSubqueryStream(meta, expr, where, options, QueryType.SELECT);
+            return;
+        }
+        /* v8 ignore next */
+        yield* res;
+    }
+    async wrapVirtualExpressionInSubquery(meta, expression, where, options, type) {
+        const qb = await this.createQueryBuilderFromOptions(meta, where, options);
+        qb.setFlag(QueryFlag.DISABLE_PAGINATE);
+        const isCursorPagination = [options.first, options.last, options.before, options.after].some(v => v != null);
+        const native = qb.getNativeQuery(false);
+        if (type === QueryType.COUNT) {
+            native
+                .clear('select')
+                .clear('limit')
+                .clear('offset')
+                .count();
         }
         native.from(raw(`(${expression}) as ${this.platform.quoteIdentifier(qb.alias)}`));
         const query = native.compile();
@@ -163,14 +182,26 @@ export class AbstractSqlDriver extends DatabaseDriver {
         if (type === QueryType.COUNT) {
             return res[0].count;
         }
-        if (isCursorPagination && !first && !!last) {
+        if (isCursorPagination && !options.first && !!options.last) {
             res.reverse();
         }
         return res.map(row => this.mapResult(row, meta));
     }
+    async *wrapVirtualExpressionInSubqueryStream(meta, expression, where, options, type) {
+        const qb = await this.createQueryBuilderFromOptions(meta, where, options);
+        qb.unsetFlag(QueryFlag.DISABLE_PAGINATE);
+        const native = qb.getNativeQuery(false);
+        native.from(raw(`(${expression}) as ${this.platform.quoteIdentifier(qb.alias)}`));
+        const query = native.compile();
+        const connectionType = this.resolveConnectionType({ ctx: options.ctx, connectionType: options.connectionType });
+        const res = this.getConnection(connectionType).stream(query.sql, query.params, options.ctx, options.loggerContext);
+        for await (const row of res) {
+            yield this.mapResult(row, meta);
+        }
+    }
     mapResult(result, meta, populate = [], qb, map = {}) {
         const ret = super.mapResult(result, meta);
-        /* v8 ignore next 3 */
+        /* v8 ignore next */
         if (!ret) {
             return null;
         }
@@ -185,7 +216,7 @@ export class AbstractSqlDriver extends DatabaseDriver {
         joinedProps.forEach(hint => {
             const [propName, ref] = hint.field.split(':', 2);
             const prop = meta.properties[propName];
-            /* v8 ignore next 3 */
+            /* v8 ignore next */
             if (!prop) {
                 return;
             }
@@ -199,7 +230,7 @@ export class AbstractSqlDriver extends DatabaseDriver {
                 path += '[pivot]';
             }
             const relationAlias = qb.getAliasForJoinPath(path, { matchPopulateJoins: true });
-            /* v8 ignore next 3 */
+            /* v8 ignore next */
             if (!relationAlias) {
                 return;
             }
@@ -221,9 +252,13 @@ export class AbstractSqlDriver extends DatabaseDriver {
                 }
                 return;
             }
+            const mapToPk = !!(ref || prop.mapToPk);
+            const targetProps = mapToPk
+                ? meta2.getPrimaryProps()
+                : meta2.props.filter(prop => this.platform.shouldHaveColumn(prop, hint.children || []));
             // If the primary key value for the relation is null, we know we haven't joined to anything
             // and therefore we don't return any record (since all values would be null)
-            const hasPK = meta2.primaryKeys.every(pk => meta2.properties[pk].fieldNames.every(name => {
+            const hasPK = meta2.getPrimaryProps().every(pk => pk.fieldNames.every(name => {
                 return root[`${relationAlias}__${name}`] != null;
             }));
             if (!hasPK) {
@@ -233,14 +268,18 @@ export class AbstractSqlDriver extends DatabaseDriver {
                 if ([ReferenceKind.MANY_TO_ONE, ReferenceKind.ONE_TO_ONE].includes(prop.kind)) {
                     result[prop.name] = null;
                 }
+                for (const prop of targetProps) {
+                    for (const name of prop.fieldNames) {
+                        delete root[`${relationAlias}__${name}`];
+                    }
+                }
                 return;
             }
             let relationPojo = {};
             meta2.props
                 .filter(prop => !ref && prop.persist === false && prop.fieldNames)
-                .filter(prop => !prop.lazy || populate.some(p => p.field === prop.name || p.all))
                 .forEach(prop => {
-                /* v8 ignore next 3 */
+                /* v8 ignore next */
                 if (prop.fieldNames.length > 1) { // composite keys
                     relationPojo[prop.name] = prop.fieldNames.map(name => root[`${relationAlias}__${name}`]);
                 }
@@ -249,10 +288,6 @@ export class AbstractSqlDriver extends DatabaseDriver {
                     relationPojo[prop.name] = root[alias];
                 }
             });
-            const mapToPk = !!(ref || prop.mapToPk);
-            const targetProps = mapToPk
-                ? meta2.getPrimaryProps()
-                : meta2.props.filter(prop => this.platform.shouldHaveColumn(prop, hint.children || []));
             const tz = this.platform.getTimezone();
             for (const prop of targetProps) {
                 if (prop.fieldNames.every(name => typeof root[`${relationAlias}__${name}`] === 'undefined')) {
@@ -293,7 +328,9 @@ export class AbstractSqlDriver extends DatabaseDriver {
             // properties can be mapped to multiple places, e.g. when sharing a column in multiple FKs,
             // so we need to delete them after everything is mapped from given level
             for (const prop of targetProps) {
-                prop.fieldNames.map(name => delete root[`${relationAlias}__${name}`]);
+                for (const name of prop.fieldNames) {
+                    delete root[`${relationAlias}__${name}`];
+                }
             }
             if (mapToPk) {
                 const tmp = Object.values(relationPojo);
@@ -312,15 +349,20 @@ export class AbstractSqlDriver extends DatabaseDriver {
         });
     }
     async count(entityName, where, options = {}) {
-        const meta = this.metadata.find(entityName);
-        if (meta?.virtual) {
+        const meta = this.metadata.get(entityName);
+        if (meta.virtual) {
             return this.countVirtual(entityName, where, options);
         }
-        const joinedProps = meta ? this.joinedProps(meta, options.populate ?? []) : [];
-        const populateWhere = meta ? this.buildPopulateWhere(meta, joinedProps, options) : undefined;
-        const populate = options.populate ?? [];
-        const qb = this.createQueryBuilder(entityName, options.ctx, options.connectionType, false, options.logging)
-            .indexHint(options.indexHint)
+        options = { populate: [], ...options };
+        const populate = options.populate;
+        const joinedProps = this.joinedProps(meta, populate, options);
+        const qb = this.createQueryBuilder(entityName, options.ctx, options.connectionType, false, options.logging);
+        const populateWhere = this.buildPopulateWhere(meta, joinedProps, options);
+        if (meta && !Utils.isEmpty(populate)) {
+            this.buildFields(meta, populate, joinedProps, qb, qb.alias, options);
+        }
+        qb.__populateWhere = options._populateWhere;
+        qb.indexHint(options.indexHint)
             .comment(options.comments)
             .hintComment(options.hintComments)
             .groupBy(options.groupBy)
@@ -328,8 +370,8 @@ export class AbstractSqlDriver extends DatabaseDriver {
             .populate(populate, joinedProps.length > 0 ? populateWhere : undefined, joinedProps.length > 0 ? options.populateFilter : undefined)
             .withSchema(this.getSchemaName(meta, options))
             .where(where);
-        if (meta && !Utils.isEmpty(populate)) {
-            this.buildFields(meta, populate, joinedProps, qb, qb.alias, options, true);
+        if (options.em) {
+            await qb.applyJoinedFilters(options.em, options.filters);
         }
         return this.rethrow(qb.getCount());
     }
@@ -338,7 +380,7 @@ export class AbstractSqlDriver extends DatabaseDriver {
         const meta = this.metadata.find(entityName);
         const collections = this.extractManyToMany(entityName, data);
         const pks = meta?.primaryKeys ?? [this.config.getNamingStrategy().referenceColumnName()];
-        const qb = this.createQueryBuilder(entityName, options.ctx, 'write', options.convertCustomTypes).withSchema(this.getSchemaName(meta, options));
+        const qb = this.createQueryBuilder(entityName, options.ctx, 'write', options.convertCustomTypes, options.loggerContext).withSchema(this.getSchemaName(meta, options));
         const res = await this.rethrow(qb.insert(data).execute('run', false));
         res.row = res.row || {};
         let pk;
@@ -386,7 +428,12 @@ export class AbstractSqlDriver extends DatabaseDriver {
             sql += ' ' + data.map(() => `select null as ${this.platform.quoteIdentifier(pks[0])}`).join(' union all ');
         }
         const addParams = (prop, row) => {
-            let value = row[prop.name] ?? prop.default;
+            const rowValue = row[prop.name];
+            if (prop.nullable && rowValue === null) {
+                params.push(null);
+                return;
+            }
+            let value = rowValue ?? prop.default;
             if (prop.kind === ReferenceKind.EMBEDDED && prop.object) {
                 if (prop.array && value) {
                     value = this.platform.cloneEmbeddable(value);
@@ -399,14 +446,14 @@ export class AbstractSqlDriver extends DatabaseDriver {
                     value = this.mapDataToFieldNames(value, false, prop.embeddedProps, options.convertCustomTypes);
                 }
             }
-            if (options.convertCustomTypes && prop.customType) {
-                params.push(prop.customType.convertToDatabaseValue(value, this.platform, { key: prop.name, mode: 'query-data' }));
-                return;
-            }
             if (typeof value === 'undefined' && this.platform.usesDefaultKeyword()) {
                 params.push(raw('default'));
                 return;
             }
+            if (options.convertCustomTypes && prop.customType) {
+                params.push(prop.customType.convertToDatabaseValue(value, this.platform, { key: prop.name, mode: 'query-data' }));
+                return;
+            }
             params.push(value);
         };
         if (fields.length > 0 || this.platform.usesDefaultKeyword()) {
@@ -465,9 +512,9 @@ export class AbstractSqlDriver extends DatabaseDriver {
         if (transform) {
             sql = transform(sql);
         }
-        const res = await this.execute(sql, params, 'run', options.ctx);
+        const res = await this.execute(sql, params, 'run', options.ctx, options.loggerContext);
         let pk;
-        /* v8 ignore next 3 */
+        /* v8 ignore next */
         if (pks.length > 1) { // owner has composite pk
             pk = data.map(d => Utils.getPrimaryKeyCond(d, pks));
         }
@@ -493,7 +540,7 @@ export class AbstractSqlDriver extends DatabaseDriver {
             where = { [meta?.primaryKeys[0] ?? pks[0]]: where };
         }
         if (Utils.hasObjectKeys(data)) {
-            const qb = this.createQueryBuilder(entityName, options.ctx, 'write', options.convertCustomTypes)
+            const qb = this.createQueryBuilder(entityName, options.ctx, 'write', options.convertCustomTypes, options.loggerContext)
                 .withSchema(this.getSchemaName(meta, options));
             if (options.upsert) {
                 /* v8 ignore next */
@@ -532,7 +579,7 @@ export class AbstractSqlDriver extends DatabaseDriver {
         const meta = this.metadata.get(entityName);
         if (options.upsert) {
             const uniqueFields = options.onConflictFields ?? (Utils.isPlainObject(where[0]) ? Object.keys(where[0]).flatMap(key => Utils.splitPrimaryKeys(key)) : meta.primaryKeys);
-            const qb = this.createQueryBuilder(entityName, options.ctx, 'write', options.convertCustomTypes).withSchema(this.getSchemaName(meta, options));
+            const qb = this.createQueryBuilder(entityName, options.ctx, 'write', options.convertCustomTypes, options.loggerContext).withSchema(this.getSchemaName(meta, options));
             const returning = getOnConflictReturningFields(meta, data[0], uniqueFields, options);
             qb.insert(data)
                 .onConflict(uniqueFields)
@@ -645,7 +692,7 @@ export class AbstractSqlDriver extends DatabaseDriver {
             /* v8 ignore next */
             sql += returningFields.length > 0 ? ` returning ${returningFields.map(field => this.platform.quoteIdentifier(field)).join(', ')}` : '';
         }
-        const res = await this.rethrow(this.execute(sql, params, 'run', options.ctx));
+        const res = await this.rethrow(this.execute(sql, params, 'run', options.ctx, options.loggerContext));
         for (let i = 0; i < collections.length; i++) {
             await this.processManyToMany(meta, where[i], collections[i], false, options);
         }
@@ -657,7 +704,7 @@ export class AbstractSqlDriver extends DatabaseDriver {
         if (Utils.isPrimaryKey(where) && pks.length === 1) {
             where = { [pks[0]]: where };
         }
-        const qb = this.createQueryBuilder(entityName, options.ctx, 'write', false).delete(where).withSchema(this.getSchemaName(meta, options));
+        const qb = this.createQueryBuilder(entityName, options.ctx, 'write', false, options.loggerContext).delete(where).withSchema(this.getSchemaName(meta, options));
         return this.rethrow(qb.execute('run', false));
     }
     /**
@@ -721,7 +768,7 @@ export class AbstractSqlDriver extends DatabaseDriver {
                     await this.rethrow(query.execute());
                     continue;
                 }
-                /* v8 ignore next 5 */
+                /* v8 ignore next */
                 const query = qb.update({ [coll.property.mappedBy]: pks })
                     .where({ [cols.join(Utils.PK_SEPARATOR)]: { $in: insertDiff } });
                 await this.rethrow(query.execute());
@@ -744,126 +791,96 @@ export class AbstractSqlDriver extends DatabaseDriver {
                 schema = this.config.get('schema');
             }
             const tableName = `${schema ?? '_'}.${pivotMeta.tableName}`;
-            const persister = groups[tableName] ??= new PivotCollectionPersister(pivotMeta, this, options?.ctx, schema);
-            persister.enqueueUpdate(coll.property, insertDiff, deleteDiff, pks);
+            const persister = groups[tableName] ??= new PivotCollectionPersister(pivotMeta, this, options?.ctx, schema, options?.loggerContext);
+            persister.enqueueUpdate(coll.property, insertDiff, deleteDiff, pks, coll.isInitialized());
         }
         for (const persister of Utils.values(groups)) {
             await this.rethrow(persister.execute());
         }
     }
     async loadFromPivotTable(prop, owners, where = {}, orderBy, ctx, options, pivotJoin) {
+        if (owners.length === 0) {
+            return {};
+        }
         const pivotMeta = this.metadata.find(prop.pivotEntity);
         const pivotProp1 = pivotMeta.relations[prop.owner ? 1 : 0];
         const pivotProp2 = pivotMeta.relations[prop.owner ? 0 : 1];
         const ownerMeta = this.metadata.find(pivotProp2.type);
-        options = { ...options };
-        const qb = this.createQueryBuilder(prop.pivotEntity, ctx, options.connectionType, undefined, options?.logging)
-            .withSchema(this.getSchemaName(pivotMeta, options))
-            .indexHint(options.indexHint)
-            .comment(options.comments)
-            .hintComment(options.hintComments);
-        const pivotAlias = qb.alias;
-        const pivotKey = pivotProp2.joinColumns.map(column => `${pivotAlias}.${column}`).join(Utils.PK_SEPARATOR);
         const cond = {
-            [pivotKey]: { $in: ownerMeta.compositePK ? owners : owners.map(o => o[0]) },
+            [pivotProp2.name]: { $in: ownerMeta.compositePK ? owners : owners.map(o => o[0]) },
         };
-        /* v8 ignore next 3 */
-        if (!Utils.isEmpty(where) && Object.keys(where).every(k => Utils.isOperator(k, false))) {
-            where = cond;
-        }
-        else {
-            where = { ...where, ...cond };
-        }
-        orderBy = this.getPivotOrderBy(prop, pivotProp1, pivotAlias, orderBy);
-        const populate = this.autoJoinOneToOneOwner(prop.targetMeta, []);
-        const fields = [];
-        const k1 = !prop.owner ? 'joinColumns' : 'inverseJoinColumns';
-        const k2 = prop.owner ? 'joinColumns' : 'inverseJoinColumns';
-        const cols = [
-            ...prop[k1].map(col => `${pivotAlias}.${col} as fk__${col}`),
-            ...prop[k2].map(col => `${pivotAlias}.${col} as fk__${col}`),
-        ];
-        fields.push(...cols);
-        if (!pivotJoin) {
-            const targetAlias = qb.getNextAlias(prop.targetMeta.tableName);
-            const targetSchema = this.getSchemaName(prop.targetMeta, options) ?? this.platform.getDefaultSchemaName();
-            qb.innerJoin(pivotProp1.name, targetAlias, {}, targetSchema);
-            const targetFields = this.buildFields(prop.targetMeta, (options.populate ?? []), [], qb, targetAlias, options);
-            const additionalFields = [];
-            for (const field of targetFields) {
-                const f = field.toString();
-                additionalFields.push(f.includes('.') ? field : `${targetAlias}.${f}`);
-                if (RawQueryFragment.isKnownFragment(field)) {
-                    qb.rawFragments.add(f);
-                }
-            }
-            fields.unshift(...additionalFields);
-            // we need to handle 1:1 owner auto-joins explicitly, as the QB type is the pivot table, not the target
-            populate.forEach(hint => {
-                const alias = qb.getNextAlias(prop.targetMeta.tableName);
-                qb.leftJoin(`${targetAlias}.${hint.field}`, alias);
-                // eslint-disable-next-line dot-notation
-                for (const join of Object.values(qb['_joins'])) {
-                    const [propName] = hint.field.split(':', 2);
-                    if (join.alias === alias && join.prop.name === propName) {
-                        fields.push(...qb.helper.mapJoinColumns(qb.type, join));
-                    }
-                }
-            });
-        }
-        qb.select(fields)
-            .where({ [pivotProp1.name]: where })
-            .orderBy(orderBy)
-            .setLockMode(options.lockMode, options.lockTableAliases);
-        if (owners.length === 1 && (options.offset != null || options.limit != null)) {
-            qb.limit(options.limit, options.offset);
-        }
-        const res = owners.length ? await this.rethrow(qb.execute('all', { mergeResults: false, mapResults: false })) : [];
-        const tmp = {};
-        const items = res.map((row) => {
-            const root = super.mapResult(row, prop.targetMeta);
-            this.mapJoinedProps(root, prop.targetMeta, populate, qb, root, tmp, pivotMeta.className + '.' + pivotProp1.name);
-            return root;
+        if (!Utils.isEmpty(where)) {
+            cond[pivotProp1.name] = { ...where };
+        }
+        where = cond;
+        const populateField = pivotJoin ? `${pivotProp1.name}:ref` : pivotProp1.name;
+        const populate = this.autoJoinOneToOneOwner(prop.targetMeta, options?.populate ?? [], options?.fields);
+        const childFields = !Utils.isEmpty(options?.fields) ? options.fields.map(f => `${pivotProp1.name}.${f}`) : [];
+        const childExclude = !Utils.isEmpty(options?.exclude) ? options.exclude.map(f => `${pivotProp1.name}.${f}`) : [];
+        const fields = pivotJoin
+            ? [pivotProp1.name, pivotProp2.name]
+            : [pivotProp1.name, pivotProp2.name, ...childFields];
+        const res = await this.find(pivotMeta.className, where, {
+            ctx,
+            ...options,
+            fields,
+            exclude: childExclude,
+            orderBy: this.getPivotOrderBy(prop, pivotProp1, orderBy, options?.orderBy),
+            populate: [{ field: populateField, strategy: LoadStrategy.JOINED, joinType: JoinType.innerJoin, children: populate }],
+            populateWhere: undefined,
+            // @ts-ignore
+            _populateWhere: 'infer',
+            populateFilter: !Utils.isEmpty(options?.populateFilter) ? { [pivotProp2.name]: options?.populateFilter } : undefined,
         });
-        qb.clearRawFragmentsCache();
         const map = {};
-        const pkProps = ownerMeta.getPrimaryProps();
         for (const owner of owners) {
-            const key = Utils.getPrimaryKeyHash(prop.joinColumns.map((_col, idx) => {
-                const pkProp = pkProps[idx];
-                return pkProp.customType ? pkProp.customType.convertToJSValue(owner[idx], this.platform) : owner[idx];
-            }));
+            const key = Utils.getPrimaryKeyHash(owner);
             map[key] = [];
         }
-        for (const item of items) {
-            const key = Utils.getPrimaryKeyHash(prop.joinColumns.map((col, idx) => {
-                const pkProp = pkProps[idx];
-                return pkProp.customType ? pkProp.customType.convertToJSValue(item[`fk__${col}`], this.platform) : item[`fk__${col}`];
-            }));
-            map[key].push(item);
-            prop.joinColumns.forEach(col => delete item[`fk__${col}`]);
-            prop.inverseJoinColumns.forEach((col, idx) => {
-                Utils.renameKey(item, `fk__${col}`, prop.targetMeta.primaryKeys[idx]);
-            });
+        for (const item of res) {
+            const key = Utils.getPrimaryKeyHash(Utils.asArray(item[pivotProp2.name]));
+            map[key].push(item[pivotProp1.name]);
         }
         return map;
     }
-    getPivotOrderBy(prop, pivotProp, pivotAlias, orderBy) {
-        // FIXME this is ignoring the rest of the array items
+    getPivotOrderBy(prop, pivotProp, orderBy, parentOrderBy) {
         if (!Utils.isEmpty(orderBy)) {
-            return [{ [pivotProp.name]: Utils.asArray(orderBy)[0] }];
+            return Utils.asArray(orderBy).map(o => ({ [pivotProp.name]: o }));
+        }
+        if (prop.kind === ReferenceKind.MANY_TO_MANY && Utils.asArray(parentOrderBy).some(o => o[prop.name])) {
+            return Utils.asArray(parentOrderBy)
+                .filter(o => o[prop.name])
+                .map(o => ({ [pivotProp.name]: o[prop.name] }));
         }
         if (!Utils.isEmpty(prop.orderBy)) {
-            return [{ [pivotProp.name]: Utils.asArray(prop.orderBy)[0] }];
+            return Utils.asArray(prop.orderBy).map(o => ({ [pivotProp.name]: o }));
         }
         if (prop.fixedOrder) {
-            return [{ [`${pivotAlias}.${prop.fixedOrderColumn}`]: QueryOrder.ASC }];
+            return [{ [prop.fixedOrderColumn]: QueryOrder.ASC }];
         }
         return [];
     }
     async execute(query, params = [], method = 'all', ctx, loggerContext) {
         return this.rethrow(this.connection.execute(query, params, method, ctx, loggerContext));
     }
+    async *stream(entityName, where, options) {
+        options = { populate: [], orderBy: [], ...options };
+        const meta = this.metadata.find(entityName);
+        if (meta?.virtual) {
+            yield* this.streamFromVirtual(entityName, where, options);
+            return;
+        }
+        const qb = await this.createQueryBuilderFromOptions(meta, where, options);
+        try {
+            const result = qb.stream(options);
+            for await (const item of result) {
+                yield item;
+            }
+        }
+        catch (e) {
+            throw this.convertException(e);
+        }
+    }
     /**
      * 1:1 owner side needs to be marked for population so QB auto-joins the owner id
      */
@@ -875,7 +892,7 @@ export class AbstractSqlDriver extends DatabaseDriver {
         const toPopulate = meta.relations
             .filter(prop => prop.kind === ReferenceKind.ONE_TO_ONE && !prop.owner && !prop.lazy && !relationsToPopulate.includes(prop.name))
             .filter(prop => fields.length === 0 || fields.some(f => prop.name === f || prop.name.startsWith(`${String(f)}.`)))
-            .map(prop => ({ field: `${prop.name}:ref`, strategy: prop.strategy }));
+            .map(prop => ({ field: `${prop.name}:ref`, strategy: LoadStrategy.JOINED }));
         return [...populate, ...toPopulate];
     }
     /**
@@ -885,14 +902,15 @@ export class AbstractSqlDriver extends DatabaseDriver {
         return populate.filter(hint => {
             const [propName, ref] = hint.field.split(':', 2);
             const prop = meta.properties[propName] || {};
-            if (hint.filter && hint.strategy === LoadStrategy.JOINED) {
+            const strategy = getLoadingStrategy(hint.strategy || prop.strategy || options?.strategy || this.config.get('loadStrategy'), prop.kind);
+            if (ref && [ReferenceKind.ONE_TO_ONE, ReferenceKind.MANY_TO_ONE].includes(prop.kind)) {
                 return true;
             }
             // skip redundant joins for 1:1 owner population hints when using `mapToPk`
             if (prop.kind === ReferenceKind.ONE_TO_ONE && prop.mapToPk && prop.owner) {
                 return false;
             }
-            if ((options?.strategy || hint.strategy || prop.strategy || this.config.get('loadStrategy')) !== LoadStrategy.JOINED) {
+            if (strategy !== LoadStrategy.JOINED) {
                 // force joined strategy for explicit 1:1 owner populate hint as it would require a join anyway
                 return prop.kind === ReferenceKind.ONE_TO_ONE && !prop.owner;
             }
@@ -903,31 +921,26 @@ export class AbstractSqlDriver extends DatabaseDriver {
      * @internal
      */
     mergeJoinedResult(rawResults, meta, joinedProps) {
+        if (rawResults.length <= 1) {
+            return rawResults;
+        }
         const res = [];
         const map = {};
+        const collectionsToMerge = {};
+        const hints = joinedProps.map(hint => {
+            const [propName, ref] = hint.field.split(':', 2);
+            return { propName, ref, children: hint.children };
+        });
         for (const item of rawResults) {
             const pk = Utils.getCompositeKeyHash(item, meta);
             if (map[pk]) {
-                for (const hint of joinedProps) {
-                    const [propName, ref] = hint.field.split(':', 2);
-                    const prop = meta.properties[propName];
+                for (const { propName } of hints) {
                     if (!item[propName]) {
                         continue;
                     }
-                    if ([ReferenceKind.ONE_TO_MANY, ReferenceKind.MANY_TO_MANY].includes(prop.kind) && ref) {
-                        map[pk][propName] = [...map[pk][propName], ...item[propName]];
-                        continue;
-                    }
-                    switch (prop.kind) {
-                        case ReferenceKind.ONE_TO_MANY:
-                        case ReferenceKind.MANY_TO_MANY:
-                            map[pk][propName] = this.mergeJoinedResult([...map[pk][propName], ...item[propName]], prop.targetMeta, hint.children ?? []);
-                            break;
-                        case ReferenceKind.MANY_TO_ONE:
-                        case ReferenceKind.ONE_TO_ONE:
-                            map[pk][propName] = this.mergeJoinedResult([map[pk][propName], item[propName]], prop.targetMeta, hint.children ?? [])[0];
-                            break;
-                    }
+                    collectionsToMerge[pk] ??= {};
+                    collectionsToMerge[pk][propName] ??= [map[pk][propName]];
+                    collectionsToMerge[pk][propName].push(item[propName]);
                 }
             }
             else {
@@ -935,37 +948,59 @@ export class AbstractSqlDriver extends DatabaseDriver {
                 res.push(item);
             }
         }
+        for (const pk in collectionsToMerge) {
+            const entity = map[pk];
+            const collections = collectionsToMerge[pk];
+            for (const { propName, ref, children } of hints) {
+                if (!collections[propName]) {
+                    continue;
+                }
+                const prop = meta.properties[propName];
+                const items = collections[propName].flat();
+                if ([ReferenceKind.ONE_TO_MANY, ReferenceKind.MANY_TO_MANY].includes(prop.kind) && ref) {
+                    entity[propName] = items;
+                    continue;
+                }
+                switch (prop.kind) {
+                    case ReferenceKind.ONE_TO_MANY:
+                    case ReferenceKind.MANY_TO_MANY:
+                        entity[propName] = this.mergeJoinedResult(items, prop.targetMeta, children ?? []);
+                        break;
+                    case ReferenceKind.MANY_TO_ONE:
+                    case ReferenceKind.ONE_TO_ONE:
+                        entity[propName] = this.mergeJoinedResult(items, prop.targetMeta, children ?? [])[0];
+                        break;
+                }
+            }
+        }
         return res;
     }
+    shouldHaveColumn(meta, prop, populate, fields, exclude) {
+        if (!this.platform.shouldHaveColumn(prop, populate, exclude)) {
+            return false;
+        }
+        if (!fields || fields.includes('*') || prop.primary || meta.root.discriminatorColumn === prop.name) {
+            return true;
+        }
+        return fields.some(f => f === prop.name || f.toString().startsWith(prop.name + '.'));
+    }
     getFieldsForJoinedLoad(qb, meta, options) {
         const fields = [];
         const populate = options.populate ?? [];
         const joinedProps = this.joinedProps(meta, populate, options);
-        const shouldHaveColumn = (meta, prop, populate, fields) => {
-            if (!this.platform.shouldHaveColumn(prop, populate, options.exclude)) {
-                return false;
-            }
-            if (!fields || fields.includes('*') || prop.primary || meta.root.discriminatorColumn === prop.name) {
-                return true;
-            }
-            return fields.some(f => f === prop.name || f.toString().startsWith(prop.name + '.'));
-        };
         const populateWhereAll = options?._populateWhere === 'all' || Utils.isEmpty(options?._populateWhere);
         // root entity is already handled, skip that
         if (options.parentJoinPath) {
             // alias all fields in the primary table
             meta.props
-                .filter(prop => shouldHaveColumn(meta, prop, populate, options.explicitFields))
-                .forEach(prop => fields.push(...this.mapPropToFieldNames(qb, prop, options.parentTableAlias)));
+                .filter(prop => this.shouldHaveColumn(meta, prop, populate, options.explicitFields, options.exclude))
+                .forEach(prop => fields.push(...this.mapPropToFieldNames(qb, prop, options.parentTableAlias, options.explicitFields)));
         }
         for (const hint of joinedProps) {
             const [propName, ref] = hint.field.split(':', 2);
             const prop = meta.properties[propName];
             // ignore ref joins of known FKs unless it's a filter hint
-            if (ref && !hint.filter && (prop.kind === ReferenceKind.MANY_TO_ONE || (prop.kind === ReferenceKind.ONE_TO_ONE && !prop.owner))) {
-                continue;
-            }
-            if (options.count && !options?.populateFilter?.[prop.name]) {
+            if (ref && !hint.filter && (prop.kind === ReferenceKind.MANY_TO_ONE || (prop.kind === ReferenceKind.ONE_TO_ONE && prop.owner))) {
                 continue;
             }
             const meta2 = this.metadata.find(prop.type);
@@ -976,11 +1011,14 @@ export class AbstractSqlDriver extends DatabaseDriver {
             if (!options.parentJoinPath && populateWhereAll && !hint.filter && !path.startsWith('[populate]')) {
                 path = '[populate]' + path;
             }
+            const mandatoryToOneProperty = [ReferenceKind.MANY_TO_ONE, ReferenceKind.ONE_TO_ONE].includes(prop.kind) && !prop.nullable;
             const joinType = pivotRefJoin
                 ? JoinType.pivotJoin
-                : hint.filter && !prop.nullable
-                    ? JoinType.innerJoin
-                    : JoinType.leftJoin;
+                : hint.joinType
+                    ? hint.joinType
+                    : (hint.filter && !prop.nullable) || mandatoryToOneProperty
+                        ? JoinType.innerJoin
+                        : JoinType.leftJoin;
             qb.join(field, tableAlias, {}, joinType, path);
             if (pivotRefJoin) {
                 fields.push(...prop.joinColumns.map(col => qb.helper.mapper(`${tableAlias}.${col}`, qb.type, undefined, `${tableAlias}__${col}`)), ...prop.inverseJoinColumns.map(col => qb.helper.mapper(`${tableAlias}.${col}`, qb.type, undefined, `${tableAlias}__${col}`)));
@@ -1012,7 +1050,7 @@ export class AbstractSqlDriver extends DatabaseDriver {
                     parentJoinPath: path,
                 }));
             }
-            else if (hint.filter || prop.mapToPk) {
+            else if (hint.filter || prop.mapToPk || (ref && [ReferenceKind.MANY_TO_ONE, ReferenceKind.ONE_TO_ONE].includes(prop.kind))) {
                 fields.push(...prop.referencedColumnNames.map(col => qb.helper.mapper(`${tableAlias}.${col}`, qb.type, undefined, `${tableAlias}__${col}`)));
             }
         }
@@ -1021,9 +1059,18 @@ export class AbstractSqlDriver extends DatabaseDriver {
     /**
      * @internal
      */
-    mapPropToFieldNames(qb, prop, tableAlias) {
+    mapPropToFieldNames(qb, prop, tableAlias, explicitFields) {
+        if (prop.kind === ReferenceKind.EMBEDDED && !prop.object) {
+            return Object.entries(prop.embeddedProps).flatMap(([name, childProp]) => {
+                const childFields = explicitFields ? Utils.extractChildElements(explicitFields, prop.name) : [];
+                if (!this.shouldHaveColumn(prop.targetMeta, { ...childProp, name }, [], childFields.length > 0 ? childFields : undefined)) {
+                    return [];
+                }
+                return this.mapPropToFieldNames(qb, childProp, tableAlias, childFields);
+            });
+        }
         const aliased = this.platform.quoteIdentifier(`${tableAlias}__${prop.fieldNames[0]}`);
-        if (prop.customTypes?.some(type => type?.convertToJSValueSQL)) {
+        if (prop.customTypes?.some(type => !!type?.convertToJSValueSQL)) {
             return prop.fieldNames.map((col, idx) => {
                 if (!prop.customTypes[idx]?.convertToJSValueSQL) {
                     return col;
@@ -1087,7 +1134,7 @@ export class AbstractSqlDriver extends DatabaseDriver {
         for (const prop of meta.relations) {
             if (collections[prop.name]) {
                 const pivotMeta = this.metadata.find(prop.pivotEntity);
-                const persister = new PivotCollectionPersister(pivotMeta, this, options?.ctx, options?.schema);
+                const persister = new PivotCollectionPersister(pivotMeta, this, options?.ctx, options?.schema, options?.loggerContext);
                 persister.enqueueUpdate(prop, collections[prop.name], clear, pks);
                 await this.rethrow(persister.execute());
             }
@@ -1154,7 +1201,7 @@ export class AbstractSqlDriver extends DatabaseDriver {
                 let path = parentPath;
                 const meta2 = this.metadata.find(prop.type);
                 const childOrder = orderHint[prop.name];
-                if (![ReferenceKind.MANY_TO_ONE, ReferenceKind.ONE_TO_ONE].includes(prop.kind) || !prop.owner || Utils.isPlainObject(childOrder)) {
+                if (prop.kind !== ReferenceKind.SCALAR && (![ReferenceKind.MANY_TO_ONE, ReferenceKind.ONE_TO_ONE].includes(prop.kind) || !prop.owner || Utils.isPlainObject(childOrder))) {
                     path += `.${propName}`;
                 }
                 if (prop.kind === ReferenceKind.MANY_TO_MANY && typeof childOrder !== 'object') {
@@ -1162,10 +1209,10 @@ export class AbstractSqlDriver extends DatabaseDriver {
                 }
                 const join = qb.getJoinForPath(path, { matchPopulateJoins: true });
                 const propAlias = qb.getAliasForJoinPath(join ?? path, { matchPopulateJoins: true }) ?? parentAlias;
-                if (!join && parentAlias === qb.alias) {
+                if (!join) {
                     continue;
                 }
-                if (![ReferenceKind.SCALAR, ReferenceKind.EMBEDDED].includes(prop.kind) && typeof childOrder === 'object') {
+                if (join && ![ReferenceKind.SCALAR, ReferenceKind.EMBEDDED].includes(prop.kind) && typeof childOrder === 'object') {
                     const children = this.buildPopulateOrderBy(qb, meta2, Utils.asArray(childOrder), path, explicit, propAlias);
                     orderBy.push(...children);
                     continue;
@@ -1243,7 +1290,7 @@ export class AbstractSqlDriver extends DatabaseDriver {
         }
         return ret;
     }
-    processField(meta, prop, field, ret, populate, joinedProps, qb) {
+    processField(meta, prop, field, ret) {
         if (!prop || (prop.kind === ReferenceKind.ONE_TO_ONE && !prop.owner)) {
             return;
         }
@@ -1256,7 +1303,7 @@ export class AbstractSqlDriver extends DatabaseDriver {
             const top = parts.shift();
             for (const key of Object.keys(prop.embeddedProps)) {
                 if (!top || key === top) {
-                    this.processField(meta, prop.embeddedProps[key], parts.join('.'), ret, populate, joinedProps, qb);
+                    this.processField(meta, prop.embeddedProps[key], parts.join('.'), ret);
                 }
             }
             return;
@@ -1266,16 +1313,7 @@ export class AbstractSqlDriver extends DatabaseDriver {
         }
         ret.push(prop.name);
     }
-    isPopulated(meta, prop, hint, name) {
-        if (hint.field === prop.name || hint.field === name || hint.all) {
-            return true;
-        }
-        if (prop.embedded && hint.children && meta.properties[prop.embedded[0]].name === hint.field) {
-            return hint.children.some(c => this.isPopulated(meta, prop, c, prop.embedded[1]));
-        }
-        return false;
-    }
-    buildFields(meta, populate, joinedProps, qb, alias, options, count = false) {
+    buildFields(meta, populate, joinedProps, qb, alias, options) {
         const lazyProps = meta.props.filter(prop => prop.lazy && !populate.some(p => this.isPopulated(meta, prop, p)));
         const hasLazyFormulas = meta.props.some(p => p.lazy && p.formula);
         const requiresSQLConversion = meta.props.some(p => p.customType?.convertToJSValueSQL && p.persist !== false);
@@ -1298,7 +1336,7 @@ export class AbstractSqlDriver extends DatabaseDriver {
                     where: {},
                     aliasMap: qb.getAliasMap(),
                 });
-                this.processField(meta, prop, parts.join('.'), ret, populate, joinedProps, qb);
+                this.processField(meta, prop, parts.join('.'), ret);
             }
             if (!options.fields.includes('*') && !options.fields.includes(`${qb.alias}.*`)) {
                 ret.unshift(...meta.primaryKeys.filter(pk => !options.fields.includes(pk)));
@@ -1308,7 +1346,7 @@ export class AbstractSqlDriver extends DatabaseDriver {
             }
         }
         else if (!Utils.isEmpty(options.exclude) || lazyProps.some(p => !p.formula && (p.kind !== '1:1' || p.owner))) {
-            const props = meta.props.filter(prop => this.platform.shouldHaveColumn(prop, populate, options.exclude, false));
+            const props = meta.props.filter(prop => this.platform.shouldHaveColumn(prop, populate, options.exclude, false, false));
             ret.push(...props.filter(p => !lazyProps.includes(p)).map(p => p.name));
             addFormulas = true;
         }
@@ -1320,16 +1358,19 @@ export class AbstractSqlDriver extends DatabaseDriver {
             ret.push('*');
         }
         if (ret.length > 0 && !hasExplicitFields && addFormulas) {
-            meta.props
-                .filter(prop => prop.formula && !lazyProps.includes(prop))
-                .forEach(prop => {
-                const a = this.platform.quoteIdentifier(alias);
-                const aliased = this.platform.quoteIdentifier(prop.fieldNames[0]);
-                ret.push(raw(`${prop.formula(a)} as ${aliased}`));
-            });
-            meta.props
-                .filter(prop => !prop.object && (prop.hasConvertToDatabaseValueSQL || prop.hasConvertToJSValueSQL))
-                .forEach(prop => ret.push(prop.name));
+            for (const prop of meta.props) {
+                if (lazyProps.includes(prop)) {
+                    continue;
+                }
+                if (prop.formula) {
+                    const a = this.platform.quoteIdentifier(alias);
+                    const aliased = this.platform.quoteIdentifier(prop.fieldNames[0]);
+                    ret.push(raw(`${prop.formula(a)} as ${aliased}`));
+                }
+                if (!prop.object && (prop.hasConvertToDatabaseValueSQL || prop.hasConvertToJSValueSQL)) {
+                    ret.push(prop.name);
+                }
+            }
         }
         // add joined relations after the root entity fields
         if (joinedProps.length > 0) {
@@ -1338,7 +1379,6 @@ export class AbstractSqlDriver extends DatabaseDriver {
                 exclude: options.exclude,
                 populate,
                 parentTableAlias: alias,
-                count,
                 ...options,
             }));
         }