@mikro-orm/knex 7.0.0-dev.7 → 7.0.0-dev.70

package/query/ObjectCriteriaNode.js

@@ -1,4 +1,4 @@
-import { ALIAS_REPLACEMENT, QueryFlag, raw, RawQueryFragment, ReferenceKind, Utils, } from '@mikro-orm/core';
+import { ALIAS_REPLACEMENT, GroupOperator, QueryFlag, raw, RawQueryFragment, ReferenceKind, Utils, } from '@mikro-orm/core';
 import { CriteriaNode } from './CriteriaNode.js';
 import { JoinType, QueryType } from './enums.js';
 /**
@@ -6,7 +6,8 @@ import { JoinType, QueryType } from './enums.js';
  */
 export class ObjectCriteriaNode extends CriteriaNode {
     process(qb, options) {
-        const nestedAlias = qb.getAliasForJoinPath(this.getPath(), options);
+        const matchPopulateJoins = options?.matchPopulateJoins || (this.prop && [ReferenceKind.MANY_TO_ONE, ReferenceKind.ONE_TO_ONE].includes(this.prop.kind));
+        const nestedAlias = qb.getAliasForJoinPath(this.getPath(), { ...options, matchPopulateJoins });
         const ownerAlias = options?.alias || qb.alias;
         const keys = Object.keys(this.payload);
         let alias = options?.alias;
@@ -55,7 +56,23 @@ export class ObjectCriteriaNode extends CriteriaNode {
                 }
                 return { $and };
             }
-            alias = this.autoJoin(qb, ownerAlias);
+            alias = this.autoJoin(qb, ownerAlias, options);
+        }
+        if (this.prop && nestedAlias) {
+            const toOneProperty = [ReferenceKind.MANY_TO_ONE, ReferenceKind.ONE_TO_ONE].includes(this.prop.kind);
+            // if the property is nullable and the filter is strict, we need to use left join, so we mimic the inner join behaviour
+            // with an exclusive condition on the join columns:
+            // - if the owning column is null, the row is missing, we don't apply the filter
+            // - if the target column is not null, the row is matched, we apply the filter
+            if (toOneProperty && this.prop.nullable && this.isStrict()) {
+                const key = this.prop.owner ? this.prop.name : this.prop.referencedPKs;
+                qb.andWhere({
+                    $or: [
+                        { [ownerAlias + '.' + key]: null },
+                        { [nestedAlias + '.' + Utils.getPrimaryKeyHash(this.prop.referencedPKs)]: { $ne: null } },
+                    ],
+                });
+            }
         }
         return keys.reduce((o, field) => {
             const childNode = this.payload[field];
@@ -66,13 +83,14 @@ export class ObjectCriteriaNode extends CriteriaNode {
             const virtual = childNode.prop?.persist === false && !childNode.prop?.formula;
             // if key is missing, we are inside group operator and we need to prefix with alias
             const primaryKey = this.key && this.metadata.find(this.entityName)?.primaryKeys.includes(field);
+            const isToOne = childNode.prop && [ReferenceKind.MANY_TO_ONE, ReferenceKind.ONE_TO_ONE].includes(childNode.prop.kind);
             if (childNode.shouldInline(payload)) {
-                const childAlias = qb.getAliasForJoinPath(childNode.getPath(), options);
+                const childAlias = qb.getAliasForJoinPath(childNode.getPath(), { preferNoBranch: isToOne, ...options });
                 const a = qb.helper.isTableNameAliasRequired(qb.type) ? alias : undefined;
                 this.inlineChildPayload(o, payload, field, a, childAlias);
             }
             else if (childNode.shouldRename(payload)) {
-                this.inlineCondition(childNode.renameFieldToPK(qb), o, payload);
+                this.inlineCondition(childNode.renameFieldToPK(qb, alias), o, payload);
             }
             else if (isRawField) {
                 const rawField = RawQueryFragment.getKnownFragment(field);
@@ -87,6 +105,11 @@ export class ObjectCriteriaNode extends CriteriaNode {
             return o;
         }, {});
     }
+    isStrict() {
+        return this.strict || Object.keys(this.payload).some(key => {
+            return this.payload[key].isStrict();
+        });
+    }
     unwrap() {
         return Object.keys(this.payload).reduce((o, field) => {
             o[field] = this.payload[field].unwrap();
@@ -137,7 +160,7 @@ export class ObjectCriteriaNode extends CriteriaNode {
                 delete payload[k];
                 o[this.aliased(field, alias)] = { [k]: tmp, ...o[this.aliased(field, alias)] };
             }
-            else if (Utils.isGroupOperator(k) && Array.isArray(payload[k])) {
+            else if (k in GroupOperator && Array.isArray(payload[k])) {
                 this.inlineArrayChildPayload(o, payload[k], k, prop, childAlias, alias);
             }
             else if (this.isPrefixed(k) || Utils.isOperator(k) || !childAlias) {
@@ -193,7 +216,7 @@ export class ObjectCriteriaNode extends CriteriaNode {
         });
         return !primaryKeys && !nestedAlias && !operatorKeys && !embeddable;
     }
-    autoJoin(qb, alias) {
+    autoJoin(qb, alias, options) {
         const nestedAlias = qb.getNextAlias(this.prop?.pivotTable ?? this.entityName);
         const customExpression = RawQueryFragment.isKnownFragment(this.key);
         const scalar = Utils.isPrimaryKey(this.payload) || this.payload instanceof RegExp || this.payload instanceof Date || customExpression;
@@ -206,12 +229,18 @@ export class ObjectCriteriaNode extends CriteriaNode {
         }
         else {
             const prev = qb._fields?.slice();
-            qb[method](field, nestedAlias, undefined, JoinType.leftJoin, path);
+            const toOneProperty = [ReferenceKind.MANY_TO_ONE, ReferenceKind.ONE_TO_ONE].includes(this.prop.kind);
+            const joinType = toOneProperty && !this.prop.nullable
+                ? JoinType.innerJoin
+                : JoinType.leftJoin;
+            qb[method](field, nestedAlias, undefined, joinType, path);
             if (!qb.hasFlag(QueryFlag.INFER_POPULATE)) {
                 qb._fields = prev;
             }
         }
-        qb.scheduleFilterCheck(path);
+        if (options?.type !== 'orderBy') {
+            qb.scheduleFilterCheck(path);
+        }
         return nestedAlias;
     }
     isPrefixed(field) {

package/query/QueryBuilder.d.ts

@@ -1,5 +1,5 @@
 import { inspect } from 'node:util';
-import { type AnyEntity, type ConnectionType, type Dictionary, type EntityData, type EntityKey, type EntityManager, type EntityMetadata, type EntityName, type EntityProperty, type ExpandProperty, type FlushMode, type GroupOperator, type Loaded, LockMode, type LoggingOptions, type MetadataStorage, type ObjectQuery, PopulateHint, type PopulateOptions, type QBFilterQuery, type QBQueryOrderMap, QueryFlag, type QueryOrderMap, type QueryResult, RawQueryFragment, type RequiredEntityData, type Transaction } from '@mikro-orm/core';
+import { type AnyEntity, type ConnectionType, type Dictionary, type EntityData, type EntityKey, type EntityManager, type EntityMetadata, type EntityName, type EntityProperty, type ExpandProperty, type FilterOptions, type FlushMode, type GroupOperator, type Loaded, LockMode, type LoggingOptions, type MetadataStorage, type ObjectQuery, PopulateHint, type PopulateOptions, type QBFilterQuery, type QBQueryOrderMap, QueryFlag, type QueryOrderMap, type QueryResult, RawQueryFragment, type RequiredEntityData, type Transaction } from '@mikro-orm/core';
 import { JoinType, QueryType } from './enums.js';
 import type { AbstractSqlDriver } from '../AbstractSqlDriver.js';
 import { type Alias, type OnConflictClause, QueryBuilderHelper } from './QueryBuilderHelper.js';
@@ -11,6 +11,30 @@ export interface ExecuteOptions {
     mapResults?: boolean;
     mergeResults?: boolean;
 }
+export interface QBStreamOptions {
+    /**
+     * Results are mapped to entities, if you set `mapResults: false` you will get POJOs instead.
+     *
+     * @default true
+     */
+    mapResults?: boolean;
+    /**
+     * When populating to-many relations, the ORM streams fully merged entities instead of yielding every row.
+     * You can opt out of this behavior by specifying `mergeResults: false`. This will yield every row from
+     * the SQL result, but still mapped to entities, meaning that to-many collections will contain at most
+     * one item, and you will get duplicate root entities when they have multiple items in the populated
+     * collection.
+     *
+     * @default true
+     */
+    mergeResults?: boolean;
+    /**
+     * When enabled, the driver will return the raw database results without renaming the fields to match the entity property names.
+     *
+     * @default false
+     */
+    rawResults?: boolean;
+}
 type AnyString = string & {};
 type Compute<T> = {
     [K in keyof T]: T[K];
@@ -140,7 +164,7 @@ export declare class QueryBuilder<Entity extends object = AnyEntity, RootAlias e
     /**
      * Apply filters to the QB where condition.
      */
-    applyFilters(filterOptions?: Dictionary<boolean | Dictionary> | string[] | boolean): Promise<void>;
+    applyFilters(filterOptions?: FilterOptions): Promise<void>;
     private readonly autoJoinedPaths;
     /**
      * @internal
@@ -149,7 +173,7 @@ export declare class QueryBuilder<Entity extends object = AnyEntity, RootAlias e
     /**
      * @internal
      */
-    applyJoinedFilters(em: EntityManager, filterOptions?: Dictionary<boolean | Dictionary> | string[] | boolean): Promise<void>;
+    applyJoinedFilters(em: EntityManager, filterOptions: FilterOptions | undefined): Promise<void>;
     withSubQuery(subQuery: RawQueryFragment | NativeQueryBuilder, alias: string): this;
     where(cond: QBFilterQuery<Entity>, operator?: keyof typeof GroupOperator): this;
     where(cond: string, params?: any[], operator?: keyof typeof GroupOperator): this;
@@ -158,6 +182,8 @@ export declare class QueryBuilder<Entity extends object = AnyEntity, RootAlias e
     orWhere(cond: QBFilterQuery<Entity>): this;
     orWhere(cond: string, params?: any[]): this;
     orderBy(orderBy: QBQueryOrderMap<Entity> | QBQueryOrderMap<Entity>[]): SelectQueryBuilder<Entity, RootAlias, Hint, Context>;
+    andOrderBy(orderBy: QBQueryOrderMap<Entity> | QBQueryOrderMap<Entity>[]): SelectQueryBuilder<Entity, RootAlias, Hint, Context>;
+    private processOrderBy;
     groupBy(fields: EntityKeyOrString<Entity> | readonly EntityKeyOrString<Entity>[]): SelectQueryBuilder<Entity, RootAlias, Hint, Context>;
     having(cond?: QBFilterQuery | string, params?: any[], operator?: keyof typeof GroupOperator): SelectQueryBuilder<Entity, RootAlias, Hint, Context>;
     andHaving(cond?: QBFilterQuery | string, params?: any[]): SelectQueryBuilder<Entity, RootAlias, Hint, Context>;
@@ -182,17 +208,17 @@ export declare class QueryBuilder<Entity extends object = AnyEntity, RootAlias e
     /**
      * Adds index hint to the FROM clause.
      */
-    indexHint(sql: string): this;
+    indexHint(sql: string | undefined): this;
     /**
      * Prepend comment to the sql query using the syntax `/* ... *&#8205;/`. Some characters are forbidden such as `/*, *&#8205;/` and `?`.
      */
-    comment(comment: string | string[]): this;
+    comment(comment: string | string[] | undefined): this;
     /**
      * Add hints to the query using comment-like syntax `/*+ ... *&#8205;/`. MySQL and Oracle use this syntax for optimizer hints.
      * Also various DB proxies and routers use this syntax to pass hints to alter their behavior. In other dialects the hints
      * are ignored as simple comments.
      */
-    hintComment(comment: string | string[]): this;
+    hintComment(comment: string | string[] | undefined): this;
     /**
      * Specifies FROM which entity's table select/update/delete will be executed, removing all previously set FROM-s.
      * Allows setting a main string alias of the selection data.
@@ -245,14 +271,35 @@ export declare class QueryBuilder<Entity extends object = AnyEntity, RootAlias e
      * Use `method` to specify what kind of result you want to get (array/single/meta).
      */
     execute<U = any>(method?: 'all' | 'get' | 'run', options?: ExecuteOptions | boolean): Promise<U>;
+    private getConnection;
+    /**
+     * Executes the query and returns an async iterable (async generator) that yields results one by one.
+     * By default, the results are merged and mapped to entity instances, without adding them to the identity map.
+     * You can disable merging and mapping by passing the options `{ mergeResults: false, mapResults: false }`.
+     * This is useful for processing large datasets without loading everything into memory at once.
+     *
+     * ```ts
+     * const qb = em.createQueryBuilder(Book, 'b');
+     * qb.select('*').where({ title: '1984' }).leftJoinAndSelect('b.author', 'a');
+     *
+     * for await (const book of qb.stream()) {
+     *   // book is an instance of Book entity
+     *   console.log(book.title, book.author.name);
+     * }
+     * ```
+     */
+    stream(options?: QBStreamOptions): AsyncIterableIterator<Loaded<Entity, Hint>>;
     /**
      * Alias for `qb.getResultList()`
      */
     getResult(): Promise<Loaded<Entity, Hint>[]>;
     /**
-     * Executes the query, returning array of results
+     * Executes the query, returning array of results mapped to entity instances.
      */
     getResultList(limit?: number): Promise<Loaded<Entity, Hint>[]>;
+    private propagatePopulateHint;
+    private mapResult;
+    private mapResults;
     /**
      * Executes the query, returning the first result or null
      */
@@ -290,6 +337,11 @@ export declare class QueryBuilder<Entity extends object = AnyEntity, RootAlias e
     processPopulateHint(): void;
     private processPopulateWhere;
     private mergeOnConditions;
+    /**
+     * When adding an inner join on a left joined relation, we need to nest them,
+     * otherwise the inner join could discard rows of the root table.
+     */
+    private processNestedJoins;
     private hasToManyJoins;
     protected wrapPaginateSubQuery(meta: EntityMetadata): void;
     private pruneExtraJoins;

package/query/QueryBuilder.js

@@ -179,10 +179,10 @@ export class QueryBuilder {
             subquery = this.platform.formatQuery(rawFragment.sql, rawFragment.params);
             field = field[0];
         }
-        const prop = this.joinReference(field, alias, cond, type, path, schema, subquery);
+        const { prop, key } = this.joinReference(field, alias, cond, type, path, schema, subquery);
         const [fromAlias] = this.helper.splitField(field);
         if (subquery) {
-            this._joins[`${fromAlias}.${prop.name}#${alias}`].subquery = subquery;
+            this._joins[key].subquery = subquery;
         }
         const populate = this._joinedProps.get(fromAlias);
         const item = { field: prop.name, strategy: LoadStrategy.JOINED, children: [] };
@@ -240,9 +240,12 @@ export class QueryBuilder {
             }
         }
         prop.targetMeta.props
-            .filter(prop => explicitFields
-            ? explicitFields.includes(prop.name) || explicitFields.includes(`${alias}.${prop.name}`) || prop.primary
-            : this.platform.shouldHaveColumn(prop, populate))
+            .filter(prop => {
+            if (!explicitFields) {
+                return this.platform.shouldHaveColumn(prop, populate);
+            }
+            return prop.primary && !explicitFields.includes(prop.name) && !explicitFields.includes(`${alias}.${prop.name}`);
+        })
             .forEach(prop => fields.push(...this.driver.mapPropToFieldNames(this, prop, alias)));
         return fields;
     }
@@ -267,16 +270,23 @@ export class QueryBuilder {
     /**
      * @internal
      */
-    async applyJoinedFilters(em, filterOptions = {}) {
+    async applyJoinedFilters(em, filterOptions) {
         for (const path of this.autoJoinedPaths) {
             const join = this.getJoinForPath(path);
             if (join.type === JoinType.pivotJoin) {
                 continue;
             }
+            filterOptions = QueryHelper.mergePropertyFilters(join.prop.filters, filterOptions);
             const cond = await em.applyFilters(join.prop.type, join.cond, filterOptions, 'read');
             if (Utils.hasObjectKeys(cond)) {
+                // remove nested filters, we only care about scalars here, nesting would require another join branch
+                for (const key of Object.keys(cond)) {
+                    if (Utils.isPlainObject(cond[key]) && Object.keys(cond[key]).every(k => !(Utils.isOperator(k) && !['$some', '$none', '$every'].includes(k)))) {
+                        delete cond[key];
+                    }
+                }
                 if (Utils.hasObjectKeys(join.cond)) {
-                    /* istanbul ignore next */
+                    /* v8 ignore next */
                     join.cond = { $and: [join.cond, cond] };
                 }
                 else {
@@ -303,7 +313,7 @@ export class QueryBuilder {
             cond = { [raw(`(${sql})`)]: Utils.asArray(params) };
             operator ??= '$and';
         }
-        else if (Utils.isString(cond)) {
+        else if (typeof cond === 'string') {
             cond = { [raw(`(${cond})`, Utils.asArray(params))]: [] };
             operator ??= '$and';
         }
@@ -350,8 +360,16 @@ export class QueryBuilder {
         return this.where(cond, params, '$or');
     }
     orderBy(orderBy) {
+        return this.processOrderBy(orderBy, true);
+    }
+    andOrderBy(orderBy) {
+        return this.processOrderBy(orderBy, false);
+    }
+    processOrderBy(orderBy, reset = true) {
         this.ensureNotFinalized();
-        this._orderBy = [];
+        if (reset) {
+            this._orderBy = [];
+        }
         Utils.asArray(orderBy).forEach(o => {
             const processed = QueryHelper.processWhere({
                 where: o,
@@ -363,7 +381,7 @@ export class QueryBuilder {
                 convertCustomTypes: false,
                 type: 'orderBy',
             });
-            this._orderBy.push(CriteriaNodeFactory.createNode(this.metadata, this.mainAlias.entityName, processed).process(this, { matchPopulateJoins: true }));
+            this._orderBy.push(CriteriaNodeFactory.createNode(this.metadata, this.mainAlias.entityName, processed).process(this, { matchPopulateJoins: true, type: 'orderBy' }));
         });
         return this;
     }
@@ -374,7 +392,7 @@ export class QueryBuilder {
     }
     having(cond = {}, params, operator) {
         this.ensureNotFinalized();
-        if (Utils.isString(cond)) {
+        if (typeof cond === 'string') {
             cond = { [raw(`(${cond})`, params)]: [] };
         }
         cond = CriteriaNodeFactory.createNode(this.metadata, this.mainAlias.entityName, cond).process(this);
@@ -459,7 +477,7 @@ export class QueryBuilder {
     }
     setLockMode(mode, tables) {
         this.ensureNotFinalized();
-        if (mode != null && mode !== LockMode.OPTIMISTIC && !this.context) {
+        if (mode != null && ![LockMode.OPTIMISTIC, LockMode.NONE].includes(mode) && !this.context) {
             throw ValidationError.transactionRequired();
         }
         this.lockMode = mode;
@@ -553,7 +571,7 @@ export class QueryBuilder {
         Utils.runIfNotEmpty(() => qb.hintComment(this._hintComments), this._hintComments);
         Utils.runIfNotEmpty(() => this.helper.appendOnConflictClause(QueryType.UPSERT, this._onConflict, qb), this._onConflict);
         if (this.lockMode) {
-            this.helper.getLockSQL(qb, this.lockMode, this.lockTables);
+            this.helper.getLockSQL(qb, this.lockMode, this.lockTables, this._joins);
         }
         this.helper.finalize(this.type, qb, this.mainAlias.metadata, this._data, this._returning);
         this.clearRawFragmentsCache();
@@ -667,7 +685,7 @@ export class QueryBuilder {
         options.mapResults ??= true;
         const isRunType = [QueryType.INSERT, QueryType.UPDATE, QueryType.DELETE, QueryType.TRUNCATE].includes(this.type);
         method ??= isRunType ? 'run' : 'all';
-        if (!this.connectionType && isRunType) {
+        if (!this.connectionType && (isRunType || this.context)) {
             this.connectionType = 'write';
         }
         if (!this.finalized && method === 'get' && this.type === QueryType.SELECT) {
@@ -675,13 +693,11 @@ export class QueryBuilder {
         }
         const query = this.toQuery();
         const cached = await this.em?.tryCache(this.mainAlias.entityName, this._cache, ['qb.execute', query.sql, query.params, method]);
-        if (cached?.data) {
+        if (cached?.data !== undefined) {
             return cached.data;
         }
-        const write = method === 'run' || !this.platform.getConfig().get('preferReadReplicas');
-        const type = this.connectionType || (write ? 'write' : 'read');
         const loggerContext = { id: this.em?.id, ...this.loggerContext };
-        const res = await this.driver.getConnection(type).execute(query.sql, query.params, method, this.context, loggerContext);
+        const res = await this.getConnection().execute(query.sql, query.params, method, this.context, loggerContext);
         const meta = this.mainAlias.metadata;
         if (!options.mapResults || !meta) {
             await this.em?.storeCache(this._cache, cached, res);
@@ -709,6 +725,64 @@ export class QueryBuilder {
         await this.em?.storeCache(this._cache, cached, mapped);
         return mapped;
     }
+    getConnection() {
+        const write = !this.platform.getConfig().get('preferReadReplicas');
+        const type = this.connectionType || (write ? 'write' : 'read');
+        return this.driver.getConnection(type);
+    }
+    /**
+     * Executes the query and returns an async iterable (async generator) that yields results one by one.
+     * By default, the results are merged and mapped to entity instances, without adding them to the identity map.
+     * You can disable merging and mapping by passing the options `{ mergeResults: false, mapResults: false }`.
+     * This is useful for processing large datasets without loading everything into memory at once.
+     *
+     * ```ts
+     * const qb = em.createQueryBuilder(Book, 'b');
+     * qb.select('*').where({ title: '1984' }).leftJoinAndSelect('b.author', 'a');
+     *
+     * for await (const book of qb.stream()) {
+     *   // book is an instance of Book entity
+     *   console.log(book.title, book.author.name);
+     * }
+     * ```
+     */
+    async *stream(options) {
+        options ??= {};
+        options.mergeResults ??= true;
+        options.mapResults ??= true;
+        const query = this.toQuery();
+        const loggerContext = { id: this.em?.id, ...this.loggerContext };
+        const res = this.getConnection().stream(query.sql, query.params, this.context, loggerContext);
+        const meta = this.mainAlias.metadata;
+        if (options.rawResults || !meta) {
+            yield* res;
+            return;
+        }
+        const joinedProps = this.driver.joinedProps(meta, this._populate);
+        const stack = [];
+        const hash = (data) => {
+            return Utils.getPrimaryKeyHash(meta.primaryKeys.map(pk => data[pk]));
+        };
+        for await (const row of res) {
+            const mapped = this.driver.mapResult(row, meta, this._populate, this);
+            if (!options.mergeResults || joinedProps.length === 0) {
+                yield this.mapResult(mapped, options.mapResults);
+                continue;
+            }
+            if (stack.length > 0 && hash(stack[stack.length - 1]) !== hash(mapped)) {
+                const res = this.driver.mergeJoinedResult(stack, this.mainAlias.metadata, joinedProps);
+                for (const row of res) {
+                    yield this.mapResult(row, options.mapResults);
+                }
+                stack.length = 0;
+            }
+            stack.push(mapped);
+        }
+        if (stack.length > 0) {
+            const merged = this.driver.mergeJoinedResult(stack, this.mainAlias.metadata, joinedProps);
+            yield this.mapResult(merged[0], options.mapResults);
+        }
+    }
     /**
      * Alias for `qb.getResultList()`
      */
@@ -716,29 +790,40 @@ export class QueryBuilder {
         return this.getResultList();
     }
     /**
-     * Executes the query, returning array of results
+     * Executes the query, returning array of results mapped to entity instances.
      */
     async getResultList(limit) {
         await this.em.tryFlush(this.mainAlias.entityName, { flushMode: this.flushMode });
         const res = await this.execute('all', true);
-        const entities = [];
-        function propagatePopulateHint(entity, hint) {
-            helper(entity).__serializationContext.populate = hint.concat(helper(entity).__serializationContext.populate ?? []);
-            hint.forEach(hint => {
-                const [propName] = hint.field.split(':', 2);
-                const value = Reference.unwrapReference(entity[propName]);
-                if (Utils.isEntity(value)) {
-                    propagatePopulateHint(value, hint.children ?? []);
-                }
-                else if (Utils.isCollection(value)) {
-                    value.populated();
-                    value.getItems(false).forEach(item => propagatePopulateHint(item, hint.children ?? []));
-                }
-            });
+        return this.mapResults(res, limit);
+    }
+    propagatePopulateHint(entity, hint) {
+        helper(entity).__serializationContext.populate = hint.concat(helper(entity).__serializationContext.populate ?? []);
+        hint.forEach(hint => {
+            const [propName] = hint.field.split(':', 2);
+            const value = Reference.unwrapReference(entity[propName]);
+            if (Utils.isEntity(value)) {
+                this.propagatePopulateHint(value, hint.children ?? []);
+            }
+            else if (Utils.isCollection(value)) {
+                value.populated();
+                value.getItems(false).forEach(item => this.propagatePopulateHint(item, hint.children ?? []));
+            }
+        });
+    }
+    mapResult(row, map = true) {
+        if (!map) {
+            return row;
         }
-        for (const r of res) {
-            const entity = this.em.map(this.mainAlias.entityName, r, { schema: this._schema });
-            propagatePopulateHint(entity, this._populate);
+        const entity = this.em.map(this.mainAlias.entityName, row, { schema: this._schema });
+        this.propagatePopulateHint(entity, this._populate);
+        return entity;
+    }
+    mapResults(res, limit) {
+        const entities = [];
+        for (const row of res) {
+            const entity = this.mapResult(row);
+            this.propagatePopulateHint(entity, this._populate);
             entities.push(entity);
             if (limit != null && --limit === 0) {
                 break;
@@ -873,7 +958,8 @@ export class QueryBuilder {
             if (field instanceof RawQueryFragment) {
                 field = this.platform.formatQuery(field.sql, field.params);
             }
-            this._joins[`${this.alias}.${prop.name}#${alias}`] = {
+            const key = `${this.alias}.${prop.name}#${alias}`;
+            this._joins[key] = {
                 prop,
                 alias,
                 type,
@@ -882,7 +968,7 @@ export class QueryBuilder {
                 subquery: field.toString(),
                 ownerAlias: this.alias,
             };
-            return prop;
+            return { prop, key };
         }
         if (!subquery && type.includes('lateral')) {
             throw new Error(`Lateral join can be used only with a sub-query.`);
@@ -907,10 +993,13 @@ export class QueryBuilder {
             aliasMap: this.getAliasMap(),
             aliased: [QueryType.SELECT, QueryType.COUNT].includes(this.type),
         });
+        const criteriaNode = CriteriaNodeFactory.createNode(this.metadata, prop.targetMeta.className, cond);
+        cond = criteriaNode.process(this, { ignoreBranching: true, alias });
         let aliasedName = `${fromAlias}.${prop.name}#${alias}`;
         path ??= `${(Object.values(this._joins).find(j => j.alias === fromAlias)?.path ?? entityName)}.${prop.name}`;
         if (prop.kind === ReferenceKind.ONE_TO_MANY) {
             this._joins[aliasedName] = this.helper.joinOneToReference(prop, fromAlias, alias, type, cond, schema);
+            this._joins[aliasedName].path ??= path;
         }
         else if (prop.kind === ReferenceKind.MANY_TO_MANY) {
             let pivotAlias = alias;
@@ -922,17 +1011,18 @@ export class QueryBuilder {
             const joins = this.helper.joinManyToManyReference(prop, fromAlias, alias, pivotAlias, type, cond, path, schema);
             Object.assign(this._joins, joins);
             this.createAlias(prop.pivotEntity, pivotAlias);
+            this._joins[aliasedName].path ??= path;
+            aliasedName = Object.keys(joins)[1];
         }
         else if (prop.kind === ReferenceKind.ONE_TO_ONE) {
             this._joins[aliasedName] = this.helper.joinOneToReference(prop, fromAlias, alias, type, cond, schema);
+            this._joins[aliasedName].path ??= path;
         }
         else { // MANY_TO_ONE
             this._joins[aliasedName] = this.helper.joinManyToOneReference(prop, fromAlias, alias, type, cond, schema);
+            this._joins[aliasedName].path ??= path;
         }
-        if (!this._joins[aliasedName].path && path) {
-            this._joins[aliasedName].path = path;
-        }
-        return prop;
+        return { prop, key: aliasedName };
     }
     prepareFields(fields, type = 'where') {
         const ret = [];
@@ -945,7 +1035,7 @@ export class QueryBuilder {
                 ret.push(rawField);
                 return;
             }
-            if (!Utils.isString(field)) {
+            if (typeof field !== 'string') {
                 ret.push(field);
                 return;
             }
@@ -1107,6 +1197,7 @@ export class QueryBuilder {
         const meta = this.mainAlias.metadata;
         this.applyDiscriminatorCondition();
         this.processPopulateHint();
+        this.processNestedJoins();
         if (meta && (this._fields?.includes('*') || this._fields?.includes(`${this.mainAlias.aliasName}.*`))) {
             meta.props
                 .filter(prop => prop.formula && (!prop.lazy || this.flags.has(QueryFlag.INCLUDE_LAZY_FORMULAS)))
@@ -1130,7 +1221,7 @@ export class QueryBuilder {
         if (!this.flags.has(QueryFlag.DISABLE_PAGINATE) && this._groupBy.length === 0 && this.hasToManyJoins()) {
             this.flags.add(QueryFlag.PAGINATE);
         }
-        if (meta && this.flags.has(QueryFlag.PAGINATE) && (this._limit > 0 || this._offset > 0)) {
+        if (meta && this.flags.has(QueryFlag.PAGINATE) && !this.flags.has(QueryFlag.DISABLE_PAGINATE) && (this._limit > 0 || this._offset > 0)) {
             this.wrapPaginateSubQuery(meta);
         }
         if (meta && (this.flags.has(QueryFlag.UPDATE_SUB_QUERY) || this.flags.has(QueryFlag.DELETE_SUB_QUERY))) {
@@ -1166,6 +1257,7 @@ export class QueryBuilder {
                 this._joins[aliasedName] = this.helper.joinOneToReference(prop, this.mainAlias.aliasName, alias, JoinType.leftJoin);
                 this._joins[aliasedName].path = `${(Object.values(this._joins).find(j => j.alias === fromAlias)?.path ?? meta.className)}.${prop.name}`;
                 this._populateMap[aliasedName] = this._joins[aliasedName].alias;
+                this.createAlias(prop.type, alias);
             }
         });
         this.processPopulateWhere(false);
@@ -1185,7 +1277,7 @@ export class QueryBuilder {
         if (typeof this[key] === 'object') {
             const cond = CriteriaNodeFactory
                 .createNode(this.metadata, this.mainAlias.entityName, this[key])
-                .process(this, { matchPopulateJoins: true, ignoreBranching: true, preferNoBranch: true });
+                .process(this, { matchPopulateJoins: true, ignoreBranching: true, preferNoBranch: true, filter });
             // there might be new joins created by processing the `populateWhere` object
             joins = Object.values(this._joins);
             this.mergeOnConditions(joins, cond, filter);
@@ -1226,10 +1318,42 @@ export class QueryBuilder {
             }
         }
     }
+    /**
+     * When adding an inner join on a left joined relation, we need to nest them,
+     * otherwise the inner join could discard rows of the root table.
+     */
+    processNestedJoins() {
+        if (this.flags.has(QueryFlag.DISABLE_NESTED_INNER_JOIN)) {
+            return;
+        }
+        const joins = Object.values(this._joins);
+        const lookupParentGroup = (j) => {
+            return j.nested ?? (j.parent ? lookupParentGroup(j.parent) : undefined);
+        };
+        for (const join of joins) {
+            if (join.type === JoinType.innerJoin) {
+                join.parent = joins.find(j => j.alias === join.ownerAlias);
+                // https://stackoverflow.com/a/56815807/3665878
+                if (join.parent?.type === JoinType.leftJoin || join.parent?.type === JoinType.nestedLeftJoin) {
+                    const nested = ((join.parent).nested ??= new Set());
+                    join.type = join.type === JoinType.innerJoin
+                        ? JoinType.nestedInnerJoin
+                        : JoinType.nestedLeftJoin;
+                    nested.add(join);
+                }
+                else if (join.parent?.type === JoinType.nestedInnerJoin) {
+                    const group = lookupParentGroup(join.parent);
+                    const nested = group ?? ((join.parent).nested ??= new Set());
+                    join.type = join.type === JoinType.innerJoin
+                        ? JoinType.nestedInnerJoin
+                        : JoinType.nestedLeftJoin;
+                    nested.add(join);
+                }
+            }
+        }
+    }
     hasToManyJoins() {
-        // console.log(this._joins);
         return Object.values(this._joins).some(join => {
-            // console.log(join.prop.name, join.prop.kind, [ReferenceKind.ONE_TO_MANY, ReferenceKind.MANY_TO_MANY].includes(join.prop.kind));
             return [ReferenceKind.ONE_TO_MANY, ReferenceKind.MANY_TO_MANY].includes(join.prop.kind);
         });
     }

package/query/QueryBuilderHelper.d.ts

@@ -49,7 +49,7 @@ export declare class QueryBuilderHelper {
     getQueryOrderFromObject(type: QueryType, orderBy: FlatQueryOrderMap, populate: Dictionary<string>): string[];
     finalize(type: QueryType, qb: NativeQueryBuilder, meta?: EntityMetadata, data?: Dictionary, returning?: Field<any>[]): void;
     splitField<T>(field: EntityKey<T>, greedyAlias?: boolean): [string, EntityKey<T>, string | undefined];
-    getLockSQL(qb: NativeQueryBuilder, lockMode: LockMode, lockTables?: string[]): void;
+    getLockSQL(qb: NativeQueryBuilder, lockMode: LockMode, lockTables?: string[], joinsMap?: Dictionary<JoinOptions>): void;
     updateVersionProperty(qb: NativeQueryBuilder, data: Dictionary): void;
     private prefix;
     private appendGroupCondition;