@mikro-orm/knex 6.0.0-dev.32 → 6.0.0-dev.320

package/query/QueryBuilder.js

@@ -26,6 +26,12 @@ const CriteriaNodeFactory_1 = require("./CriteriaNodeFactory");
  * ```
  */
 class QueryBuilder {
+    metadata;
+    driver;
+    context;
+    connectionType;
+    em;
+    loggerContext;
     get mainAlias() {
         this.ensureFromClause();
         return this._mainAlias;
@@ -37,31 +43,58 @@ class QueryBuilder {
         this.ensureFromClause();
         return this._helper;
     }
+    /** @internal */
+    type;
+    /** @internal */
+    _fields;
+    /** @internal */
+    _populate = [];
+    /** @internal */
+    _populateWhere;
+    /** @internal */
+    __populateWhere;
+    /** @internal */
+    _populateMap = {};
+    aliasCounter = 0;
+    flags = new Set([core_1.QueryFlag.CONVERT_CUSTOM_TYPES]);
+    finalized = false;
+    _joins = {};
+    _explicitAlias = false;
+    _schema;
+    _cond = {};
+    _data;
+    _orderBy = [];
+    _groupBy = [];
+    _having = {};
+    _returning;
+    _onConflict;
+    _limit;
+    _offset;
+    _distinctOn;
+    _joinedProps = new Map();
+    _cache;
+    _indexHint;
+    _comments = [];
+    _hintComments = [];
+    flushMode;
+    lockMode;
+    lockTables;
+    subQueries = {};
+    _mainAlias;
+    _aliases = {};
+    _helper;
+    platform;
+    knex;
     /**
      * @internal
      */
-    constructor(entityName, metadata, driver, context, alias, connectionType, em) {
+    constructor(entityName, metadata, driver, context, alias, connectionType, em, loggerContext) {
         this.metadata = metadata;
         this.driver = driver;
         this.context = context;
         this.connectionType = connectionType;
         this.em = em;
-        /** @internal */
-        this._populate = [];
-        /** @internal */
-        this._populateMap = {};
-        this.aliasCounter = 0;
-        this.flags = new Set([core_1.QueryFlag.CONVERT_CUSTOM_TYPES]);
-        this.finalized = false;
-        this._joins = {};
-        this._explicitAlias = false;
-        this._cond = {};
-        this._orderBy = [];
-        this._groupBy = [];
-        this._having = {};
-        this._joinedProps = new Map();
-        this.subQueries = {};
-        this._aliases = {};
+        this.loggerContext = loggerContext;
         this.platform = this.driver.getPlatform();
         this.knex = this.driver.getConnection(this.connectionType).getKnex();
         if (alias) {
@@ -116,27 +149,45 @@ class QueryBuilder {
             this._fields = this.mainAlias.metadata.primaryKeys;
         }
         else {
-            this._fields = [this.raw('*')];
+            this._fields = [(0, core_1.raw)('*')];
         }
         if (distinct) {
             this.flags.add(core_1.QueryFlag.DISTINCT);
         }
         return this.init(enums_1.QueryType.COUNT);
     }
-    join(field, alias, cond = {}, type = 'innerJoin', path) {
-        this.joinReference(field, alias, cond, type, path);
+    join(field, alias, cond = {}, type = enums_1.JoinType.innerJoin, path, schema) {
+        this.joinReference(field, alias, cond, type, path, schema);
+        return this;
+    }
+    innerJoin(field, alias, cond = {}, schema) {
+        this.join(field, alias, cond, enums_1.JoinType.innerJoin, undefined, schema);
         return this;
     }
-    leftJoin(field, alias, cond = {}) {
-        return this.join(field, alias, cond, 'leftJoin');
+    innerJoinLateral(field, alias, cond = {}, schema) {
+        this.join(field, alias, cond, enums_1.JoinType.innerJoinLateral, undefined, schema);
+        return this;
     }
-    joinAndSelect(field, alias, cond = {}, type = 'innerJoin', path) {
+    leftJoin(field, alias, cond = {}, schema) {
+        return this.join(field, alias, cond, enums_1.JoinType.leftJoin, undefined, schema);
+    }
+    leftJoinLateral(field, alias, cond = {}, schema) {
+        return this.join(field, alias, cond, enums_1.JoinType.leftJoinLateral, undefined, schema);
+    }
+    joinAndSelect(field, alias, cond = {}, type = enums_1.JoinType.innerJoin, path, fields, schema) {
         if (!this.type) {
             this.select('*');
         }
-        const prop = this.joinReference(field, alias, cond, type, path);
-        this.addSelect(this.getFieldsForJoinedLoad(prop, alias));
+        let subquery;
+        if (Array.isArray(field)) {
+            subquery = field[1] instanceof QueryBuilder ? field[1].getFormattedQuery() : field[1].toString();
+            field = field[0];
+        }
+        const prop = this.joinReference(field, alias, cond, type, path, schema, subquery);
         const [fromAlias] = this.helper.splitField(field);
+        if (subquery) {
+            this._joins[`${fromAlias}.${prop.name}#${alias}`].subquery = subquery;
+        }
         const populate = this._joinedProps.get(fromAlias);
         const item = { field: prop.name, strategy: core_1.LoadStrategy.JOINED, children: [] };
         if (populate) {
@@ -146,15 +197,39 @@ class QueryBuilder {
             this._populate.push(item);
         }
         this._joinedProps.set(alias, item);
+        this.addSelect(this.getFieldsForJoinedLoad(prop, alias, fields));
         return this;
     }
-    leftJoinAndSelect(field, alias, cond = {}) {
-        return this.joinAndSelect(field, alias, cond, 'leftJoin');
+    leftJoinAndSelect(field, alias, cond = {}, fields, schema) {
+        return this.joinAndSelect(field, alias, cond, enums_1.JoinType.leftJoin, undefined, fields, schema);
+    }
+    leftJoinLateralAndSelect(field, alias, cond = {}, fields, schema) {
+        return this.joinAndSelect(field, alias, cond, enums_1.JoinType.leftJoinLateral, undefined, fields, schema);
+    }
+    innerJoinAndSelect(field, alias, cond = {}, fields, schema) {
+        return this.joinAndSelect(field, alias, cond, enums_1.JoinType.innerJoin, undefined, fields, schema);
+    }
+    innerJoinLateralAndSelect(field, alias, cond = {}, fields, schema) {
+        return this.joinAndSelect(field, alias, cond, enums_1.JoinType.innerJoinLateral, undefined, fields, schema);
     }
-    getFieldsForJoinedLoad(prop, alias) {
+    getFieldsForJoinedLoad(prop, alias, explicitFields) {
         const fields = [];
+        const populate = [];
+        const joinKey = Object.keys(this._joins).find(join => join.endsWith(`#${alias}`));
+        if (joinKey) {
+            const path = this._joins[joinKey].path.split('.').slice(1);
+            let children = this._populate;
+            for (let i = 0; i < path.length; i++) {
+                const child = children.filter(hint => {
+                    const [propName] = hint.field.split(':', 2);
+                    return propName === path[i];
+                });
+                children = child.flatMap(c => c.children);
+            }
+            populate.push(...children);
+        }
         prop.targetMeta.props
-            .filter(prop => this.platform.shouldHaveColumn(prop, this._populate))
+            .filter(prop => explicitFields ? explicitFields.includes(prop.name) || prop.primary : this.platform.shouldHaveColumn(prop, populate))
             .forEach(prop => fields.push(...this.driver.mapPropToFieldNames(this, prop, alias)));
         return fields;
     }
@@ -165,9 +240,15 @@ class QueryBuilder {
     }
     where(cond, params, operator) {
         this.ensureNotFinalized();
-        if (core_1.Utils.isString(cond)) {
-            cond = { [`(${cond})`]: core_1.Utils.asArray(params) };
-            operator = operator || '$and';
+        const rawField = core_1.RawQueryFragment.getKnownFragment(cond);
+        if (rawField) {
+            const sql = this.platform.formatQuery(rawField.sql, rawField.params);
+            cond = { [(0, core_1.raw)(`(${sql})`)]: core_1.Utils.asArray(params) };
+            operator ??= '$and';
+        }
+        else if (core_1.Utils.isString(cond)) {
+            cond = { [(0, core_1.raw)(`(${cond})`, core_1.Utils.asArray(params))]: [] };
+            operator ??= '$and';
         }
         else {
             cond = core_1.QueryHelper.processWhere({
@@ -183,19 +264,20 @@ class QueryBuilder {
         const op = operator || params;
         const topLevel = !op || !core_1.Utils.hasObjectKeys(this._cond);
         const criteriaNode = CriteriaNodeFactory_1.CriteriaNodeFactory.createNode(this.metadata, this.mainAlias.entityName, cond);
+        const ignoreBranching = this.__populateWhere === 'infer';
         if ([enums_1.QueryType.UPDATE, enums_1.QueryType.DELETE].includes(this.type) && criteriaNode.willAutoJoin(this)) {
             // use sub-query to support joining
             this.setFlag(this.type === enums_1.QueryType.UPDATE ? core_1.QueryFlag.UPDATE_SUB_QUERY : core_1.QueryFlag.DELETE_SUB_QUERY);
             this.select(this.mainAlias.metadata.primaryKeys, true);
         }
         if (topLevel) {
-            this._cond = criteriaNode.process(this);
+            this._cond = criteriaNode.process(this, { ignoreBranching });
         }
         else if (Array.isArray(this._cond[op])) {
-            this._cond[op].push(criteriaNode.process(this));
+            this._cond[op].push(criteriaNode.process(this, { ignoreBranching }));
         }
         else {
-            const cond1 = [this._cond, criteriaNode.process(this)];
+            const cond1 = [this._cond, criteriaNode.process(this, { ignoreBranching })];
             this._cond = { [op]: cond1 };
         }
         if (this._onConflict) {
@@ -222,8 +304,9 @@ class QueryBuilder {
                 aliasMap: this.getAliasMap(),
                 aliased: !this.type || [enums_1.QueryType.SELECT, enums_1.QueryType.COUNT].includes(this.type),
                 convertCustomTypes: false,
+                type: 'orderBy',
             });
-            this._orderBy.push(CriteriaNodeFactory_1.CriteriaNodeFactory.createNode(this.metadata, this.mainAlias.entityName, processed).process(this));
+            this._orderBy.push(CriteriaNodeFactory_1.CriteriaNodeFactory.createNode(this.metadata, this.mainAlias.entityName, processed).process(this, { matchPopulateJoins: true }));
         });
         return this;
     }
@@ -235,15 +318,22 @@ class QueryBuilder {
     having(cond = {}, params) {
         this.ensureNotFinalized();
         if (core_1.Utils.isString(cond)) {
-            cond = { [`(${cond})`]: core_1.Utils.asArray(params) };
+            cond = { [(0, core_1.raw)(`(${cond})`, params)]: [] };
         }
         this._having = CriteriaNodeFactory_1.CriteriaNodeFactory.createNode(this.metadata, this.mainAlias.entityName, cond).process(this);
         return this;
     }
     onConflict(fields = []) {
+        const meta = this.mainAlias.metadata;
         this.ensureNotFinalized();
-        this._onConflict = this._onConflict || [];
-        this._onConflict.push({ fields: core_1.Utils.asArray(fields).map(f => f.toString()) });
+        this._onConflict ??= [];
+        this._onConflict.push({
+            fields: core_1.Utils.asArray(fields).flatMap(f => {
+                const key = f.toString();
+                /* istanbul ignore next */
+                return meta.properties[key]?.fieldNames ?? [key];
+            }),
+        });
         return this;
     }
     ignore() {
@@ -263,6 +353,10 @@ class QueryBuilder {
         this._onConflict[this._onConflict.length - 1].merge = data;
         return this;
     }
+    returning(fields) {
+        this._returning = core_1.Utils.asArray(fields);
+        return this;
+    }
     /**
      * @internal
      */
@@ -272,17 +366,6 @@ class QueryBuilder {
         this._populateWhere = populateWhere;
         return this;
     }
-    /**
-     * @internal
-     */
-    ref(field) {
-        return this.knex.ref(field);
-    }
-    raw(sql, bindings = []) {
-        const raw = this.knex.raw(sql, bindings);
-        raw.__raw = true; // tag it as there is now way to check via `instanceof`
-        return raw;
-    }
     limit(limit, offset = 0) {
         this.ensureNotFinalized();
         this._limit = limit;
@@ -325,6 +408,9 @@ class QueryBuilder {
         this.flags.delete(flag);
         return this;
     }
+    hasFlag(flag) {
+        return this.flags.has(flag);
+    }
     cache(config = true) {
         this.ensureNotFinalized();
         this._cache = config;
@@ -338,6 +424,24 @@ class QueryBuilder {
         this._indexHint = sql;
         return this;
     }
+    /**
+     * Prepend comment to the sql query using the syntax `/* ... *&#8205;/`. Some characters are forbidden such as `/*, *&#8205;/` and `?`.
+     */
+    comment(comment) {
+        this.ensureNotFinalized();
+        this._comments.push(...core_1.Utils.asArray(comment));
+        return this;
+    }
+    /**
+     * Add hints to the query using comment-like syntax `/*+ ... *&#8205;/`. MySQL and Oracle use this syntax for optimizer hints.
+     * Also various DB proxies and routers use this syntax to pass hints to alter their behavior. In other dialects the hints
+     * are ignored as simple comments.
+     */
+    hintComment(comment) {
+        this.ensureNotFinalized();
+        this._hintComments.push(...core_1.Utils.asArray(comment));
+        return this;
+    }
     from(target, aliasName) {
         this.ensureNotFinalized();
         if (target instanceof QueryBuilder) {
@@ -352,63 +456,108 @@ class QueryBuilder {
         }
         return this;
     }
-    getKnexQuery() {
+    getKnexQuery(processVirtualEntity = true) {
         this.finalize();
-        const qb = this.getQueryBase();
-        core_1.Utils.runIfNotEmpty(() => this.helper.appendQueryCondition(this.type ?? enums_1.QueryType.SELECT, this._cond, qb), this._cond && !this._onConflict);
+        const qb = this.getQueryBase(processVirtualEntity);
+        const type = this.type ?? enums_1.QueryType.SELECT;
+        qb.__raw = true; // tag it as there is now way to check via `instanceof`
+        core_1.Utils.runIfNotEmpty(() => this.helper.appendQueryCondition(type, this._cond, qb), this._cond && !this._onConflict);
         core_1.Utils.runIfNotEmpty(() => qb.groupBy(this.prepareFields(this._groupBy, 'groupBy')), this._groupBy);
-        core_1.Utils.runIfNotEmpty(() => this.helper.appendQueryCondition(this.type ?? enums_1.QueryType.SELECT, this._having, qb, undefined, 'having'), this._having);
+        core_1.Utils.runIfNotEmpty(() => this.helper.appendQueryCondition(type, this._having, qb, undefined, 'having'), this._having);
         core_1.Utils.runIfNotEmpty(() => {
-            const queryOrder = this.helper.getQueryOrder(this.type ?? enums_1.QueryType.SELECT, this._orderBy, this._populateMap);
-            if (queryOrder) {
-                qb.orderByRaw(queryOrder);
+            const queryOrder = this.helper.getQueryOrder(type, this._orderBy, this._populateMap);
+            if (queryOrder.length > 0) {
+                const sql = core_1.Utils.unique(queryOrder).join(', ');
+                qb.orderByRaw(sql);
                 return;
             }
         }, this._orderBy);
         core_1.Utils.runIfNotEmpty(() => qb.limit(this._limit), this._limit != null);
         core_1.Utils.runIfNotEmpty(() => qb.offset(this._offset), this._offset);
-        core_1.Utils.runIfNotEmpty(() => this.helper.appendOnConflictClause(this.type ?? enums_1.QueryType.SELECT, this._onConflict, qb), this._onConflict);
+        core_1.Utils.runIfNotEmpty(() => this._comments.forEach(comment => qb.comment(comment)), this._comments);
+        core_1.Utils.runIfNotEmpty(() => this._hintComments.forEach(comment => qb.hintComment(comment)), this._hintComments);
+        core_1.Utils.runIfNotEmpty(() => this.helper.appendOnConflictClause(type, this._onConflict, qb), this._onConflict);
         if (this.type === enums_1.QueryType.TRUNCATE && this.platform.usesCascadeStatement()) {
             return this.knex.raw(qb.toSQL().toNative().sql + ' cascade');
         }
         if (this.lockMode) {
             this.helper.getLockSQL(qb, this.lockMode, this.lockTables);
         }
-        this.helper.finalize(this.type ?? enums_1.QueryType.SELECT, qb, this.mainAlias.metadata);
+        this.helper.finalize(type, qb, this.mainAlias.metadata, this._data, this._returning);
         return qb;
     }
     /**
      * Returns the query with parameters as wildcards.
      */
     getQuery() {
-        return this.getKnexQuery().toSQL().toNative().sql;
+        return this.toQuery().sql;
+    }
+    #query;
+    toQuery() {
+        if (this.#query) {
+            return this.#query;
+        }
+        const sql = this.getKnexQuery().toSQL();
+        const query = sql.toNative();
+        return this.#query = { sql: query.sql, params: query.bindings ?? [], _sql: sql };
     }
     /**
      * Returns the list of all parameters for this query.
      */
     getParams() {
-        return this.getKnexQuery().toSQL().toNative().bindings;
+        return this.toQuery().params;
     }
     /**
      * Returns raw interpolated query string with all the parameters inlined.
      */
     getFormattedQuery() {
-        const query = this.getKnexQuery().toSQL();
+        const query = this.toQuery()._sql;
         return this.platform.formatQuery(query.sql, query.bindings);
     }
     /**
      * @internal
      */
-    getAliasForJoinPath(path) {
+    getAliasForJoinPath(path, options) {
         if (!path || path === this.mainAlias.entityName) {
             return this.mainAlias.aliasName;
         }
-        const join = Object.values(this._joins).find(j => j.path === path);
-        if (path.endsWith('[pivot]') && join) {
+        const join = typeof path === 'string' ? this.getJoinForPath(path, options) : path;
+        if (join?.path?.endsWith('[pivot]')) {
             return join.alias;
         }
         return join?.inverseAlias || join?.alias;
     }
+    /**
+     * @internal
+     */
+    getJoinForPath(path, options) {
+        const joins = Object.values(this._joins);
+        if (joins.length === 0) {
+            return undefined;
+        }
+        let join = joins.find(j => j.path === path);
+        if (options?.preferNoBranch) {
+            join = joins.find(j => {
+                return j.path?.replace(/\[\d+]|\[populate]/g, '') === path.replace(/\[\d+]|\[populate]/g, '');
+            });
+        }
+        if (!join && options?.ignoreBranching) {
+            join = joins.find(j => {
+                return j.path?.replace(/\[\d+]/g, '') === path.replace(/\[\d+]/g, '');
+            });
+        }
+        if (!join && options?.matchPopulateJoins && options?.ignoreBranching) {
+            join = joins.find(j => {
+                return j.path?.replace(/\[\d+]|\[populate]/g, '') === path.replace(/\[\d+]|\[populate]/g, '');
+            });
+        }
+        if (!join && options?.matchPopulateJoins) {
+            join = joins.find(j => {
+                return j.path?.replace(/\[populate]/g, '') === path.replace(/\[populate]/g, '');
+            });
+        }
+        return join;
+    }
     /**
      * @internal
      */
@@ -425,7 +574,10 @@ class QueryBuilder {
      * Executes this QB and returns the raw results, mapped to the property names (unless disabled via last parameter).
      * Use `method` to specify what kind of result you want to get (array/single/meta).
      */
-    async execute(method = 'all', mapResults = true) {
+    async execute(method = 'all', options) {
+        options = typeof options === 'boolean' ? { mapResults: options } : (options ?? {});
+        options.mergeResults ??= true;
+        options.mapResults ??= true;
         if (!this.connectionType && method !== 'run' && [enums_1.QueryType.INSERT, enums_1.QueryType.UPDATE, enums_1.QueryType.DELETE, enums_1.QueryType.TRUNCATE].includes(this.type ?? enums_1.QueryType.SELECT)) {
             this.connectionType = 'write';
         }
@@ -434,21 +586,33 @@ class QueryBuilder {
         if (cached?.data) {
             return cached.data;
         }
-        const type = this.connectionType || (method === 'run' ? 'write' : 'read');
-        const res = await this.driver.getConnection(type).execute(query.sql, query.bindings, method, this.context);
+        const write = method === 'run' || !this.platform.getConfig().get('preferReadReplicas');
+        const type = this.connectionType || (write ? 'write' : 'read');
+        const res = await this.driver.getConnection(type).execute(query.sql, query.bindings, method, this.context, this.loggerContext);
         const meta = this.mainAlias.metadata;
-        if (!mapResults || !meta) {
+        if (!options.mapResults || !meta) {
             await this.em?.storeCache(this._cache, cached, res);
             return res;
         }
-        if (method === 'all' && Array.isArray(res)) {
+        if (method === 'run') {
+            return res;
+        }
+        const joinedProps = this.driver.joinedProps(meta, this._populate);
+        let mapped;
+        if (Array.isArray(res)) {
             const map = {};
-            const mapped = res.map(r => this.driver.mapResult(r, meta, this._populate, this, map));
-            await this.em?.storeCache(this._cache, cached, mapped);
-            return mapped;
+            mapped = res.map(r => this.driver.mapResult(r, meta, this._populate, this, map));
+            if (options.mergeResults && joinedProps.length > 0) {
+                mapped = this.driver.mergeJoinedResult(mapped, this.mainAlias.metadata, joinedProps);
+            }
+        }
+        else {
+            mapped = [this.driver.mapResult(res, meta, joinedProps, this)];
         }
-        const mapped = this.driver.mapResult(res, meta, this._populate, this);
         await this.em?.storeCache(this._cache, cached, mapped);
+        if (method === 'get') {
+            return mapped[0];
+        }
         return mapped;
     }
     /**
@@ -462,11 +626,29 @@ class QueryBuilder {
      */
     async getResultList() {
         await this.em.tryFlush(this.mainAlias.entityName, { flushMode: this.flushMode });
-        let res = await this.execute('all', true);
-        if (this._joinedProps.size > 0) {
-            res = this.driver.mergeJoinedResult(res, this.mainAlias.metadata);
+        const res = await this.execute('all', true);
+        const entities = [];
+        function propagatePopulateHint(entity, hint) {
+            (0, core_1.helper)(entity).__serializationContext.populate ??= hint;
+            hint.forEach(hint => {
+                const [propName] = hint.field.split(':', 2);
+                const value = entity[propName];
+                if (core_1.Utils.isEntity(value, true)) {
+                    (0, core_1.helper)(value).populated();
+                    propagatePopulateHint(value, hint.children ?? []);
+                }
+                else if (core_1.Utils.isCollection(value)) {
+                    value.populated();
+                    value.getItems(false).forEach(item => propagatePopulateHint(item, hint.children ?? []));
+                }
+            });
         }
-        return res.map(r => this.em.map(this.mainAlias.entityName, r, { schema: this._schema }));
+        for (const r of res) {
+            const entity = this.em.map(this.mainAlias.entityName, r, { schema: this._schema });
+            propagatePopulateHint(entity, this._populate);
+            entities.push(entity);
+        }
+        return entities;
     }
     /**
      * Executes the query, returning the first result or null
@@ -484,7 +666,7 @@ class QueryBuilder {
             res = await this.execute('get', false);
         }
         else {
-            const qb = this.clone();
+            const qb = this.type === undefined ? this : this.clone();
             qb.count(field, distinct ?? qb.hasToManyJoins()).limit(undefined).offset(undefined).orderBy([]);
             res = await qb.execute('get', false);
         }
@@ -503,7 +685,11 @@ class QueryBuilder {
      * Provides promise-like interface so we can await the QB instance.
      */
     then(onfulfilled, onrejected) {
-        switch (this.type ?? enums_1.QueryType.SELECT) {
+        let type = this.type ?? enums_1.QueryType.SELECT;
+        if (this.flags.has(core_1.QueryFlag.UPDATE_SUB_QUERY) || this.flags.has(core_1.QueryFlag.DELETE_SUB_QUERY)) {
+            type = enums_1.QueryType.UPDATE;
+        }
+        switch (type) {
             case enums_1.QueryType.INSERT:
             case enums_1.QueryType.UPDATE:
             case enums_1.QueryType.DELETE:
@@ -526,15 +712,22 @@ class QueryBuilder {
             /* istanbul ignore next */
             alias = meta?.properties[f]?.fieldNames[0] ?? alias;
         }
-        return qb.as(alias);
+        const ret = qb.as(alias);
+        // tag the instance, so it is possible to detect it easily
+        Object.defineProperty(ret, '__as', { enumerable: false, value: alias });
+        return ret;
     }
-    clone() {
+    clone(reset) {
         const qb = new QueryBuilder(this.mainAlias.entityName, this.metadata, this.driver, this.context, this.mainAlias.aliasName, this.connectionType, this.em);
+        if (reset) {
+            return qb;
+        }
         Object.assign(qb, this);
         // clone array/object properties
         const properties = [
-            'flags', '_populate', '_populateWhere', '_populateMap', '_joins', '_joinedProps', '_cond', '_data', '_orderBy',
-            '_schema', '_indexHint', '_cache', 'subQueries', 'lockMode', 'lockTables',
+            'flags', '_populate', '_populateWhere', '__populateWhere', '_populateMap', '_joins', '_joinedProps', '_cond', '_data', '_orderBy',
+            '_schema', '_indexHint', '_cache', 'subQueries', 'lockMode', 'lockTables', '_groupBy', '_having', '_returning',
+            '_comments', '_hintComments',
         ];
         properties.forEach(prop => qb[prop] = core_1.Utils.copy(this[prop]));
         /* istanbul ignore else */
@@ -545,9 +738,9 @@ class QueryBuilder {
         qb.finalized = false;
         return qb;
     }
-    getKnex() {
+    getKnex(processVirtualEntity = true) {
         const qb = this.knex.queryBuilder();
-        const { subQuery, aliasName, entityName } = this.mainAlias;
+        const { subQuery, aliasName, entityName, metadata } = this.mainAlias;
         const ref = subQuery ? subQuery : this.knex.ref(this.helper.getTableName(entityName));
         if (this.finalized && (this._explicitAlias || this.helper.isTableNameAliasRequired(this.type))) {
             ref.as(aliasName);
@@ -556,20 +749,72 @@ class QueryBuilder {
         if (schema) {
             ref.withSchema(schema);
         }
-        qb.from(ref);
+        if (metadata?.virtual && processVirtualEntity) {
+            qb.fromRaw(this.fromVirtual(metadata));
+        }
+        else {
+            qb.from(ref);
+        }
         if (this.context) {
             qb.transacting(this.context);
         }
         return qb;
     }
-    joinReference(field, alias, cond, type, path) {
+    fromVirtual(meta) {
+        if (typeof meta.expression === 'string') {
+            return `(${meta.expression}) as ${this.platform.quoteIdentifier(this.alias)}`;
+        }
+        const res = meta.expression(this.em, this._cond, {});
+        if (typeof res === 'string') {
+            return `(${res}) as ${this.platform.quoteIdentifier(this.alias)}`;
+        }
+        if (res instanceof QueryBuilder) {
+            return `(${res.getFormattedQuery()}) as ${this.platform.quoteIdentifier(this.alias)}`;
+        }
+        if (core_1.Utils.isObject(res)) {
+            const { sql, bindings } = res.toSQL();
+            const query = this.platform.formatQuery(sql, bindings);
+            return `(${query}) as ${this.platform.quoteIdentifier(this.alias)}`;
+        }
+        /* istanbul ignore next */
+        return res;
+    }
+    joinReference(field, alias, cond, type, path, schema, subquery) {
         this.ensureNotFinalized();
+        if (typeof field === 'object') {
+            const prop = {
+                name: '__subquery__',
+                kind: core_1.ReferenceKind.MANY_TO_ONE,
+            };
+            if (field instanceof QueryBuilder) {
+                prop.type = field.mainAlias.entityName;
+                prop.targetMeta = field.mainAlias.metadata;
+                field = field.getKnexQuery();
+            }
+            this._joins[`${this.alias}.${prop.name}#${alias}`] = {
+                prop,
+                alias,
+                type,
+                cond,
+                schema,
+                subquery: field.toString(),
+                ownerAlias: this.alias,
+            };
+            return prop;
+        }
+        if (!subquery && type.includes('lateral')) {
+            throw new Error(`Lateral join can be used only with a sub-query.`);
+        }
         const [fromAlias, fromField] = this.helper.splitField(field);
-        const entityName = this._aliases[fromAlias]?.entityName;
+        const q = (str) => `'${str}'`;
+        if (!this._aliases[fromAlias]) {
+            throw new Error(`Trying to join ${q(fromField)} with alias ${q(fromAlias)}, but ${q(fromAlias)} is not a known alias. Available aliases are: ${Object.keys(this._aliases).map(q).join(', ')}.`);
+        }
+        const entityName = this._aliases[fromAlias].entityName;
         const meta = this.metadata.get(entityName);
         const prop = meta.properties[fromField];
         if (!prop) {
-            throw new Error(`Trying to join ${field}, but ${fromField} is not a defined relation on ${meta.className}`);
+            throw new Error(`Trying to join ${q(field)}, but ${q(fromField)} is not a defined relation on ${meta.className}.`);
         }
         this.createAlias(prop.type, alias);
         cond = core_1.QueryHelper.processWhere({
@@ -583,24 +828,24 @@ class QueryBuilder {
         let aliasedName = `${fromAlias}.${prop.name}#${alias}`;
         path ??= `${(Object.values(this._joins).find(j => j.alias === fromAlias)?.path ?? entityName)}.${prop.name}`;
         if (prop.kind === core_1.ReferenceKind.ONE_TO_MANY) {
-            this._joins[aliasedName] = this.helper.joinOneToReference(prop, fromAlias, alias, type, cond);
+            this._joins[aliasedName] = this.helper.joinOneToReference(prop, fromAlias, alias, type, cond, schema);
         }
         else if (prop.kind === core_1.ReferenceKind.MANY_TO_MANY) {
             let pivotAlias = alias;
-            if (type !== 'pivotJoin') {
+            if (type !== enums_1.JoinType.pivotJoin) {
                 const oldPivotAlias = this.getAliasForJoinPath(path + '[pivot]');
                 pivotAlias = oldPivotAlias ?? this.getNextAlias(prop.pivotEntity);
                 aliasedName = `${fromAlias}.${prop.name}#${pivotAlias}`;
             }
-            const joins = this.helper.joinManyToManyReference(prop, fromAlias, alias, pivotAlias, type, cond, path);
+            const joins = this.helper.joinManyToManyReference(prop, fromAlias, alias, pivotAlias, type, cond, path, schema);
             Object.assign(this._joins, joins);
             this.createAlias(prop.pivotEntity, pivotAlias);
         }
         else if (prop.kind === core_1.ReferenceKind.ONE_TO_ONE) {
-            this._joins[aliasedName] = this.helper.joinOneToReference(prop, fromAlias, alias, type, cond);
+            this._joins[aliasedName] = this.helper.joinOneToReference(prop, fromAlias, alias, type, cond, schema);
         }
         else { // MANY_TO_ONE
-            this._joins[aliasedName] = this.helper.joinManyToOneReference(prop, fromAlias, alias, type, cond);
+            this._joins[aliasedName] = this.helper.joinManyToOneReference(prop, fromAlias, alias, type, cond, schema);
         }
         if (!this._joins[aliasedName].path && path) {
             this._joins[aliasedName].path = path;
@@ -609,7 +854,19 @@ class QueryBuilder {
     }
     prepareFields(fields, type = 'where') {
         const ret = [];
+        const getFieldName = (name) => {
+            if (type === 'groupBy') {
+                return this.helper.mapper(name, this.type, undefined, null);
+            }
+            return this.helper.mapper(name, this.type);
+        };
         fields.forEach(field => {
+            const rawField = core_1.RawQueryFragment.getKnownFragment(field);
+            if (rawField) {
+                const sql = this.platform.formatQuery(rawField.sql, rawField.params);
+                ret.push(this.knex.raw(sql));
+                return;
+            }
             if (!core_1.Utils.isString(field)) {
                 ret.push(field);
                 return;
@@ -626,19 +883,20 @@ class QueryBuilder {
                 return;
             }
             if (prop?.embedded) {
-                const fieldName = this.helper.mapper(prop.fieldNames[0], this.type);
-                ret.push(fieldName);
+                const name = prop.embeddedPath?.join('.') ?? prop.fieldNames[0];
+                const aliased = this._aliases[a] ? `${a}.${name}` : name;
+                ret.push(getFieldName(aliased));
                 return;
             }
             if (prop?.kind === core_1.ReferenceKind.EMBEDDED) {
                 if (prop.object) {
-                    ret.push(this.helper.mapper(prop.fieldNames[0], this.type));
+                    ret.push(getFieldName(prop.fieldNames[0]));
                 }
                 else {
                     const nest = (prop) => {
                         for (const childProp of Object.values(prop.embeddedProps)) {
-                            if (childProp.fieldNames) {
-                                ret.push(this.helper.mapper(childProp.fieldNames[0], this.type));
+                            if (childProp.fieldNames && (childProp.kind !== core_1.ReferenceKind.EMBEDDED || childProp.object) && childProp.persist !== false) {
+                                ret.push(getFieldName(childProp.fieldNames[0]));
                             }
                             else {
                                 nest(childProp);
@@ -649,11 +907,15 @@ class QueryBuilder {
                 }
                 return;
             }
-            ret.push(this.helper.mapper(field, this.type));
+            if (prop && prop.fieldNames.length > 1) {
+                ret.push(...prop.fieldNames.map(f => getFieldName(f)));
+                return;
+            }
+            ret.push(getFieldName(field));
         });
         const meta = this.mainAlias.metadata;
         /* istanbul ignore next */
-        const requiresSQLConversion = meta?.props.filter(p => p.customType?.convertToJSValueSQL) ?? [];
+        const requiresSQLConversion = meta?.props.filter(p => p.hasConvertToJSValueSQL && p.persist !== false) ?? [];
         if (this.flags.has(core_1.QueryFlag.CONVERT_CUSTOM_TYPES) && (fields.includes('*') || fields.includes(`${this.mainAlias.aliasName}.*`)) && requiresSQLConversion.length > 0) {
             requiresSQLConversion.forEach(p => ret.push(this.helper.mapper(p.name, this.type)));
         }
@@ -662,11 +924,6 @@ class QueryBuilder {
                 const cols = this.helper.mapJoinColumns(this.type ?? enums_1.QueryType.SELECT, this._joins[f]);
                 ret.push(...cols);
             }
-            if (this._joins[f].prop.kind !== core_1.ReferenceKind.ONE_TO_ONE && this._joins[f].inverseJoinColumns) {
-                this._joins[f].inverseJoinColumns.forEach(inverseJoinColumn => {
-                    core_1.Utils.renameKey(this._cond, inverseJoinColumn, `${this._joins[f].alias}.${inverseJoinColumn}`);
-                });
-            }
         });
         return ret;
     }
@@ -681,19 +938,20 @@ class QueryBuilder {
         }
         if (data) {
             if (core_1.Utils.isEntity(data)) {
-                // data = helper(data).toJSON();
                 data = this.em?.getComparator().prepareEntity(data) ?? (0, core_1.serialize)(data);
             }
-            this._data = this.helper.processData(data, this.flags.has(core_1.QueryFlag.CONVERT_CUSTOM_TYPES));
+            this._data = this.helper.processData(data, this.flags.has(core_1.QueryFlag.CONVERT_CUSTOM_TYPES), false);
         }
         if (cond) {
             this.where(cond);
         }
         return this;
     }
-    getQueryBase() {
-        const qb = this.getKnex();
+    getQueryBase(processVirtualEntity) {
+        const qb = this.getKnex(processVirtualEntity);
         const schema = this.getSchema(this.mainAlias);
+        // Joined tables doesn't need to belong to the same schema as the main table
+        const joinSchema = this._schema ?? this.em?.schema ?? schema;
         if (schema) {
             qb.withSchema(schema);
         }
@@ -712,12 +970,12 @@ class QueryBuilder {
                 else if (this.flags.has(core_1.QueryFlag.DISTINCT)) {
                     qb.distinct();
                 }
-                this.helper.processJoins(qb, this._joins, schema);
+                this.helper.processJoins(qb, this._joins, joinSchema);
                 break;
             case enums_1.QueryType.COUNT: {
                 const m = this.flags.has(core_1.QueryFlag.DISTINCT) ? 'countDistinct' : 'count';
                 qb[m]({ count: this._fields.map(f => this.helper.mapper(f, this.type)) });
-                this.helper.processJoins(qb, this._joins, schema);
+                this.helper.processJoins(qb, this._joins, joinSchema);
                 break;
             }
             case enums_1.QueryType.INSERT:
@@ -736,6 +994,24 @@ class QueryBuilder {
         }
         return qb;
     }
+    applyDiscriminatorCondition() {
+        const meta = this.mainAlias.metadata;
+        if (!meta?.discriminatorValue) {
+            return;
+        }
+        const types = Object.values(meta.root.discriminatorMap).map(cls => this.metadata.find(cls));
+        const children = [];
+        const lookUpChildren = (ret, type) => {
+            const children = types.filter(meta2 => meta2.extends === type);
+            children.forEach(m => lookUpChildren(ret, m.className));
+            ret.push(...children.filter(c => c.discriminatorValue));
+            return children;
+        };
+        lookUpChildren(children, meta.className);
+        this.andWhere({
+            [meta.root.discriminatorColumn]: children.length > 0 ? { $in: [meta.discriminatorValue, ...children.map(c => c.discriminatorValue)] } : meta.discriminatorValue,
+        });
+    }
     finalize() {
         if (this.finalized) {
             return;
@@ -744,11 +1020,12 @@ class QueryBuilder {
             this.select('*');
         }
         const meta = this.mainAlias.metadata;
+        this.applyDiscriminatorCondition();
         if (meta && this.flags.has(core_1.QueryFlag.AUTO_JOIN_ONE_TO_ONE_OWNER)) {
             const relationsToPopulate = this._populate.map(({ field }) => field);
             meta.relations
-                .filter(prop => prop.kind === core_1.ReferenceKind.ONE_TO_ONE && !prop.owner && !relationsToPopulate.includes(prop.name))
-                .map(prop => ({ field: prop.name }))
+                .filter(prop => prop.kind === core_1.ReferenceKind.ONE_TO_ONE && !prop.owner && !relationsToPopulate.includes(prop.name) && !relationsToPopulate.includes(`${prop.name}:ref`))
+                .map(prop => ({ field: `${prop.name}:ref` }))
                 .forEach(item => this._populate.push(item));
         }
         this._populate.forEach(({ field }) => {
@@ -759,14 +1036,11 @@ class QueryBuilder {
                 this._populateMap[join] = this._joins[join].alias;
                 return;
             }
-            if (this.metadata.find(field)?.pivotTable) { // pivot table entity
-                this.autoJoinPivotTable(field);
-            }
-            else if (meta && this.helper.isOneToOneInverse(fromField)) {
+            if (meta && this.helper.isOneToOneInverse(fromField)) {
                 const prop = meta.properties[fromField];
                 const alias = this.getNextAlias(prop.pivotEntity ?? prop.type);
                 const aliasedName = `${fromAlias}.${prop.name}#${alias}`;
-                this._joins[aliasedName] = this.helper.joinOneToReference(prop, this.mainAlias.aliasName, alias, 'leftJoin');
+                this._joins[aliasedName] = this.helper.joinOneToReference(prop, this.mainAlias.aliasName, alias, enums_1.JoinType.leftJoin);
                 this._populateMap[aliasedName] = this._joins[aliasedName].alias;
             }
         });
@@ -778,14 +1052,20 @@ class QueryBuilder {
                 const aliased = this.knex.ref(prop.fieldNames[0]).toString();
                 return `${prop.formula(alias)} as ${aliased}`;
             })
-                .filter(field => !this._fields.includes(field))
-                .forEach(field => this.addSelect(field));
+                .filter(field => !this._fields.some(f => {
+                if (f instanceof core_1.RawQueryFragment) {
+                    return f.sql === field && f.params.length === 0;
+                }
+                return f === field;
+            }))
+                .forEach(field => this._fields.push((0, core_1.raw)(field)));
         }
+        this.processPopulateWhere();
         core_1.QueryHelper.processObjectParams(this._data);
         core_1.QueryHelper.processObjectParams(this._cond);
         core_1.QueryHelper.processObjectParams(this._having);
-        // automatically enable paginate flag when we detect to-many joins
-        if (!this.flags.has(core_1.QueryFlag.DISABLE_PAGINATE) && this.hasToManyJoins()) {
+        // automatically enable paginate flag when we detect to-many joins, but only if there is no `group by` clause
+        if (!this.flags.has(core_1.QueryFlag.DISABLE_PAGINATE) && this._groupBy.length === 0 && this.hasToManyJoins()) {
             this.flags.add(core_1.QueryFlag.PAGINATE);
         }
         if (meta && this.flags.has(core_1.QueryFlag.PAGINATE) && (this._limit > 0 || this._offset > 0)) {
@@ -796,6 +1076,43 @@ class QueryBuilder {
         }
         this.finalized = true;
     }
+    processPopulateWhere() {
+        if (this._populateWhere == null || this._populateWhere === core_1.PopulateHint.ALL) {
+            return;
+        }
+        const joins = Object.values(this._joins);
+        joins.forEach(join => {
+            join.cond_ = join.cond;
+            join.cond = {};
+        });
+        const replaceOnConditions = (cond, op) => {
+            Object.keys(cond).forEach(k => {
+                if (core_1.Utils.isOperator(k)) {
+                    if (Array.isArray(cond[k])) {
+                        return cond[k].forEach((c) => replaceOnConditions(c, k));
+                    }
+                    /* istanbul ignore next */
+                    return replaceOnConditions(cond[k], k);
+                }
+                const [alias] = this.helper.splitField(k);
+                const join = joins.find(j => j.alias === alias);
+                if (join) {
+                    if (join.cond[k]) {
+                        join.cond = { [op ?? '$and']: [join.cond, { [k]: cond[k] }] };
+                    }
+                    else {
+                        join.cond = { ...join.cond, [k]: cond[k] };
+                    }
+                }
+            });
+        };
+        if (typeof this._populateWhere === 'object') {
+            const cond = CriteriaNodeFactory_1.CriteriaNodeFactory
+                .createNode(this.metadata, this.mainAlias.entityName, this._populateWhere)
+                .process(this, { matchPopulateJoins: true, ignoreBranching: true, preferNoBranch: true });
+            replaceOnConditions(cond);
+        }
+    }
     hasToManyJoins() {
         return Object.values(this._joins).some(join => {
             return [core_1.ReferenceKind.ONE_TO_MANY, core_1.ReferenceKind.MANY_TO_MANY].includes(join.prop.kind);
@@ -804,9 +1121,12 @@ class QueryBuilder {
     wrapPaginateSubQuery(meta) {
         const pks = this.prepareFields(meta.primaryKeys, 'sub-query');
         const subQuery = this.clone().select(pks).groupBy(pks).limit(this._limit);
+        // revert the on conditions added via populateWhere, we want to apply those only once
+        Object.values(subQuery._joins).forEach(join => join.cond = join.cond_ ?? {});
         if (this._offset) {
             subQuery.offset(this._offset);
         }
+        const addToSelect = [];
         if (this._orderBy.length > 0) {
             const orderBy = [];
             for (const orderMap of this._orderBy) {
@@ -814,28 +1134,38 @@ class QueryBuilder {
                     const [a, f] = this.helper.splitField(field);
                     const prop = this.helper.getProperty(f, a);
                     const type = this.platform.castColumn(prop);
-                    orderBy.push({
-                        [`min(${this.ref(this.helper.mapper(field, this.type, undefined, null))}${type})`]: direction,
-                    });
+                    const fieldName = this.helper.mapper(field, this.type, undefined, null);
+                    if (!prop?.persist && !prop?.formula) {
+                        addToSelect.push(fieldName);
+                    }
+                    orderBy.push({ [(0, core_1.raw)(`min(${this.knex.ref(fieldName)}${type})`)]: direction });
                 }
             }
             subQuery.orderBy(orderBy);
         }
         subQuery.finalized = true;
         const knexQuery = subQuery.as(this.mainAlias.aliasName).clearSelect().select(pks);
+        if (addToSelect.length > 0) {
+            addToSelect.forEach(prop => {
+                const field = this._fields.find(field => {
+                    if (typeof field === 'object' && field && '__as' in field) {
+                        return field.__as === prop;
+                    }
+                    // not perfect, but should work most of the time, ideally we should check only the alias (`... as alias`)
+                    return field.toString().includes(prop);
+                });
+                if (field) {
+                    knexQuery.select(field);
+                }
+            });
+        }
         // multiple sub-queries are needed to get around mysql limitations with order by + limit + where in + group by (o.O)
         // https://stackoverflow.com/questions/17892762/mysql-this-version-of-mysql-doesnt-yet-support-limit-in-all-any-some-subqu
         const subSubQuery = this.getKnex().select(pks).from(knexQuery);
+        subSubQuery.__raw = true; // tag it as there is now way to check via `instanceof`
         this._limit = undefined;
         this._offset = undefined;
-        const cond = this._cond;
         this.select(this._fields).where({ [core_1.Utils.getPrimaryKeyHash(meta.primaryKeys)]: { $in: subSubQuery } });
-        if (this._populateWhere === core_1.PopulateHint.INFER) {
-            this.andWhere(cond);
-        }
-        else if (typeof this._populateWhere === 'object') {
-            this.andWhere(this._populateWhere);
-        }
     }
     wrapModifySubQuery(meta) {
         const subQuery = this.clone();
@@ -849,21 +1179,10 @@ class QueryBuilder {
             [core_1.Utils.getPrimaryKeyHash(meta.primaryKeys)]: { $in: subSubQuery },
         });
     }
-    autoJoinPivotTable(field) {
-        const pivotMeta = this.metadata.find(field);
-        const owner = pivotMeta.relations[0];
-        const inverse = pivotMeta.relations[1];
-        const prop = this._cond[pivotMeta.name + '.' + owner.name] || this._orderBy[pivotMeta.name + '.' + owner.name] ? inverse : owner;
-        const pivotAlias = this.getNextAlias(pivotMeta.name);
-        this._joins[field] = this.helper.joinPivotTable(field, prop, this.mainAlias.aliasName, pivotAlias, 'leftJoin');
-        core_1.Utils.renameKey(this._cond, `${field}.${owner.name}`, core_1.Utils.getPrimaryKeyHash(owner.fieldNames.map(fieldName => `${pivotAlias}.${fieldName}`)));
-        core_1.Utils.renameKey(this._cond, `${field}.${inverse.name}`, core_1.Utils.getPrimaryKeyHash(inverse.fieldNames.map(fieldName => `${pivotAlias}.${fieldName}`)));
-        this._populateMap[field] = this._joins[field].alias;
-    }
     getSchema(alias) {
         const { metadata } = alias;
         const metaSchema = metadata?.schema && metadata.schema !== '*' ? metadata.schema : undefined;
-        return this._schema ?? metaSchema ?? this.em?.config.get('schema');
+        return this._schema ?? metaSchema ?? this.em?.schema ?? this.em?.config.get('schema');
     }
     createAlias(entityName, aliasName, subQuery) {
         const metadata = this.metadata.find(entityName);
@@ -901,6 +1220,7 @@ class QueryBuilder {
         }
     }
     /* istanbul ignore next */
+    /** @ignore */
     [util_1.inspect.custom](depth) {
         const object = { ...this };
         const hidden = ['metadata', 'driver', 'context', 'platform', 'knex', 'type'];
@@ -911,6 +1231,9 @@ class QueryBuilder {
         if (this._data) {
             object.data = this._data;
         }
+        if (this._schema) {
+            object.schema = this._schema;
+        }
         if (!core_1.Utils.isEmpty(this._cond)) {
             object.where = this._cond;
         }