@mikro-orm/core 7.0.0-dev.33 → 7.0.0-dev.35

package/EntityManager.d.ts

@@ -9,7 +9,7 @@ import { type EntityRepository } from './entity/EntityRepository.js';
 import { EntityLoader, type EntityLoaderOptions } from './entity/EntityLoader.js';
 import { Reference } from './entity/Reference.js';
 import { UnitOfWork } from './unit-of-work/UnitOfWork.js';
-import type { CountOptions, DeleteOptions, FindAllOptions, FindByCursorOptions, FindOneOptions, FindOneOrFailOptions, FindOptions, GetReferenceOptions, IDatabaseDriver, LockOptions, NativeInsertUpdateOptions, UpdateOptions, UpsertManyOptions, UpsertOptions } from './drivers/IDatabaseDriver.js';
+import type { CountOptions, DeleteOptions, FindAllOptions, FindByCursorOptions, FindOneOptions, FindOneOrFailOptions, FindOptions, GetReferenceOptions, IDatabaseDriver, LockOptions, NativeInsertUpdateOptions, StreamOptions, UpdateOptions, UpsertManyOptions, UpsertOptions } from './drivers/IDatabaseDriver.js';
 import type { AnyEntity, AnyString, ArrayElement, AutoPath, ConnectionType, Dictionary, EntityData, EntityDictionary, EntityDTO, EntityMetadata, EntityName, FilterQuery, FromEntityType, GetRepository, IHydrator, IsSubset, Loaded, MaybePromise, MergeLoaded, MergeSelected, NoInfer, ObjectQuery, Primary, Ref, RequiredEntityData, UnboxArray } from './typings.js';
 import { FlushMode, LockMode, PopulatePath, type TransactionOptions } from './enums.js';
 import type { MetadataStorage } from './metadata/MetadataStorage.js';
@@ -81,6 +81,24 @@ export declare class EntityManager<Driver extends IDatabaseDriver = IDatabaseDri
      * Finds all entities matching your `where` query. You can pass additional options via the `options` parameter.
      */
     find<Entity extends object, Hint extends string = never, Fields extends string = PopulatePath.ALL, Excludes extends string = never>(entityName: EntityName<Entity>, where: FilterQuery<NoInfer<Entity>>, options?: FindOptions<Entity, Hint, Fields, Excludes>): Promise<Loaded<Entity, Hint, Fields, Excludes>[]>;
+    /**
+     * Finds all entities and returns an async iterable (async generator) that yields results one by one.
+     * The results are merged and mapped to entity instances, without adding them to the identity map.
+     * You can disable merging by passing the options `{ mergeResults: false }`.
+     * With `mergeResults` disabled, to-many collections will contain at most one item, and you will get duplicate
+     * root entities when there are multiple items in the populated collection.
+     * This is useful for processing large datasets without loading everything into memory at once.
+     *
+     * ```ts
+     * const stream = em.stream(Book, { populate: ['author'] });
+     *
+     * for await (const book of stream) {
+     *   // book is an instance of Book entity
+     *   console.log(book.title, book.author.name);
+     * }
+     * ```
+     */
+    stream<Entity extends object, Hint extends string = never, Fields extends string = '*', Excludes extends string = never>(entityName: EntityName<Entity>, options?: StreamOptions<NoInfer<Entity>, Hint, Fields, Excludes>): AsyncIterableIterator<Loaded<Entity, Hint, Fields, Excludes>>;
     /**
      * Finds all entities of given type, optionally matching the `where` condition provided in the `options` parameter.
      */

package/EntityManager.js

@@ -1,6 +1,6 @@
 import { inspect } from 'node:util';
 import DataLoader from 'dataloader';
-import { getOnConflictReturningFields } from './utils/upsert-utils.js';
+import { getOnConflictReturningFields, getWhereCondition } from './utils/upsert-utils.js';
 import { Utils } from './utils/Utils.js';
 import { Cursor } from './utils/Cursor.js';
 import { DataloaderUtils } from './utils/DataloaderUtils.js';
@@ -182,6 +182,61 @@ export class EntityManager {
         }
         return unique;
     }
+    /**
+     * Finds all entities and returns an async iterable (async generator) that yields results one by one.
+     * The results are merged and mapped to entity instances, without adding them to the identity map.
+     * You can disable merging by passing the options `{ mergeResults: false }`.
+     * With `mergeResults` disabled, to-many collections will contain at most one item, and you will get duplicate
+     * root entities when there are multiple items in the populated collection.
+     * This is useful for processing large datasets without loading everything into memory at once.
+     *
+     * ```ts
+     * const stream = em.stream(Book, { populate: ['author'] });
+     *
+     * for await (const book of stream) {
+     *   // book is an instance of Book entity
+     *   console.log(book.title, book.author.name);
+     * }
+     * ```
+     */
+    async *stream(entityName, options = {}) {
+        const em = this.getContext();
+        em.prepareOptions(options);
+        options.strategy = 'joined';
+        await em.tryFlush(entityName, options);
+        entityName = Utils.className(entityName);
+        const where = await em.processWhere(entityName, options.where ?? {}, options, 'read');
+        em.validator.validateParams(where);
+        options.orderBy = options.orderBy || {};
+        options.populate = await em.preparePopulate(entityName, options);
+        const meta = this.metadata.get(entityName);
+        options = { ...options };
+        // save the original hint value so we know it was infer/all
+        options._populateWhere = options.populateWhere ?? this.config.get('populateWhere');
+        options.populateWhere = this.createPopulateWhere({ ...where }, options);
+        options.populateFilter = await this.getJoinedFilters(meta, { ...where }, options);
+        const stream = em.driver.stream(entityName, where, {
+            ctx: em.transactionContext,
+            mapResults: false,
+            ...options,
+        });
+        for await (const data of stream) {
+            const fork = em.fork();
+            const entity = fork.entityFactory.create(entityName, data, {
+                refresh: options.refresh,
+                schema: options.schema,
+                convertCustomTypes: true,
+            });
+            helper(entity).setSerializationContext({
+                populate: options.populate,
+                fields: options.fields,
+                exclude: options.exclude,
+            });
+            await fork.unitOfWork.dispatchOnLoadEvent();
+            fork.clear();
+            yield entity;
+        }
+    }
     /**
      * Finds all entities of given type, optionally matching the `where` condition provided in the `options` parameter.
      */
@@ -685,24 +740,7 @@ export class EntityManager {
                 }
             }
         }
-        const unique = options.onConflictFields ?? meta.props.filter(p => p.unique).map(p => p.name);
-        const propIndex = !isRaw(unique) && unique.findIndex(p => data[p] != null);
-        if (options.onConflictFields || where == null) {
-            if (propIndex !== false && propIndex >= 0) {
-                where = { [unique[propIndex]]: data[unique[propIndex]] };
-            }
-            else if (meta.uniques.length > 0) {
-                for (const u of meta.uniques) {
-                    if (Utils.asArray(u.properties).every(p => data[p] != null)) {
-                        where = Utils.asArray(u.properties).reduce((o, key) => {
-                            o[key] = data[key];
-                            return o;
-                        }, {});
-                        break;
-                    }
-                }
-            }
-        }
+        where = getWhereCondition(meta, options.onConflictFields, data, where).where;
         data = QueryHelper.processObjectParams(data);
         em.validator.validateParams(data, 'insert data');
         if (em.eventManager.hasListeners(EventType.beforeUpsert, meta)) {
@@ -845,31 +883,17 @@ export class EntityManager {
                     }
                 }
             }
-            const unique = meta.props.filter(p => p.unique).map(p => p.name);
-            propIndex = unique.findIndex(p => row[p] != null);
-            if (options.onConflictFields || where == null) {
-                if (propIndex >= 0) {
-                    where = { [unique[propIndex]]: row[unique[propIndex]] };
-                }
-                else if (meta.uniques.length > 0) {
-                    for (const u of meta.uniques) {
-                        if (Utils.asArray(u.properties).every(p => row[p] != null)) {
-                            where = Utils.asArray(u.properties).reduce((o, key) => {
-                                o[key] = row[key];
-                                return o;
-                            }, {});
-                            break;
-                        }
-                    }
-                }
-            }
-            row = QueryHelper.processObjectParams(row);
+            const unique = options.onConflictFields ?? meta.props.filter(p => p.unique).map(p => p.name);
+            propIndex = !isRaw(unique) && unique.findIndex(p => data[p] ?? data[p.substring(0, p.indexOf('.'))] != null);
+            const tmp = getWhereCondition(meta, options.onConflictFields, row, where);
+            propIndex = tmp.propIndex;
             where = QueryHelper.processWhere({
-                where,
+                where: tmp.where,
                 entityName,
                 metadata: this.metadata,
                 platform: this.getPlatform(),
             });
+            row = QueryHelper.processObjectParams(row);
             em.validator.validateParams(row, 'insert data');
             allData.push(row);
             allWhere.push(where);
@@ -911,7 +935,7 @@ export class EntityManager {
         const reloadFields = returning.length > 0 && !(this.getPlatform().usesReturningStatement() && res.rows?.length);
         if (options.onConflictAction === 'ignore' || (!res.rows?.length && loadPK.size > 0) || reloadFields) {
             const unique = meta.hydrateProps.filter(p => !p.lazy).map(p => p.name);
-            const add = new Set(propIndex >= 0 ? [unique[propIndex]] : []);
+            const add = new Set(propIndex !== false && propIndex >= 0 ? [unique[propIndex]] : []);
             for (const cond of loadPK.values()) {
                 Utils.keys(cond).forEach(key => add.add(key));
             }

package/decorators/Entity.d.ts

@@ -27,7 +27,7 @@ export type EntityOptions<T, E = T extends EntityClass<infer P> ? P : T> = {
     /** Used to make ORM aware of externally defined triggers. This is needed for MS SQL Server multi inserts, ignored in other dialects. */
     hasTriggers?: boolean;
     /** SQL query that maps to a {@doclink virtual-entities | virtual entity}. */
-    expression?: string | ((em: any, where: ObjectQuery<E>, options: FindOptions<E, any, any, any>) => object);
+    expression?: string | ((em: any, where: ObjectQuery<E>, options: FindOptions<E, any, any, any>, stream?: boolean) => object);
     /** Set {@doclink repositories#custom-repository | custom repository class}. */
     repository?: () => Constructor;
 };

package/drivers/DatabaseDriver.d.ts

@@ -1,5 +1,5 @@
-import { type CountOptions, type DeleteOptions, type DriverMethodOptions, EntityManagerType, type FindOneOptions, type FindOptions, type IDatabaseDriver, type LockOptions, type NativeInsertUpdateManyOptions, type NativeInsertUpdateOptions, type OrderDefinition } from './IDatabaseDriver.js';
-import type { ConnectionType, Dictionary, EntityData, EntityDictionary, EntityMetadata, EntityProperty, FilterQuery, PopulateOptions, Primary } from '../typings.js';
+import { type CountOptions, type DeleteOptions, type DriverMethodOptions, EntityManagerType, type FindOneOptions, type FindOptions, type IDatabaseDriver, type LockOptions, type NativeInsertUpdateManyOptions, type NativeInsertUpdateOptions, type OrderDefinition, type StreamOptions } from './IDatabaseDriver.js';
+import type { ConnectionType, Dictionary, EntityData, EntityDictionary, EntityMetadata, EntityName, EntityProperty, FilterQuery, PopulateOptions, Primary } from '../typings.js';
 import type { MetadataStorage } from '../metadata/MetadataStorage.js';
 import type { Connection, QueryResult, Transaction } from '../connections/Connection.js';
 import { type Configuration, type ConnectionOptions } from '../utils/Configuration.js';
@@ -30,7 +30,7 @@ export declare abstract class DatabaseDriver<C extends Connection> implements ID
     abstract nativeDelete<T extends object>(entityName: string, where: FilterQuery<T>, options?: DeleteOptions<T>): Promise<QueryResult<T>>;
     abstract count<T extends object, P extends string = never>(entityName: string, where: FilterQuery<T>, options?: CountOptions<T, P>): Promise<number>;
     createEntityManager(useContext?: boolean): this[typeof EntityManagerType];
-    findVirtual<T extends object>(entityName: string, where: FilterQuery<T>, options: FindOptions<T, any, any, any>): Promise<EntityData<T>[]>;
+    findVirtual<T extends object>(entityName: EntityName<T>, where: FilterQuery<T>, options: FindOptions<T, any, any, any>): Promise<EntityData<T>[]>;
     countVirtual<T extends object>(entityName: string, where: FilterQuery<T>, options: CountOptions<T, any>): Promise<number>;
     aggregate(entityName: string, pipeline: any[]): Promise<any[]>;
     loadFromPivotTable<T extends object, O extends object>(prop: EntityProperty, owners: Primary<O>[][], where?: FilterQuery<any>, orderBy?: OrderDefinition<T>, ctx?: Transaction, options?: FindOptions<T, any, any, any>, pivotJoin?: boolean): Promise<Dictionary<T[]>>;
@@ -55,6 +55,7 @@ export declare abstract class DatabaseDriver<C extends Connection> implements ID
     protected getPrimaryKeyFields(entityName: string): string[];
     protected createReplicas(cb: (c: ConnectionOptions) => C): C[];
     lockPessimistic<T extends object>(entity: T, options: LockOptions): Promise<void>;
+    abstract stream<T extends object>(entityName: EntityName<T>, where: FilterQuery<T>, options: StreamOptions<T>): AsyncIterableIterator<T>;
     /**
      * @inheritDoc
      */

package/drivers/IDatabaseDriver.d.ts

@@ -1,4 +1,4 @@
-import type { ConnectionType, EntityData, EntityMetadata, EntityProperty, FilterQuery, Primary, Dictionary, QBFilterQuery, IPrimaryKey, PopulateOptions, EntityDictionary, AutoPath, ObjectQuery, FilterObject, Populate } from '../typings.js';
+import type { ConnectionType, EntityData, EntityMetadata, EntityProperty, FilterQuery, Primary, Dictionary, QBFilterQuery, IPrimaryKey, PopulateOptions, EntityDictionary, AutoPath, ObjectQuery, FilterObject, Populate, EntityName } from '../typings.js';
 import type { Connection, QueryResult, Transaction } from '../connections/Connection.js';
 import type { FlushMode, LockMode, QueryOrderMap, QueryFlag, LoadStrategy, PopulateHint, PopulatePath } from '../enums.js';
 import type { Platform } from '../platforms/Platform.js';
@@ -27,6 +27,7 @@ export interface IDatabaseDriver<C extends Connection = Connection> {
      */
     findOne<T extends object, P extends string = never, F extends string = '*', E extends string = never>(entityName: string, where: FilterQuery<T>, options?: FindOneOptions<T, P, F, E>): Promise<EntityData<T> | null>;
     findVirtual<T extends object>(entityName: string, where: FilterQuery<T>, options: FindOptions<T, any, any, any>): Promise<EntityData<T>[]>;
+    stream<T extends object>(entityName: EntityName<T>, where: FilterQuery<T>, options: StreamOptions<T>): AsyncIterableIterator<T>;
     nativeInsert<T extends object>(entityName: string, data: EntityDictionary<T>, options?: NativeInsertUpdateOptions<T>): Promise<QueryResult<T>>;
     nativeInsertMany<T extends object>(entityName: string, data: EntityDictionary<T>[], options?: NativeInsertUpdateManyOptions<T>, transform?: (sql: string) => string): Promise<QueryResult<T>>;
     nativeUpdate<T extends object>(entityName: string, where: FilterQuery<T>, data: EntityDictionary<T>, options?: NativeInsertUpdateOptions<T>): Promise<QueryResult<T>>;
@@ -68,6 +69,18 @@ export type OrderDefinition<T> = (QueryOrderMap<T> & {
 export interface FindAllOptions<T, P extends string = never, F extends string = '*', E extends string = never> extends FindOptions<T, P, F, E> {
     where?: FilterQuery<T>;
 }
+export interface StreamOptions<Entity, Populate extends string = never, Fields extends string = '*', Exclude extends string = never> extends Omit<FindAllOptions<Entity, Populate, Fields, Exclude>, 'cache' | 'before' | 'after' | 'first' | 'last' | 'overfetch' | 'strategy'> {
+    /**
+     * When populating to-many relations, the ORM streams fully merged entities instead of yielding every row.
+     * You can opt out of this behavior by specifying `mergeResults: false`. This will yield every row from
+     * the SQL result, but still mapped to entities, meaning that to-many collections will contain at most
+     * a single item, and you will get duplicate root entities when they have multiple items in the populated
+     * collection.
+     *
+     * @default true
+     */
+    mergeResults?: boolean;
+}
 export type FilterOptions = Dictionary<boolean | Dictionary> | string[] | boolean;
 export interface LoadHint<Entity, Hint extends string = never, Fields extends string = PopulatePath.ALL, Excludes extends string = never> {
     populate?: Populate<Entity, Hint>;

package/entity/EntityFactory.js

@@ -136,6 +136,10 @@ export class EntityFactory {
             if ([ReferenceKind.MANY_TO_ONE, ReferenceKind.ONE_TO_ONE].includes(prop.kind) && Utils.isPlainObject(data[prop.name])) {
                 diff2[key] = entity[prop.name] ? helper(entity[prop.name]).getPrimaryKey(options.convertCustomTypes) : null;
             }
+            if ([ReferenceKind.MANY_TO_ONE, ReferenceKind.ONE_TO_ONE, ReferenceKind.SCALAR].includes(prop.kind) && prop.customType?.ensureComparable(meta, prop) && diff2[key] != null) {
+                const converted = prop.customType.convertToJSValue(diff2[key], this.platform, { force: true });
+                diff2[key] = prop.customType.convertToDatabaseValue(converted, this.platform, { fromQuery: true });
+            }
             originalEntityData[key] = diff2[key] === null ? nullVal : diff2[key];
             helper(entity).__loadedProperties.add(key);
         });