joist-core 2.2.0 → 2.3.0-next.10

package/build/EntityManager.js

@@ -29,7 +29,6 @@ const IndexManager_1 = require("./IndexManager");
 // We alias `Entity => EntityW` to denote "Entity wide" i.e. the non-narrowed Entity
 const loadBatchLoader_1 = require("./batchloaders/loadBatchLoader");
 const populateBatchLoader_1 = require("./batchloaders/populateBatchLoader");
-const caches_1 = require("./caches");
 const config_1 = require("./config");
 const configure_1 = require("./configure");
 const findByUniqueDataLoader_1 = require("./dataloaders/findByUniqueDataLoader");
@@ -37,20 +36,23 @@ const findCountDataLoader_1 = require("./dataloaders/findCountDataLoader");
 const findDataLoader_1 = require("./dataloaders/findDataLoader");
 const findIdsDataLoader_1 = require("./dataloaders/findIdsDataLoader");
 const findOrCreateDataLoader_1 = require("./dataloaders/findOrCreateDataLoader");
+const findPaginatedDataLoader_1 = require("./dataloaders/findPaginatedDataLoader");
 const Entity_1 = require("./Entity");
 const FlushLock_1 = require("./FlushLock");
 const index_1 = require("./index");
 const IsLoadedCache_1 = require("./IsLoadedCache");
 const JoinRows_1 = require("./JoinRows");
+const loadHints_1 = require("./loadHints");
 const newEntity_1 = require("./newEntity");
 const newTestInstance_1 = require("./newTestInstance");
 const PluginManager_1 = require("./PluginManager");
 const ReactionsManager_1 = require("./ReactionsManager");
 const reactiveHints_1 = require("./reactiveHints");
 const relations_1 = require("./relations");
-const hasAsyncMethod_1 = require("./relations/hasAsyncMethod");
 const AsyncProperty_1 = require("./relations/AsyncProperty");
+const hasAsyncMethod_1 = require("./relations/hasAsyncMethod");
 const RecursiveCollection_1 = require("./relations/RecursiveCollection");
+const scopes_1 = require("./scopes");
 const Todo_1 = require("./Todo");
 const trusted_1 = require("./trusted");
 const upsert_1 = require("./upsert");
@@ -87,6 +89,8 @@ class EntityManager {
     txn;
     entityLimit = defaultEntityLimit;
     #entitiesArray = [];
+    // Incrementally track dirty entities so we don't have to scan `#entityArray` during flush
+    #maybePendingFlushEntities = new Set();
     // Indexes the currently loaded entities by their tagged ids and `toTaggedString` ids (i.e. `a#`). This fixes
     // real-world performance issues where `findExistingInstance` scanning `#entities` was an `O(n^2)`.
     #entitiesById = new Map();
@@ -105,6 +109,7 @@ class EntityManager {
      * so that both see the most accurate state.
      */
     #pendingDeletes = [];
+    #hasAnyDeletes = false;
     #dataloaders = {};
     #batchLoaders = {};
     #joinRows = {};
@@ -154,6 +159,20 @@ class EntityManager {
             indexManager: this.#indexManager,
             isLoadedCache: this.#isLoadedCache,
             pluginManager,
+            markMaybePending(entity) {
+                em.#maybePendingFlushEntities.add(entity);
+            },
+            unmarkMaybePending(entity) {
+                em.#maybePendingFlushEntities.delete(entity);
+            },
+            hasAnyDeletes() {
+                return em.#hasAnyDeletes;
+            },
+            pendingDeleteIds(type) {
+                return em.#pendingDeletes
+                    .filter((entity) => entity instanceof type && entity.idMaybe !== undefined)
+                    .map((entity) => (0, index_1.keyToNumber)((0, index_1.getMetadata)(entity), entity.id));
+            },
             isMerging(entity) {
                 return em.#merging?.has(entity) ?? false;
             },
@@ -219,7 +238,8 @@ class EntityManager {
         }
         for (const joinRows of Object.values(this.#joinRows)) {
             const todo = joinRows.toTodo();
-            if (todo) {
+            // Enum m2ms (EnumCollections) aren't entity-to-entity, so skip them here.
+            if (todo && !todo.m2m.otherEnum) {
                 for (const row of todo.newRows) {
                     const entities = Object.values(row.columns);
                     changes.push({ kind: "m2m", op: "add", joinTableName: todo.m2m.joinTableName, entities });
@@ -246,10 +266,11 @@ class EntityManager {
     }
     async find(type, where, options) {
         const { populate, ...rest } = options || {};
-        const settings = { where, ...rest };
-        const result = await (0, findDataLoader_1.findDataLoader)(this, type, settings, populate)
-            .load(settings)
-            .catch(function find(err) {
+        const normalized = (0, index_1.mergeFindOptions)(where, rest);
+        const settings = { where: normalized.where, ...normalized.options };
+        const result = await (hasPaginationSettings(normalized.options)
+            ? (0, findPaginatedDataLoader_1.findPaginatedDataLoader)(this, type, settings, populate)
+            : (0, findDataLoader_1.findDataLoader)(this, type, settings, populate)).catch(function find(err) {
             throw appendStack(err, new Error());
         });
         if (populate) {
@@ -257,37 +278,52 @@ class EntityManager {
         }
         return result;
     }
-    async findPaginated(type, where, options) {
-        const { populate, limit, offset, ...rest } = options || {};
-        const meta = (0, index_1.getMetadata)(type);
-        const query = (0, index_1.parseFindQuery)(meta, where, rest);
-        const rows = await this.executeFind(meta, "findPaginated", query, { ...rest, limit, offset, checkLimit: false });
-        const result = this.hydrate(type, rows);
-        if (populate) {
-            await this.populate(result, populate);
-        }
-        return result;
-    }
     /** Runs the post-parse find pipeline: plugins mutate the logical AST, then Joist optimizes/prunes before SQL. */
     async executeFind(meta, operation, parsed, settings) {
-        const { checkLimit, ...driverSettings } = settings;
+        const { checkLimit, findSettings } = this.prepareFind(meta, operation, parsed, settings);
+        return this.executePreparedFind(meta, operation, parsed, findSettings, checkLimit);
+    }
+    /** Executes a query that has already had find hooks and optimizations applied. */
+    async executePreparedFind(meta, operation, parsed, findSettings, checkLimit) {
         const { pluginManager } = getEmInternalApi(this);
-        pluginManager.beforeFind(meta, operation, parsed, driverSettings);
-        (0, index_1.optimizeCollectionJoins)(parsed, settings);
-        const rows = await this.driver.executeFind(this, parsed, driverSettings);
+        const rows = await this.driver.executeFind(this, parsed, findSettings);
         // Check by default unless explicitly disabled or the caller removed the LIMIT via `limit: undefined`
-        const shouldCheck = checkLimit ?? !("limit" in settings && settings.limit === undefined);
+        const shouldCheck = checkLimit ?? !("limit" in findSettings && findSettings.limit === undefined);
         if (shouldCheck && rows.length >= this.entityLimit) {
             throw new Error(`Query returned more than ${this.entityLimit} entityLimit rows`);
         }
         pluginManager.afterFind(meta, operation, rows);
         return rows;
     }
-    async findGql(type, where, options) {
-        return this.find(type, where, options);
+    /**
+     * Runs pre-SQL find hooks and optimizations against a parsed query.
+     *
+     * This allows plugins to see "pre-batched" / "logical" query ASTs, instead of our
+     * more complicated `_find` batched queries. The flow would be:
+     *
+     * - A loader calls `prepareFind(originalQuery)`
+     * - Plugins inspect/modify the query as/if needed
+     * - The loader crafts a new, more complicated query that embeds the originalQuery
+     * - The loader calls `executePreparedFind` with the 2nd query
+     */
+    prepareFind(meta, operation, parsed, settings) {
+        const { checkLimit, ...findSettings } = settings;
+        const { pluginManager } = getEmInternalApi(this);
+        // Plugins may mutate the settings object, so return the post-hook version that loaders must reuse.
+        pluginManager.beforeFind(meta, operation, parsed, findSettings);
+        (0, index_1.optimizeCollectionJoins)(parsed, settings);
+        return { checkLimit, findSettings };
     }
-    async findGqlPaginated(type, where, options) {
-        return this.findPaginated(type, where, options);
+    async findGql(type, where, options) {
+        if (!options) {
+            return this.find(type, where);
+        }
+        const normalized = { ...options };
+        if ("limit" in normalized)
+            normalized.limit = normalized.limit ?? undefined;
+        if ("offset" in normalized)
+            normalized.offset = normalized.offset ?? undefined;
+        return this.find(type, where, normalized);
     }
     async findOne(type, where, options) {
         const list = await this.find(type, where, options);
@@ -329,6 +365,9 @@ class EntityManager {
         }
         else {
             const [entity] = this.hydrate(type, [row]);
+            if (this.#hasAnyDeletes && entity.isDeletedEntity) {
+                return undefined;
+            }
             if (populate) {
                 await this.populate(entity, populate);
             }
@@ -344,17 +383,16 @@ class EntityManager {
      * Note: this method is not currently auto-batched, so it will cause N+1s if called in a loop.
      */
     async findCount(type, where, options = {}) {
-        const settings = { where, ...options };
-        let count = await (0, findCountDataLoader_1.findCountDataLoader)(this, type, settings)
-            .load(settings)
-            .catch(function findCount(err) {
+        const normalized = (0, findCountDataLoader_1.mergeCountOptions)(where, options);
+        const settings = { where: normalized.where, ...normalized.options };
+        let count = await (0, findCountDataLoader_1.findCountDataLoader)(this, type, settings).catch(function findCount(err) {
             throw appendStack(err, new Error());
         });
         // If the user is do "count all", we can adjust the number up/down based on
         // WIP creates/deletes. We can't do this if the WHERE clause is populated b/c
         // then we'd also have to eval each created/deleted entity against the WHERE
         // clause before knowing if it should adjust teh amount.
-        const isSelectAll = Object.keys(where).length === 0;
+        const isSelectAll = (0, scopes_1.isSelectAllFilter)(where, options.conditions);
         if (isSelectAll) {
             const tagged = this.#entitiesByTag.get((0, index_1.getMetadata)(type).tagName) ?? [];
             for (const entity of tagged) {
@@ -378,10 +416,9 @@ class EntityManager {
      * as `em.find`.
      */
     async findIds(type, where, options = {}) {
-        const settings = { where, ...options };
-        return (0, findIdsDataLoader_1.findIdsDataLoader)(this, type, settings)
-            .load(settings)
-            .catch(function findIds(err) {
+        const normalized = (0, findCountDataLoader_1.mergeCountOptions)(where, options);
+        const settings = { where: normalized.where, ...normalized.options };
+        return (0, findIdsDataLoader_1.findIdsDataLoader)(this, type, settings).catch(function findIds(err) {
             throw appendStack(err, new Error());
         });
     }
@@ -485,6 +522,7 @@ class EntityManager {
                     case "primaryKey":
                     case "o2m":
                     case "m2m":
+                    case "m2mEnum":
                     case "o2o":
                     case "lo2m":
                         return undefined;
@@ -613,6 +651,7 @@ class EntityManager {
                 else if (kind === "primitive" ||
                     kind === "primaryKey" ||
                     kind === "enum" ||
+                    kind === "m2mEnum" ||
                     kind === "poly" ||
                     kind === "m2o" ||
                     kind === "lo2m") {
@@ -670,48 +709,86 @@ class EntityManager {
     }
     async loadAll(type, _ids, hint) {
         const meta = (0, index_1.getMetadata)(type);
-        const ids = _ids.map((id) => (0, index_1.tagId)(meta, id));
-        const idsToLoad = ids.filter((id) => !this.findExistingInstance(id));
-        if (idsToLoad.length > 0) {
+        // Use pre-allocated arrays/for loops instead of `.filter`s since this can be a hot spot
+        const ids = new Array(_ids.length);
+        const entities = new Array(_ids.length);
+        let idsToLoad;
+        let positionsToLoad;
+        for (let i = 0; i < _ids.length; i++) {
+            const id = (0, index_1.tagId)(meta, _ids[i]);
+            ids[i] = id;
+            const entity = this.findExistingInstance(id);
+            if (entity) {
+                entities[i] = entity;
+            }
+            else {
+                (idsToLoad ??= []).push(id);
+                (positionsToLoad ??= []).push(i);
+            }
+        }
+        if (idsToLoad && idsToLoad.length > 0) {
             await (0, loadBatchLoader_1.loadBatchLoader)(this, meta)
                 .loadAll(idsToLoad.map((id) => ({ taggedId: id, hint })))
                 .catch(function loadAll(err) {
                 throw appendStack(err, new Error());
             });
+            for (const i of positionsToLoad) {
+                entities[i] = this.findExistingInstance(ids[i]);
+            }
         }
-        const entities = [];
-        for (const id of ids) {
-            const entity = this.findExistingInstance(id);
-            if (entity)
-                entities.push(entity);
+        let idsNotFound;
+        for (let i = 0; i < entities.length; i++) {
+            if (entities[i] === undefined) {
+                (idsNotFound ??= []).push(ids[i]);
+            }
         }
-        if (entities.length !== ids.length) {
-            const idsNotFound = ids.filter((_, i) => entities[i] === undefined);
+        if (idsNotFound) {
             throw new NotFoundError(`${idsNotFound.join(",")} were not found`);
         }
+        const loadedEntities = entities;
         if (hint) {
-            await this.populate(entities, hint);
+            await this.populate(loadedEntities, hint);
         }
         if (meta.inheritanceType === "sti" && meta.baseType) {
-            const wrongType = entities.filter((e) => !(e instanceof meta.cstr));
+            const wrongType = loadedEntities.filter((e) => !(e instanceof meta.cstr));
             if (wrongType.length > 0) {
                 throw new Error(`${wrongType.join(", ")} were not of type ${meta.cstr.name}`);
             }
         }
-        return entities;
+        return loadedEntities;
     }
     async loadAllIfExists(type, _ids, hint) {
         const meta = (0, index_1.getMetadata)(type);
-        const ids = _ids.map((id) => (0, index_1.tagId)(meta, id));
-        const idsToLoad = ids.filter((id) => !this.findExistingInstance(id));
-        if (idsToLoad.length > 0) {
+        // Use pre-allocated arrays/for loops instead of `.filter`s since this can be a hot spot
+        const ids = new Array(_ids.length);
+        const entities = [];
+        let idsToLoad;
+        // Ensure the ids are tagged, and find any not-yet-loaded
+        for (let i = 0; i < _ids.length; i++) {
+            const id = (0, index_1.tagId)(meta, _ids[i]);
+            ids[i] = id;
+            const entity = this.findExistingInstance(id);
+            if (entity) {
+                entities.push(entity);
+            }
+            else {
+                (idsToLoad ??= []).push(id);
+            }
+        }
+        if (idsToLoad && idsToLoad.length > 0) {
             await (0, loadBatchLoader_1.loadBatchLoader)(this, meta)
                 .loadAll(idsToLoad.map((id) => ({ taggedId: id, hint })))
                 .catch(function loadAllIfExists(err) {
                 throw appendStack(err, new Error());
             });
+            // Now that everything is loaded, recalc `entities`
+            entities.length = 0;
+            for (const id of ids) {
+                const entity = this.findExistingInstance(id);
+                if (entity)
+                    entities.push(entity);
+            }
         }
-        const entities = ids.map((id) => this.findExistingInstance(id)).filter(Boolean);
         if (hint) {
             await this.populate(entities, hint);
         }
@@ -762,6 +839,12 @@ class EntityManager {
         if (list.length === 0) {
             return !fn ? entityOrList : fn(entityOrList);
         }
+        // Avoid building a HintTree/batchloader for persisted entities when everything is already loaded; in the phase 9
+        // benchmark this moved already-loaded populates from ~1.10ms to ~0.89ms, and nested already-loaded populates from
+        // ~2.39ms to ~1.94ms.
+        if (!opts.forceReload && list.every((entity) => !entity.isNewEntity && (0, loadHints_1.isLoadedForPopulate)(entity, hintOpt))) {
+            return fn ? fn(entityOrList) : entityOrList;
+        }
         const meta = (0, index_1.getMetadata)(list[0]);
         if (this.#preloader) {
             // If we can preload, prevent promise deadlocking by one large-batch preload populate (which can't have
@@ -810,9 +893,11 @@ class EntityManager {
     delete(entityOrArray) {
         for (const entity of (0, utils_1.toArray)(entityOrArray)) {
             // Early return if already deleted.
-            const alreadyMarked = (0, BaseEntity_1.getInstanceData)(entity).markDeleted(entity);
+            const alreadyMarked = (0, BaseEntity_1.getInstanceData)(entity).markDeleted();
             if (!alreadyMarked)
                 continue;
+            // This monotonic flag lets find hot paths skip scanning results until a delete has ever happened in this EM.
+            this.#hasAnyDeletes = true;
             // Any derived fields that read this entity will need recalc-d
             this.#rm.queueAllDownstreamFields(entity, "deleted");
             // Synchronously unhook the entity if the relations are loaded
@@ -906,7 +991,7 @@ class EntityManager {
                 const pendingHooks = new Set();
                 // Subset of pendingFlush entities that had hooks invoked in a prior `runHooksOnPendingEntities`
                 const alreadyRanHooks = new Set();
-                findPendingFlushEntities(this.entities, hooksInvoked, pendingFlush, pendingHooks, alreadyRanHooks);
+                findPendingFlushEntities(this.#maybePendingFlushEntities, hooksInvoked, pendingFlush, pendingHooks, alreadyRanHooks);
                 // If we're re-looping for AsyncReactiveField, make sure to bump updatedAt
                 // each time, so that for an INSERT-then-UPDATE the triggers don't think the
                 // UPDATE forgot to self-bump updatedAt, and then "helpfully" bump it for us.
@@ -930,9 +1015,13 @@ class EntityManager {
                         // fields, they'd be via the synchronous getter and would not be stale.
                         recalcSynchronousDerivedFields(todos);
                         // The hooks could have deleted this-loop or prior-loop entities, so re-cascade again.
-                        await this.flushDeletes();
-                        // The hooks could have changed fields, so recalc again.
-                        await this.#rm.recalcPendingReactables("reactables");
+                        // Reactions (the recalc below) can themselves `em.delete` entities, so keep draining
+                        // until both deletes and recalcs have settled.
+                        do {
+                            await this.flushDeletes();
+                            // The hooks could have changed fields, so recalc again.
+                            await this.#rm.recalcPendingReactables("reactables");
+                        } while (this.#pendingDeletes.length > 0);
                         // We may have reactables that failed earlier, but will succeed now that hooks have been run and cascade
                         // deletes have been processed
                         if (this.#rm.hasPendingTypeErrors) {
@@ -945,7 +1034,7 @@ class EntityManager {
                             hooksInvoked.add(e);
                         pendingHooks.clear();
                         // See if the hooks mutated any new, not-yet-hooksInvoked entities
-                        findPendingFlushEntities(this.entities, hooksInvoked, pendingFlush, pendingHooks, alreadyRanHooks);
+                        findPendingFlushEntities(this.#maybePendingFlushEntities, hooksInvoked, pendingFlush, pendingHooks, alreadyRanHooks);
                         // The final run of recalcPendingReactables could have left us with pending type errors and no entities in
                         // pendingHooks.  If so, we need to re-run recalcPendingTypeErrors to get those errors to transition into
                         // suppressed errors so that we will fail after simpleValidation.
@@ -1004,7 +1093,7 @@ class EntityManager {
                 // The driver will handle the right thing if we're already in an existing transaction.
                 await this.driver.transaction(this, async () => {
                     do {
-                        if (Object.keys(entityTodos).length > 0 || Object.keys(joinRowTodos)) {
+                        if (Object.keys(entityTodos).length > 0 || Object.keys(joinRowTodos).length > 0) {
                             await this.driver.flush(this, entityTodos, joinRowTodos);
                         }
                         // Now that we've flushed, we can let plugins know what we've done.
@@ -1080,7 +1169,7 @@ class EntityManager {
             for (const e of createdThenDeleted)
                 (0, BaseEntity_1.getInstanceData)(e).fixupCreatedThenDeleted();
             this.#merging?.clear();
-            return [...allFlushedEntities];
+            return [...allFlushedEntities].sort((a, b) => (0, BaseEntity_1.getInstanceData)(a).entityIndex - (0, BaseEntity_1.getInstanceData)(b).entityIndex);
         }
         catch (e) {
             if (e instanceof RecursiveCollection_1.RecursiveCycleError) {
@@ -1096,15 +1185,28 @@ class EntityManager {
                     }
                 }
             }
+            // node-pg keeps the DETAIL line (i.e. "Key (identity_id, final_id)=(102591, null) already exists")
+            // on a separate `detail` field, so keep it to suffix onto messages to help debugging.
+            const detail = e && typeof e === "object" && "detail" in e && typeof e.detail === "string" ? e.detail : undefined;
             if (e && typeof e === "object" && "constraint" in e && typeof e.constraint === "string") {
                 // node-pg errors use `constraint` to indicate the constraint name
                 const message = config_1.constraintNameToValidationError[e.constraint];
                 if (message) {
-                    throw new index_1.ValidationErrors(message);
+                    const errors = new index_1.ValidationErrors(message);
+                    // Keep the user-facing `errors[].message` pretty, but suffix the DETAIL onto the top-level
+                    // `message` so the debugging context survives into logs/stack traces.
+                    if (detail)
+                        errors.message = `${message} — ${detail}`;
+                    throw errors;
                 }
             }
-            if (e instanceof InMemoryRollbackError)
-                return [...allFlushedEntities];
+            if (e instanceof InMemoryRollbackError) {
+                return [...allFlushedEntities].sort((a, b) => (0, BaseEntity_1.getInstanceData)(a).entityIndex - (0, BaseEntity_1.getInstanceData)(b).entityIndex);
+            }
+            // For raw (non-pretty) pg errors, fold the DETAIL into `message` so it survives into logs.
+            if (detail && e instanceof Error) {
+                e.message = `${e.message} — ${detail}`;
+            }
             throw e;
         }
         finally {
@@ -1212,10 +1314,13 @@ class EntityManager {
      */
     hydrate(type, rows, options) {
         const maybeBaseMeta = (0, index_1.getMetadata)(type);
+        const taggedIdPrefix = `${maybeBaseMeta.tagName}:`;
+        const overwriteExisting = options?.overwriteExisting === true;
         let i = 0;
         const entities = new Array(rows.length);
         for (const row of rows) {
-            const taggedId = (0, index_1.keyToTaggedId)(maybeBaseMeta, row["id"]) || (0, utils_1.fail)("No id column was available");
+            const id = row["id"];
+            const taggedId = id === undefined || id === null ? (0, utils_1.fail)("No id column was available") : `${taggedIdPrefix}${id}`;
             // See if this is already in our UoW
             let entity = this.findExistingInstance(taggedId);
             if (!entity) {
@@ -1224,23 +1329,36 @@ class EntityManager {
                 // Pass id as a hint that we're in hydrate mode
                 entity = (0, newEntity_1.newEntity)(this, (0, index_1.asConcreteCstr)(meta.cstr), false);
                 (0, BaseEntity_1.getInstanceData)(entity).row = row;
-                this.#doRegister(entity, taggedId);
+                this.#doRegister(entity, taggedId, meta, true);
             }
-            else if (options?.overwriteExisting === true) {
+            else if (overwriteExisting) {
                 // Usually if the entity already exists, we don't write over it, but in this case we assume that
                 // `EntityManager.refresh` is telling us to explicitly load the latest data.
                 // First swap out the old row with the new row
-                (0, BaseEntity_1.getInstanceData)(entity).row = row;
+                const instanceData = (0, BaseEntity_1.getInstanceData)(entity);
+                instanceData.row = row;
                 // And then only refresh the data keys that have already been serde-d from rows
                 // (this keeps us from deserializing data out of rows that we don't need).
-                const { data, originalData } = (0, BaseEntity_1.getInstanceData)(entity);
-                const changedFields = entity.changes.fieldsWithoutRelations;
-                for (const fieldName of Object.keys(data)) {
-                    const serde = (0, index_1.getMetadata)(entity).allFields[fieldName].serde ?? (0, utils_1.fail)(`Missing serde for ${fieldName}`);
-                    serde.setOnEntity(data, row);
-                    // Make the field look not-dirty
-                    if (changedFields.includes(fieldName)) {
-                        delete originalData[fieldName];
+                const { data } = instanceData;
+                const dataKeys = Object.keys(data);
+                if (dataKeys.length > 0) {
+                    const allFields = (0, index_1.getMetadata)(entity).allFields;
+                    const changedFields = entity.changes.fieldsWithoutRelations;
+                    if (changedFields.length === 0) {
+                        for (const fieldName of dataKeys) {
+                            const serde = allFields[fieldName].serde ?? (0, utils_1.fail)(`Missing serde for ${fieldName}`);
+                            serde.setOnEntity(data, row);
+                        }
+                    }
+                    else {
+                        for (const fieldName of dataKeys) {
+                            const serde = allFields[fieldName].serde ?? (0, utils_1.fail)(`Missing serde for ${fieldName}`);
+                            serde.setOnEntity(data, row);
+                            // Make the field look not-dirty
+                            if (changedFields.includes(fieldName)) {
+                                instanceData.markFieldClean(fieldName);
+                            }
+                        }
                     }
                 }
             }
@@ -1249,8 +1367,9 @@ class EntityManager {
         return entities;
     }
     touch(entityOrEntities) {
-        for (const entity of (0, utils_1.toArray)(entityOrEntities))
-            (0, BaseEntity_1.getInstanceData)(entity).isTouched = true;
+        for (const entity of (0, utils_1.toArray)(entityOrEntities)) {
+            (0, BaseEntity_1.getInstanceData)(entity).markTouched();
+        }
     }
     async recalc(entityOrEntities) {
         // Look for async reactive fields
@@ -1327,9 +1446,16 @@ class EntityManager {
             // Run the beforeDelete hook before we unhook the entity
             const todos = (0, Todo_1.createTodos)(entities);
             await beforeDelete(this.ctx, todos);
-            // For all relations, unhook the entity from the other side
-            // (...we're using `concat` because `.push(...reallyBigArray)` with ~100k relations can blow the stack size
-            relationsToCleanup = relationsToCleanup.concat(entities.flatMap(index_1.getRelations));
+            // For all relations, unhook the entity from the other side; this append path is optimized for
+            // large deletes with ~100k relations by avoiding `flatMap` intermediates and `concat` copies.
+            for (const entity of entities) {
+                const relations = (0, index_1.getRelations)(entity);
+                const start = relationsToCleanup.length;
+                relationsToCleanup.length += relations.length;
+                for (let i = 0; i < relations.length; i++) {
+                    relationsToCleanup[start + i] = relations[i];
+                }
+            }
             entities = this.#pendingDeletes;
             this.#pendingDeletes = [];
         }
@@ -1385,9 +1511,9 @@ class EntityManager {
             return entities.filter((e) => e instanceof cstr && !e.isDeletedEntity);
         }
         if (this.#indexManager.shouldIndexType(entities.length)) {
-            this.#indexManager.enableIndexingForType(meta, entities);
+            this.#indexManager.enableIndexingForType(meta, entities, where);
             return (this.#indexManager
-                .findMatching(meta, where)
+                .findMatching(meta, entities, where)
                 // Still filter by `instanceof cstr` to handle subtyping
                 .filter((e) => e instanceof cstr && !e.isDeletedEntity));
         }
@@ -1487,7 +1613,8 @@ class EntityManager {
                     continue;
                 const { originalData: oldOriginalData, data: oldData } = oldInstanceData;
                 const newEntity = mapEntity(oldEntity);
-                const { originalData: newOriginalData, data: newData } = newEntity.__data;
+                const newInstanceData = newEntity.__data;
+                const { data: newData } = newInstanceData;
                 // for new entities, anything in `data` is changed and should be copied across. for existing entities, we
                 // only care about changed fields, which are enumerated by originalData
                 const maybeEntity = (value) => ((0, Entity_1.isEntity)(value) ? mapEntity(value) : value);
@@ -1495,7 +1622,7 @@ class EntityManager {
                 for (const field of fields) {
                     // copy over originalData so .changes is consistent across ems
                     if (field in oldOriginalData)
-                        newOriginalData[field] = maybeEntity(oldOriginalData[field]);
+                        newInstanceData.markFieldDirty(field, maybeEntity(oldOriginalData[field]));
                     newData[field] = maybeEntity(oldData[field]);
                 }
             }
@@ -1505,7 +1632,7 @@ class EntityManager {
             const newEntity = mapEntity(oldEntity);
             if (oldEntity.isDeletedEntity) {
                 // If the old entity was deleted, that should be persisted in the new em
-                newEntity.__data.markDeleted(newEntity);
+                newEntity.__data.markDeleted();
                 // deleted entities will fail if you try to `get` their relations, so skip them since they should be cleared
                 // out regardless
                 continue;
@@ -1650,11 +1777,11 @@ class EntityManager {
         }
     }
     /** Registers a newly-instantiated entity with our EntityManager; only called by #doCreate and hydrate. */
-    #doRegister(entity, id) {
+    #doRegister(entity, id, meta, skipDuplicateCheck = false) {
         // Keep our indexes up to date...
         const maybeId = id ?? entity.idTaggedMaybe;
         if (maybeId) {
-            if (this.findExistingInstance(maybeId) !== undefined) {
+            if (!skipDuplicateCheck && this.findExistingInstance(maybeId) !== undefined) {
                 throw new Error(`Entity ${entity} has a duplicate instance already loaded`);
             }
             this.#entitiesById.set(maybeId, entity);
@@ -1663,8 +1790,9 @@ class EntityManager {
             // Also register by the `a#1` style tagged string for new entities
             this.#entitiesById.set(entity.toTaggedString(), entity);
         }
+        (0, BaseEntity_1.getInstanceData)(entity).entityIndex = this.#entitiesArray.length;
         this.#entitiesArray.push(entity);
-        const meta = (0, index_1.getMetadata)(entity);
+        meta ??= (0, index_1.getMetadata)(entity);
         const set = this.#entitiesByTag.get(meta.tagName) ?? [];
         if (set.length === 0)
             this.#entitiesByTag.set(meta.tagName, set);
@@ -1678,8 +1806,6 @@ class EntityManager {
                 .join(", ");
             throw new Error(`More than ${this.entityLimit} entities have been instantiated (top entities: ${topTypes})`);
         }
-        // If indexing is enabled for this type, add it...
-        this.#indexManager.maybeIndexEntity(entity);
     }
 }
 exports.EntityManager = EntityManager;
@@ -1763,14 +1889,14 @@ async function validateReactiveRules(em, logger, todos, joinRowTodos) {
     const p1 = Object.values(todos).flatMap((todo) => {
         const entities = [...todo.inserts, ...todo.updates, ...todo.deletes];
         // Find each statically-declared reactive rule for the given entity type
-        const rules = (0, caches_1.getReactiveRules)(todo.metadata);
+        const rules = todo.metadata.reactiveRules;
         return rules.map((rule) => {
             // Of all changed entities of this type, how many specifically trigger this rule?
             const triggered = entities.filter((e) => {
                 // If the rule is for a different subtype, skip it
                 if (!(e instanceof rule.source))
                     return false;
-                // Any new-or-deleted entity fires every rule (getReactiveRules has already filtered out read-only)
+                // Any new-or-deleted entity fires every rule (reactiveRules has already filtered out read-only)
                 if (e.isNewEntity || e.isDeletedEntity)
                     return true;
                 // Otherwise see if the changed fields overlaps with the rule's fields
@@ -1786,22 +1912,26 @@ async function validateReactiveRules(em, logger, todos, joinRowTodos) {
         });
     });
     const p2 = Object.values(joinRowTodos).flatMap((todo) => {
-        const entities = [...todo.newRows, ...todo.deletedRows].flatMap((jr) => Object.values(jr.columns));
-        // Do the first side
-        const p1 = (0, caches_1.getReactiveRules)(todo.m2m.meta)
-            .filter((rule) => rule.fields.includes(todo.m2m.fieldName))
+        // For enum m2ms the columns include the enum's numeric id, so keep only the entity values.
+        const entities = [...todo.newRows, ...todo.deletedRows].flatMap((jr) => Object.values(jr.columns)).filter(Entity_1.isEntity);
+        // The owning-entity side (applies to both entity-to-entity and entity-to-enum m2ms).
+        const a = todo.m2m.meta
+            .reactiveRules.filter((rule) => rule.fields.includes(todo.m2m.fieldName))
             .map((rule) => {
             const triggered = entities.filter((e) => e instanceof todo.m2m.meta.cstr);
             return followAndQueue(triggered, rule);
         });
-        // And the second side
-        const p2 = (0, caches_1.getReactiveRules)(todo.m2m.otherMeta)
-            .filter((rule) => rule.fields.includes(todo.m2m.otherFieldName))
-            .map((rule) => {
-            const triggered = entities.filter((e) => e instanceof todo.m2m.otherMeta.cstr);
-            return followAndQueue(triggered, rule);
-        });
-        return [...p1, ...p2];
+        // The "other" side only exists for entity-to-entity m2ms; enums have no reverse field.
+        const { otherMeta, otherFieldName } = todo.m2m;
+        const b = otherMeta
+            ? otherMeta
+                .reactiveRules.filter((rule) => rule.fields.includes(otherFieldName))
+                .map((rule) => {
+                const triggered = entities.filter((e) => e instanceof otherMeta.cstr);
+                return followAndQueue(triggered, rule);
+            })
+            : [];
+        return [...a, ...b];
     });
     (0, utils_1.failIfAnyRejected)(await Promise.allSettled([...p1, ...p2]));
     // Now that we've found the fn+entities to run, run them and collect any errors
@@ -1883,7 +2013,11 @@ function entitiesFromTodos(entityTodos, joinRowTodos) {
     }
     for (const todo of Object.values(joinRowTodos)) {
         [...todo.newRows, ...todo.deletedRows].forEach((row) => {
-            Object.values(row.columns).forEach((entity) => entities.add(entity));
+            // For enum m2ms, a column value is the enum's numeric id rather than an entity.
+            Object.values(row.columns).forEach((value) => {
+                if ((0, Entity_1.isEntity)(value))
+                    entities.add(value);
+            });
         });
     }
     return [...entities];
@@ -2025,7 +2159,7 @@ function maybeBumpUpdatedAt(rm, todos, now) {
                 // it has changed. This is technically true, but this will break the oplock SQL generation,
                 // so force the field to be dirty.
                 const orm = (0, BaseEntity_1.getInstanceData)(e);
-                orm.originalData[updatedAt] = (0, fields_1.getField)(e, updatedAt);
+                orm.markFieldDirty(updatedAt, (0, fields_1.getField)(e, updatedAt));
                 const serde = todo.metadata.fields[updatedAt].serde;
                 orm.data[updatedAt] = serde.mapFromNow(now);
                 rm.queueDownstreamReactables(e, updatedAt);
@@ -2062,17 +2196,13 @@ function findConcreteMeta(maybeBaseMeta, row) {
             throw new Error(`${maybeBaseMeta.type} ${(0, index_1.tagId)(maybeBaseMeta, row.id)} must be instantiated via a subtype`);
         }
         // Look for the CTI __class from the driver telling us which subtype to instantiate
-        return maybeBaseMeta.subTypes.find((st) => st.type === row.__class) ?? maybeBaseMeta;
+        return maybeBaseMeta.subTypesByType.get(row.__class) ?? maybeBaseMeta;
     }
     else if (maybeBaseMeta.inheritanceType === "sti") {
         // Look for the STI discriminator value
         const baseMeta = (0, index_1.getBaseMeta)(maybeBaseMeta);
-        const field = baseMeta.fields[baseMeta.stiDiscriminatorField];
-        if (field.kind !== "enum")
-            throw new Error("Discriminator field must be an enum");
-        const columnName = field.serde.columns[0].columnName;
-        const value = row[columnName];
-        return baseMeta.subTypes.find((st) => st.stiDiscriminatorValue === value) ?? baseMeta;
+        const value = row[baseMeta.stiDiscriminatorColumnName];
+        return baseMeta.subTypesByStiValue.get(value) ?? baseMeta;
     }
     else {
         throw new Error("Unknown inheritance type");
@@ -2081,7 +2211,7 @@ function findConcreteMeta(maybeBaseMeta, row) {
 /** Sets the `Animal.type` enum to the right subtype value. */
 function setStiDiscriminatorValue(baseMeta, entity) {
     const typeName = entity.constructor.name;
-    const st = baseMeta.subTypes.find((st) => st.type === typeName);
+    const st = baseMeta.subTypesByType.get(typeName);
     if (st) {
         const field = baseMeta.fields[baseMeta.stiDiscriminatorField];
         const code = field.enumDetailType.findById(st.stiDiscriminatorValue).code;
@@ -2091,19 +2221,25 @@ function setStiDiscriminatorValue(baseMeta, entity) {
         entity[baseMeta.stiDiscriminatorField] = undefined;
     }
 }
-function findPendingFlushEntities(entities, hooksInvoked, pendingFlush, pendingHooks, alreadyRanHooks) {
-    for (const e of entities) {
-        if ((0, BaseEntity_1.getInstanceData)(e).pendingOperation !== "none") {
-            if (!hooksInvoked.has(e)) {
-                pendingHooks.add(e);
-            }
-            else {
-                alreadyRanHooks.add(e);
-            }
-            pendingFlush.add(e);
+function findPendingFlushEntities(maybePendingFlushEntities, hooksInvoked, pendingFlush, pendingHooks, alreadyRanHooks) {
+    for (const e of maybePendingFlushEntities) {
+        if ((0, BaseEntity_1.getInstanceData)(e).pendingOperation === "none") {
+            maybePendingFlushEntities.delete(e);
+            continue;
         }
+        if (!hooksInvoked.has(e)) {
+            pendingHooks.add(e);
+        }
+        else {
+            alreadyRanHooks.add(e);
+        }
+        pendingFlush.add(e);
     }
 }
+/** Returns true if the caller explicitly asked `find` to use SQL pagination. */
+function hasPaginationSettings(options) {
+    return "limit" in options || "offset" in options;
+}
 /** An error we throw to get knex to `ROLLBACK`, but then catch. */
 class InMemoryRollbackError extends Error {
 }