joist-core 2.1.0 → 2.2.0-next.10

package/build/EntityManager.js

@@ -29,7 +29,6 @@ const IndexManager_1 = require("./IndexManager");
 // We alias `Entity => EntityW` to denote "Entity wide" i.e. the non-narrowed Entity
 const loadBatchLoader_1 = require("./batchloaders/loadBatchLoader");
 const populateBatchLoader_1 = require("./batchloaders/populateBatchLoader");
-const caches_1 = require("./caches");
 const config_1 = require("./config");
 const configure_1 = require("./configure");
 const findByUniqueDataLoader_1 = require("./dataloaders/findByUniqueDataLoader");
@@ -37,17 +36,20 @@ const findCountDataLoader_1 = require("./dataloaders/findCountDataLoader");
 const findDataLoader_1 = require("./dataloaders/findDataLoader");
 const findIdsDataLoader_1 = require("./dataloaders/findIdsDataLoader");
 const findOrCreateDataLoader_1 = require("./dataloaders/findOrCreateDataLoader");
+const findPaginatedDataLoader_1 = require("./dataloaders/findPaginatedDataLoader");
 const Entity_1 = require("./Entity");
 const FlushLock_1 = require("./FlushLock");
 const index_1 = require("./index");
 const IsLoadedCache_1 = require("./IsLoadedCache");
 const JoinRows_1 = require("./JoinRows");
+const loadHints_1 = require("./loadHints");
 const newEntity_1 = require("./newEntity");
 const newTestInstance_1 = require("./newTestInstance");
 const PluginManager_1 = require("./PluginManager");
 const ReactionsManager_1 = require("./ReactionsManager");
 const reactiveHints_1 = require("./reactiveHints");
 const relations_1 = require("./relations");
+const AsyncProperty_1 = require("./relations/AsyncProperty");
 const hasAsyncMethod_1 = require("./relations/hasAsyncMethod");
 const RecursiveCollection_1 = require("./relations/RecursiveCollection");
 const Todo_1 = require("./Todo");
@@ -86,6 +88,8 @@ class EntityManager {
     txn;
     entityLimit = defaultEntityLimit;
     #entitiesArray = [];
+    // Incrementally track dirty entities so we don't have to scan `#entityArray` during flush
+    #maybePendingFlushEntities = new Set();
     // Indexes the currently loaded entities by their tagged ids and `toTaggedString` ids (i.e. `a#`). This fixes
     // real-world performance issues where `findExistingInstance` scanning `#entities` was an `O(n^2)`.
     #entitiesById = new Map();
@@ -153,6 +157,12 @@ class EntityManager {
             indexManager: this.#indexManager,
             isLoadedCache: this.#isLoadedCache,
             pluginManager,
+            markMaybePending(entity) {
+                em.#maybePendingFlushEntities.add(entity);
+            },
+            unmarkMaybePending(entity) {
+                em.#maybePendingFlushEntities.delete(entity);
+            },
             isMerging(entity) {
                 return em.#merging?.has(entity) ?? false;
             },
@@ -246,9 +256,9 @@ class EntityManager {
     async find(type, where, options) {
         const { populate, ...rest } = options || {};
         const settings = { where, ...rest };
-        const result = await (0, findDataLoader_1.findDataLoader)(this, type, settings, populate)
-            .load(settings)
-            .catch(function find(err) {
+        const result = await (hasPaginationSettings(rest)
+            ? (0, findPaginatedDataLoader_1.findPaginatedDataLoader)(this, type, settings, populate)
+            : (0, findDataLoader_1.findDataLoader)(this, type, settings, populate)).catch(function find(err) {
             throw appendStack(err, new Error());
         });
         if (populate) {
@@ -256,30 +266,52 @@ class EntityManager {
         }
         return result;
     }
-    async findPaginated(type, where, options) {
-        const { populate, limit, offset, ...rest } = options || {};
-        const meta = (0, index_1.getMetadata)(type);
-        const query = (0, index_1.parseFindQuery)(meta, where, rest);
-        const rows = await this.executeFind(meta, "findPaginated", query, { limit, offset });
-        // check row limit
-        const result = this.hydrate(type, rows);
-        if (populate) {
-            await this.populate(result, populate);
-        }
-        return result;
-    }
+    /** Runs the post-parse find pipeline: plugins mutate the logical AST, then Joist optimizes/prunes before SQL. */
     async executeFind(meta, operation, parsed, settings) {
+        const { checkLimit, findSettings } = this.prepareFind(meta, operation, parsed, settings);
+        return this.executePreparedFind(meta, operation, parsed, findSettings, checkLimit);
+    }
+    /** Executes a query that has already had find hooks and optimizations applied. */
+    async executePreparedFind(meta, operation, parsed, findSettings, checkLimit) {
         const { pluginManager } = getEmInternalApi(this);
-        pluginManager.beforeFind(meta, operation, parsed, settings);
-        const rows = await this.driver.executeFind(this, parsed, settings);
+        const rows = await this.driver.executeFind(this, parsed, findSettings);
+        // Check by default unless explicitly disabled or the caller removed the LIMIT via `limit: undefined`
+        const shouldCheck = checkLimit ?? !("limit" in findSettings && findSettings.limit === undefined);
+        if (shouldCheck && rows.length >= this.entityLimit) {
+            throw new Error(`Query returned more than ${this.entityLimit} entityLimit rows`);
+        }
         pluginManager.afterFind(meta, operation, rows);
         return rows;
     }
-    async findGql(type, where, options) {
-        return this.find(type, where, options);
+    /**
+     * Runs pre-SQL find hooks and optimizations against a parsed query.
+     *
+     * This allows plugins to see "pre-batched" / "logical" query ASTs, instead of our
+     * more complicated `_find` batched queries. The flow would be:
+     *
+     * - A loader calls `prepareFind(originalQuery)`
+     * - Plugins inspect/modify the query as/if needed
+     * - The loader crafts a new, more complicated query that embeds the originalQuery
+     * - The loader calls `executePreparedFind` with the 2nd query
+     */
+    prepareFind(meta, operation, parsed, settings) {
+        const { checkLimit, ...findSettings } = settings;
+        const { pluginManager } = getEmInternalApi(this);
+        // Plugins may mutate the settings object, so return the post-hook version that loaders must reuse.
+        pluginManager.beforeFind(meta, operation, parsed, findSettings);
+        (0, index_1.optimizeCollectionJoins)(parsed, settings);
+        return { checkLimit, findSettings };
     }
-    async findGqlPaginated(type, where, options) {
-        return this.findPaginated(type, where, options);
+    async findGql(type, where, options) {
+        if (!options) {
+            return this.find(type, where);
+        }
+        const normalized = { ...options };
+        if ("limit" in normalized)
+            normalized.limit = normalized.limit ?? undefined;
+        if ("offset" in normalized)
+            normalized.offset = normalized.offset ?? undefined;
+        return this.find(type, where, normalized);
     }
     async findOne(type, where, options) {
         const list = await this.find(type, where, options);
@@ -337,9 +369,7 @@ class EntityManager {
      */
     async findCount(type, where, options = {}) {
         const settings = { where, ...options };
-        let count = await (0, findCountDataLoader_1.findCountDataLoader)(this, type, settings)
-            .load(settings)
-            .catch(function findCount(err) {
+        let count = await (0, findCountDataLoader_1.findCountDataLoader)(this, type, settings).catch(function findCount(err) {
             throw appendStack(err, new Error());
         });
         // If the user is do "count all", we can adjust the number up/down based on
@@ -371,9 +401,7 @@ class EntityManager {
      */
     async findIds(type, where, options = {}) {
         const settings = { where, ...options };
-        return (0, findIdsDataLoader_1.findIdsDataLoader)(this, type, settings)
-            .load(settings)
-            .catch(function findIds(err) {
+        return (0, findIdsDataLoader_1.findIdsDataLoader)(this, type, settings).catch(function findIds(err) {
             throw appendStack(err, new Error());
         });
     }
@@ -662,48 +690,86 @@ class EntityManager {
     }
     async loadAll(type, _ids, hint) {
         const meta = (0, index_1.getMetadata)(type);
-        const ids = _ids.map((id) => (0, index_1.tagId)(meta, id));
-        const idsToLoad = ids.filter((id) => !this.findExistingInstance(id));
-        if (idsToLoad.length > 0) {
+        // Use pre-allocated arrays/for loops instead of `.filter`s since this can be a hot spot
+        const ids = new Array(_ids.length);
+        const entities = new Array(_ids.length);
+        let idsToLoad;
+        let positionsToLoad;
+        for (let i = 0; i < _ids.length; i++) {
+            const id = (0, index_1.tagId)(meta, _ids[i]);
+            ids[i] = id;
+            const entity = this.findExistingInstance(id);
+            if (entity) {
+                entities[i] = entity;
+            }
+            else {
+                (idsToLoad ??= []).push(id);
+                (positionsToLoad ??= []).push(i);
+            }
+        }
+        if (idsToLoad && idsToLoad.length > 0) {
             await (0, loadBatchLoader_1.loadBatchLoader)(this, meta)
                 .loadAll(idsToLoad.map((id) => ({ taggedId: id, hint })))
                 .catch(function loadAll(err) {
                 throw appendStack(err, new Error());
             });
+            for (const i of positionsToLoad) {
+                entities[i] = this.findExistingInstance(ids[i]);
+            }
         }
-        const entities = [];
-        for (const id of ids) {
-            const entity = this.findExistingInstance(id);
-            if (entity)
-                entities.push(entity);
+        let idsNotFound;
+        for (let i = 0; i < entities.length; i++) {
+            if (entities[i] === undefined) {
+                (idsNotFound ??= []).push(ids[i]);
+            }
         }
-        if (entities.length !== ids.length) {
-            const idsNotFound = ids.filter((_, i) => entities[i] === undefined);
+        if (idsNotFound) {
             throw new NotFoundError(`${idsNotFound.join(",")} were not found`);
         }
+        const loadedEntities = entities;
         if (hint) {
-            await this.populate(entities, hint);
+            await this.populate(loadedEntities, hint);
         }
         if (meta.inheritanceType === "sti" && meta.baseType) {
-            const wrongType = entities.filter((e) => !(e instanceof meta.cstr));
+            const wrongType = loadedEntities.filter((e) => !(e instanceof meta.cstr));
             if (wrongType.length > 0) {
                 throw new Error(`${wrongType.join(", ")} were not of type ${meta.cstr.name}`);
             }
         }
-        return entities;
+        return loadedEntities;
     }
     async loadAllIfExists(type, _ids, hint) {
         const meta = (0, index_1.getMetadata)(type);
-        const ids = _ids.map((id) => (0, index_1.tagId)(meta, id));
-        const idsToLoad = ids.filter((id) => !this.findExistingInstance(id));
-        if (idsToLoad.length > 0) {
+        // Use pre-allocated arrays/for loops instead of `.filter`s since this can be a hot spot
+        const ids = new Array(_ids.length);
+        const entities = [];
+        let idsToLoad;
+        // Ensure the ids are tagged, and find any not-yet-loaded
+        for (let i = 0; i < _ids.length; i++) {
+            const id = (0, index_1.tagId)(meta, _ids[i]);
+            ids[i] = id;
+            const entity = this.findExistingInstance(id);
+            if (entity) {
+                entities.push(entity);
+            }
+            else {
+                (idsToLoad ??= []).push(id);
+            }
+        }
+        if (idsToLoad && idsToLoad.length > 0) {
             await (0, loadBatchLoader_1.loadBatchLoader)(this, meta)
                 .loadAll(idsToLoad.map((id) => ({ taggedId: id, hint })))
                 .catch(function loadAllIfExists(err) {
                 throw appendStack(err, new Error());
             });
+            // Now that everything is loaded, recalc `entities`
+            entities.length = 0;
+            for (const id of ids) {
+                const entity = this.findExistingInstance(id);
+                if (entity)
+                    entities.push(entity);
+            }
         }
-        const entities = ids.map((id) => this.findExistingInstance(id)).filter(Boolean);
         if (hint) {
             await this.populate(entities, hint);
         }
@@ -754,6 +820,12 @@ class EntityManager {
         if (list.length === 0) {
             return !fn ? entityOrList : fn(entityOrList);
         }
+        // Avoid building a HintTree/batchloader for persisted entities when everything is already loaded; in the phase 9
+        // benchmark this moved already-loaded populates from ~1.10ms to ~0.89ms, and nested already-loaded populates from
+        // ~2.39ms to ~1.94ms.
+        if (!opts.forceReload && list.every((entity) => !entity.isNewEntity && (0, loadHints_1.isLoadedForPopulate)(entity, hintOpt))) {
+            return fn ? fn(entityOrList) : entityOrList;
+        }
         const meta = (0, index_1.getMetadata)(list[0]);
         if (this.#preloader) {
             // If we can preload, prevent promise deadlocking by one large-batch preload populate (which can't have
@@ -802,7 +874,7 @@ class EntityManager {
     delete(entityOrArray) {
         for (const entity of (0, utils_1.toArray)(entityOrArray)) {
             // Early return if already deleted.
-            const alreadyMarked = (0, BaseEntity_1.getInstanceData)(entity).markDeleted(entity);
+            const alreadyMarked = (0, BaseEntity_1.getInstanceData)(entity).markDeleted();
             if (!alreadyMarked)
                 continue;
             // Any derived fields that read this entity will need recalc-d
@@ -878,11 +950,11 @@ class EntityManager {
             const createdThenDeleted = new Set();
             // We'll only invoke hooks once/entity (the 1st time that entity goes through runHooksOnPendingEntities)
             const hooksInvoked = new Set();
-            // Make sure two ReactiveQueryFields don't ping-pong each other forever
+            // Make sure two AsyncReactiveFields don't ping-pong each other forever
             let hookLoops = 0;
             let now = getNow();
             const suppressedDefaultTypeErrors = [];
-            // Make a lambda that we can invoke multiple times, if we loop for ReactiveQueryFields
+            // Make a lambda that we can invoke multiple times, if we loop for AsyncReactiveFields
             const runHooksOnPendingEntities = async () => {
                 if (hookLoops++ >= 10)
                     throw new Error("runHooksOnPendingEntities has ran 10 iterations, aborting");
@@ -898,19 +970,19 @@ class EntityManager {
                 const pendingHooks = new Set();
                 // Subset of pendingFlush entities that had hooks invoked in a prior `runHooksOnPendingEntities`
                 const alreadyRanHooks = new Set();
-                findPendingFlushEntities(this.entities, hooksInvoked, pendingFlush, pendingHooks, alreadyRanHooks);
-                // If we're re-looping for ReactiveQueryField, make sure to bump updatedAt
+                findPendingFlushEntities(this.#maybePendingFlushEntities, hooksInvoked, pendingFlush, pendingHooks, alreadyRanHooks);
+                // If we're re-looping for AsyncReactiveField, make sure to bump updatedAt
                 // each time, so that for an INSERT-then-UPDATE the triggers don't think the
                 // UPDATE forgot to self-bump updatedAt, and then "helpfully" bump it for us.
                 if (alreadyRanHooks.size > 0) {
-                    maybeBumpUpdatedAt((0, Todo_1.createTodos)([...alreadyRanHooks]), now);
+                    maybeBumpUpdatedAt(this.#rm, (0, Todo_1.createTodos)([...alreadyRanHooks]), now);
                 }
                 // Run hooks in a series of loops until things "settle down"
                 while (pendingHooks.size > 0) {
                     await this.#fl.allowWrites(async () => {
                         let todos = (0, Todo_1.createTodos)([...pendingHooks]);
                         await (0, defaults_1.setAsyncDefaults)(suppressedDefaultTypeErrors, this.ctx, Todo_1.Todo.groupInsertsByTypeAndSubType(todos));
-                        maybeBumpUpdatedAt(todos, now);
+                        maybeBumpUpdatedAt(this.#rm, todos, now);
                         // Run our hooks
                         for (const group of maybeSetupHookOrdering(todos)) {
                             await beforeCreate(this.ctx, group);
@@ -937,7 +1009,7 @@ class EntityManager {
                             hooksInvoked.add(e);
                         pendingHooks.clear();
                         // See if the hooks mutated any new, not-yet-hooksInvoked entities
-                        findPendingFlushEntities(this.entities, hooksInvoked, pendingFlush, pendingHooks, alreadyRanHooks);
+                        findPendingFlushEntities(this.#maybePendingFlushEntities, hooksInvoked, pendingFlush, pendingHooks, alreadyRanHooks);
                         // The final run of recalcPendingReactables could have left us with pending type errors and no entities in
                         // pendingHooks.  If so, we need to re-run recalcPendingTypeErrors to get those errors to transition into
                         // suppressed errors so that we will fail after simpleValidation.
@@ -955,24 +1027,30 @@ class EntityManager {
                     return !createThenDelete;
                 });
             };
-            const runValidation = async (entityTodos, joinRowTodos) => {
+            const { pluginManager } = getEmInternalApi(this);
+            const runValidation = async (entityTodos, joinRowTodos, validate) => {
+                const changedEntities = entitiesFromTodos(entityTodos, joinRowTodos);
                 try {
                     this.#isValidating = true;
-                    // Run simple rules first b/c it includes not-null/required rules, so that then when we run
-                    // `validateReactiveRules` next, the app's lambdas won't see fundamentally invalid entities & NPE.
-                    await validateSimpleRules(entityTodos);
-                    // After we've let any "author is not set" simple rules fail before prematurely throwing
-                    // the "of course that caused an NPE" `TypeError`s, if all the authors *were* valid/set,
-                    // and we still have TypeErrors (from derived valeus), they were real, unrelated errors
-                    // that the user should see.
-                    if (suppressedDefaultTypeErrors.length > 0)
-                        throw suppressedDefaultTypeErrors[0];
-                    await validateReactiveRules(this, this.#rm.logger, entityTodos, joinRowTodos);
+                    await pluginManager.beforeValidate(changedEntities);
+                    if (validate) {
+                        // Run simple rules first b/c it includes not-null/required rules, so that then when we run
+                        // `validateReactiveRules` next, the app's lambdas won't see fundamentally invalid entities & NPE.
+                        await validateSimpleRules(entityTodos);
+                        // After we've let any "author is not set" simple rules fail before prematurely throwing
+                        // the "of course that caused an NPE" `TypeError`s, if all the authors *were* valid/set,
+                        // and we still have TypeErrors (from derived valeus), they were real, unrelated errors
+                        // that the user should see.
+                        if (suppressedDefaultTypeErrors.length > 0)
+                            throw suppressedDefaultTypeErrors[0];
+                        await validateReactiveRules(this, this.#rm.logger, entityTodos, joinRowTodos);
+                        await afterValidation(this.ctx, entityTodos);
+                    }
+                    await pluginManager.afterValidate(changedEntities);
                 }
                 finally {
                     this.#isValidating = false;
                 }
-                await afterValidation(this.ctx, entityTodos);
             };
             // Run hooks (in iterative loops if hooks mutate new entities) on pending entities
             let entitiesToFlush = await runHooksOnPendingEntities();
@@ -982,13 +1060,10 @@ class EntityManager {
             // the full set of entities that will be INSERT/UPDATE/DELETE-d in the database.
             let entityTodos = (0, Todo_1.createTodos)(entitiesToFlush);
             let joinRowTodos = (0, Todo_1.combineJoinRows)(this.#joinRows);
-            if (!skipValidation) {
-                await runValidation(entityTodos, joinRowTodos);
-            }
+            await runValidation(entityTodos, joinRowTodos, !skipValidation);
             this.#rm.throwIfAnySuppressedTypeErrors();
             if (suppressedDefaultTypeErrors.length > 0)
                 throw suppressedDefaultTypeErrors[0];
-            const { pluginManager } = getEmInternalApi(this);
             if (Object.keys(entityTodos).length > 0 || Object.keys(joinRowTodos).length > 0) {
                 // The driver will handle the right thing if we're already in an existing transaction.
                 await this.driver.transaction(this, async () => {
@@ -1009,7 +1084,7 @@ class EntityManager {
                             // Actually do the recalc
                             await this.#fl.allowWrites(async () => {
                                 await this.#rm.recalcPendingReactables("reactiveQueries");
-                                // If any ReactiveFields depended on ReactiveQueryFields, go ahead and calc those now
+                                // If any ReactiveFields depended on AsyncReactiveFields, go ahead and calc those now
                                 await this.#rm.recalcPendingReactables("reactables");
                             });
                             // Advance `now` so that our triggers don't think our UPDATEs are forgetting to self-bump
@@ -1022,7 +1097,7 @@ class EntityManager {
                             // Recreate `entityTodos` against the only-the-just-changed entities
                             entityTodos = (0, Todo_1.createTodos)(entitiesToFlush);
                             joinRowTodos = (0, Todo_1.combineJoinRows)(this.#joinRows);
-                            await runValidation(entityTodos, joinRowTodos);
+                            await runValidation(entityTodos, joinRowTodos, true);
                             this.#rm.throwIfAnySuppressedTypeErrors();
                         }
                         else {
@@ -1044,6 +1119,11 @@ class EntityManager {
                     if (e.isNewEntity && !e.isDeletedEntity)
                         this.#entitiesById.set(e.idTagged, e);
                     (0, BaseEntity_1.getInstanceData)(e).resetAfterFlushed();
+                    // Reset AsyncQueryProperties since DB state may have changed
+                    for (const rel of Object.values((0, BaseEntity_1.getInstanceData)(e).relations)) {
+                        if (rel instanceof AsyncProperty_1.AsyncPropertyImpl)
+                            rel.resetAfterFlush();
+                    }
                 }
                 // Update the joinRows refs to reflect the new state
                 for (const joinRow of Object.values(joinRowTodos)) {
@@ -1196,10 +1276,13 @@ class EntityManager {
      */
     hydrate(type, rows, options) {
         const maybeBaseMeta = (0, index_1.getMetadata)(type);
+        const taggedIdPrefix = `${maybeBaseMeta.tagName}:`;
+        const overwriteExisting = options?.overwriteExisting === true;
         let i = 0;
         const entities = new Array(rows.length);
         for (const row of rows) {
-            const taggedId = (0, index_1.keyToTaggedId)(maybeBaseMeta, row["id"]) || (0, utils_1.fail)("No id column was available");
+            const id = row["id"];
+            const taggedId = id === undefined || id === null ? (0, utils_1.fail)("No id column was available") : `${taggedIdPrefix}${id}`;
             // See if this is already in our UoW
             let entity = this.findExistingInstance(taggedId);
             if (!entity) {
@@ -1208,23 +1291,36 @@ class EntityManager {
                 // Pass id as a hint that we're in hydrate mode
                 entity = (0, newEntity_1.newEntity)(this, (0, index_1.asConcreteCstr)(meta.cstr), false);
                 (0, BaseEntity_1.getInstanceData)(entity).row = row;
-                this.#doRegister(entity, taggedId);
+                this.#doRegister(entity, taggedId, meta, true);
             }
-            else if (options?.overwriteExisting === true) {
+            else if (overwriteExisting) {
                 // Usually if the entity already exists, we don't write over it, but in this case we assume that
                 // `EntityManager.refresh` is telling us to explicitly load the latest data.
                 // First swap out the old row with the new row
-                (0, BaseEntity_1.getInstanceData)(entity).row = row;
+                const instanceData = (0, BaseEntity_1.getInstanceData)(entity);
+                instanceData.row = row;
                 // And then only refresh the data keys that have already been serde-d from rows
                 // (this keeps us from deserializing data out of rows that we don't need).
-                const { data, originalData } = (0, BaseEntity_1.getInstanceData)(entity);
-                const changedFields = entity.changes.fieldsWithoutRelations;
-                for (const fieldName of Object.keys(data)) {
-                    const serde = (0, index_1.getMetadata)(entity).allFields[fieldName].serde ?? (0, utils_1.fail)(`Missing serde for ${fieldName}`);
-                    serde.setOnEntity(data, row);
-                    // Make the field look not-dirty
-                    if (changedFields.includes(fieldName)) {
-                        delete originalData[fieldName];
+                const { data } = instanceData;
+                const dataKeys = Object.keys(data);
+                if (dataKeys.length > 0) {
+                    const allFields = (0, index_1.getMetadata)(entity).allFields;
+                    const changedFields = entity.changes.fieldsWithoutRelations;
+                    if (changedFields.length === 0) {
+                        for (const fieldName of dataKeys) {
+                            const serde = allFields[fieldName].serde ?? (0, utils_1.fail)(`Missing serde for ${fieldName}`);
+                            serde.setOnEntity(data, row);
+                        }
+                    }
+                    else {
+                        for (const fieldName of dataKeys) {
+                            const serde = allFields[fieldName].serde ?? (0, utils_1.fail)(`Missing serde for ${fieldName}`);
+                            serde.setOnEntity(data, row);
+                            // Make the field look not-dirty
+                            if (changedFields.includes(fieldName)) {
+                                instanceData.markFieldClean(fieldName);
+                            }
+                        }
                     }
                 }
             }
@@ -1233,8 +1329,9 @@ class EntityManager {
         return entities;
     }
     touch(entityOrEntities) {
-        for (const entity of (0, utils_1.toArray)(entityOrEntities))
-            (0, BaseEntity_1.getInstanceData)(entity).isTouched = true;
+        for (const entity of (0, utils_1.toArray)(entityOrEntities)) {
+            (0, BaseEntity_1.getInstanceData)(entity).markTouched();
+        }
     }
     async recalc(entityOrEntities) {
         // Look for async reactive fields
@@ -1251,7 +1348,7 @@ class EntityManager {
             (0, fields_1.setField)(entity, field.fieldName, entity[field.fieldName]);
         }));
         // `.load()` recalculated the immediate relations, go ahead and recalc any downstream reactables.
-        // We'll still defer ReactiveQueryFields to the em.flush loop.
+        // We'll still defer AsyncReactiveFields to the em.flush loop.
         await this.#rm.recalcPendingReactables("reactables");
     }
     beforeBegin(fn) {
@@ -1311,9 +1408,16 @@ class EntityManager {
             // Run the beforeDelete hook before we unhook the entity
             const todos = (0, Todo_1.createTodos)(entities);
             await beforeDelete(this.ctx, todos);
-            // For all relations, unhook the entity from the other side
-            // (...we're using `concat` because `.push(...reallyBigArray)` with ~100k relations can blow the stack size
-            relationsToCleanup = relationsToCleanup.concat(entities.flatMap(index_1.getRelations));
+            // For all relations, unhook the entity from the other side; this append path is optimized for
+            // large deletes with ~100k relations by avoiding `flatMap` intermediates and `concat` copies.
+            for (const entity of entities) {
+                const relations = (0, index_1.getRelations)(entity);
+                const start = relationsToCleanup.length;
+                relationsToCleanup.length += relations.length;
+                for (let i = 0; i < relations.length; i++) {
+                    relationsToCleanup[start + i] = relations[i];
+                }
+            }
             entities = this.#pendingDeletes;
             this.#pendingDeletes = [];
         }
@@ -1369,7 +1473,7 @@ class EntityManager {
             return entities.filter((e) => e instanceof cstr && !e.isDeletedEntity);
         }
         if (this.#indexManager.shouldIndexType(entities.length)) {
-            this.#indexManager.enableIndexingForType(meta, entities);
+            this.#indexManager.enableIndexingForType(meta, entities, where);
             return (this.#indexManager
                 .findMatching(meta, where)
                 // Still filter by `instanceof cstr` to handle subtyping
@@ -1471,7 +1575,8 @@ class EntityManager {
                     continue;
                 const { originalData: oldOriginalData, data: oldData } = oldInstanceData;
                 const newEntity = mapEntity(oldEntity);
-                const { originalData: newOriginalData, data: newData } = newEntity.__data;
+                const newInstanceData = newEntity.__data;
+                const { data: newData } = newInstanceData;
                 // for new entities, anything in `data` is changed and should be copied across. for existing entities, we
                 // only care about changed fields, which are enumerated by originalData
                 const maybeEntity = (value) => ((0, Entity_1.isEntity)(value) ? mapEntity(value) : value);
@@ -1479,7 +1584,7 @@ class EntityManager {
                 for (const field of fields) {
                     // copy over originalData so .changes is consistent across ems
                     if (field in oldOriginalData)
-                        newOriginalData[field] = maybeEntity(oldOriginalData[field]);
+                        newInstanceData.markFieldDirty(field, maybeEntity(oldOriginalData[field]));
                     newData[field] = maybeEntity(oldData[field]);
                 }
             }
@@ -1489,7 +1594,7 @@ class EntityManager {
             const newEntity = mapEntity(oldEntity);
             if (oldEntity.isDeletedEntity) {
                 // If the old entity was deleted, that should be persisted in the new em
-                newEntity.__data.markDeleted(newEntity);
+                newEntity.__data.markDeleted();
                 // deleted entities will fail if you try to `get` their relations, so skip them since they should be cleared
                 // out regardless
                 continue;
@@ -1634,11 +1739,11 @@ class EntityManager {
         }
     }
     /** Registers a newly-instantiated entity with our EntityManager; only called by #doCreate and hydrate. */
-    #doRegister(entity, id) {
+    #doRegister(entity, id, meta, skipDuplicateCheck = false) {
         // Keep our indexes up to date...
         const maybeId = id ?? entity.idTaggedMaybe;
         if (maybeId) {
-            if (this.findExistingInstance(maybeId) !== undefined) {
+            if (!skipDuplicateCheck && this.findExistingInstance(maybeId) !== undefined) {
                 throw new Error(`Entity ${entity} has a duplicate instance already loaded`);
             }
             this.#entitiesById.set(maybeId, entity);
@@ -1648,7 +1753,7 @@ class EntityManager {
             this.#entitiesById.set(entity.toTaggedString(), entity);
         }
         this.#entitiesArray.push(entity);
-        const meta = (0, index_1.getMetadata)(entity);
+        meta ??= (0, index_1.getMetadata)(entity);
         const set = this.#entitiesByTag.get(meta.tagName) ?? [];
         if (set.length === 0)
             this.#entitiesByTag.set(meta.tagName, set);
@@ -1747,14 +1852,14 @@ async function validateReactiveRules(em, logger, todos, joinRowTodos) {
     const p1 = Object.values(todos).flatMap((todo) => {
         const entities = [...todo.inserts, ...todo.updates, ...todo.deletes];
         // Find each statically-declared reactive rule for the given entity type
-        const rules = (0, caches_1.getReactiveRules)(todo.metadata);
+        const rules = todo.metadata.reactiveRules;
         return rules.map((rule) => {
             // Of all changed entities of this type, how many specifically trigger this rule?
             const triggered = entities.filter((e) => {
                 // If the rule is for a different subtype, skip it
                 if (!(e instanceof rule.source))
                     return false;
-                // Any new-or-deleted entity fires every rule (getReactiveRules has already filtered out read-only)
+                // Any new-or-deleted entity fires every rule (reactiveRules has already filtered out read-only)
                 if (e.isNewEntity || e.isDeletedEntity)
                     return true;
                 // Otherwise see if the changed fields overlaps with the rule's fields
@@ -1772,15 +1877,15 @@ async function validateReactiveRules(em, logger, todos, joinRowTodos) {
     const p2 = Object.values(joinRowTodos).flatMap((todo) => {
         const entities = [...todo.newRows, ...todo.deletedRows].flatMap((jr) => Object.values(jr.columns));
         // Do the first side
-        const p1 = (0, caches_1.getReactiveRules)(todo.m2m.meta)
-            .filter((rule) => rule.fields.includes(todo.m2m.fieldName))
+        const p1 = todo.m2m.meta
+            .reactiveRules.filter((rule) => rule.fields.includes(todo.m2m.fieldName))
             .map((rule) => {
             const triggered = entities.filter((e) => e instanceof todo.m2m.meta.cstr);
             return followAndQueue(triggered, rule);
         });
         // And the second side
-        const p2 = (0, caches_1.getReactiveRules)(todo.m2m.otherMeta)
-            .filter((rule) => rule.fields.includes(todo.m2m.otherFieldName))
+        const p2 = todo.m2m.otherMeta
+            .reactiveRules.filter((rule) => rule.fields.includes(todo.m2m.otherFieldName))
             .map((rule) => {
             const triggered = entities.filter((e) => e instanceof todo.m2m.otherMeta.cstr);
             return followAndQueue(triggered, rule);
@@ -1859,6 +1964,19 @@ function beforeUpdate(ctx, todos) {
 function afterValidation(ctx, todos) {
     return runHookOnTodos(ctx, "afterValidation", todos, ["inserts", "updates"]);
 }
+/** Collects changed entities from flush todos, i.e. m2m endpoint entities. */
+function entitiesFromTodos(entityTodos, joinRowTodos) {
+    const entities = new Set();
+    for (const todo of Object.values(entityTodos)) {
+        [...todo.inserts, ...todo.updates, ...todo.deletes].forEach((entity) => entities.add(entity));
+    }
+    for (const todo of Object.values(joinRowTodos)) {
+        [...todo.newRows, ...todo.deletedRows].forEach((row) => {
+            Object.values(row.columns).forEach((entity) => entities.add(entity));
+        });
+    }
+    return [...entities];
+}
 function beforeCommit(ctx, entities) {
     return runHook(ctx, "beforeCommit", [...entities]);
 }
@@ -1986,7 +2104,7 @@ function getCascadeDeleteRelations(entity) {
 function isCustomRelation(r) {
     return r instanceof index_1.CustomCollection || r instanceof index_1.CustomReference || r instanceof relations_1.ReactiveReferenceImpl;
 }
-function maybeBumpUpdatedAt(todos, now) {
+function maybeBumpUpdatedAt(rm, todos, now) {
     for (const todo of Object.values(todos)) {
         const { updatedAt } = todo.metadata.timestampFields ?? {};
         if (updatedAt) {
@@ -1996,9 +2114,10 @@ function maybeBumpUpdatedAt(todos, now) {
                 // it has changed. This is technically true, but this will break the oplock SQL generation,
                 // so force the field to be dirty.
                 const orm = (0, BaseEntity_1.getInstanceData)(e);
-                orm.originalData[updatedAt] = (0, fields_1.getField)(e, updatedAt);
+                orm.markFieldDirty(updatedAt, (0, fields_1.getField)(e, updatedAt));
                 const serde = todo.metadata.fields[updatedAt].serde;
                 orm.data[updatedAt] = serde.mapFromNow(now);
+                rm.queueDownstreamReactables(e, updatedAt);
             }
         }
     }
@@ -2032,17 +2151,13 @@ function findConcreteMeta(maybeBaseMeta, row) {
             throw new Error(`${maybeBaseMeta.type} ${(0, index_1.tagId)(maybeBaseMeta, row.id)} must be instantiated via a subtype`);
         }
         // Look for the CTI __class from the driver telling us which subtype to instantiate
-        return maybeBaseMeta.subTypes.find((st) => st.type === row.__class) ?? maybeBaseMeta;
+        return maybeBaseMeta.subTypesByType.get(row.__class) ?? maybeBaseMeta;
     }
     else if (maybeBaseMeta.inheritanceType === "sti") {
         // Look for the STI discriminator value
         const baseMeta = (0, index_1.getBaseMeta)(maybeBaseMeta);
-        const field = baseMeta.fields[baseMeta.stiDiscriminatorField];
-        if (field.kind !== "enum")
-            throw new Error("Discriminator field must be an enum");
-        const columnName = field.serde.columns[0].columnName;
-        const value = row[columnName];
-        return baseMeta.subTypes.find((st) => st.stiDiscriminatorValue === value) ?? baseMeta;
+        const value = row[baseMeta.stiDiscriminatorColumnName];
+        return baseMeta.subTypesByStiValue.get(value) ?? baseMeta;
     }
     else {
         throw new Error("Unknown inheritance type");
@@ -2051,7 +2166,7 @@ function findConcreteMeta(maybeBaseMeta, row) {
 /** Sets the `Animal.type` enum to the right subtype value. */
 function setStiDiscriminatorValue(baseMeta, entity) {
     const typeName = entity.constructor.name;
-    const st = baseMeta.subTypes.find((st) => st.type === typeName);
+    const st = baseMeta.subTypesByType.get(typeName);
     if (st) {
         const field = baseMeta.fields[baseMeta.stiDiscriminatorField];
         const code = field.enumDetailType.findById(st.stiDiscriminatorValue).code;
@@ -2061,19 +2176,25 @@ function setStiDiscriminatorValue(baseMeta, entity) {
         entity[baseMeta.stiDiscriminatorField] = undefined;
     }
 }
-function findPendingFlushEntities(entities, hooksInvoked, pendingFlush, pendingHooks, alreadyRanHooks) {
-    for (const e of entities) {
-        if ((0, BaseEntity_1.getInstanceData)(e).pendingOperation !== "none") {
-            if (!hooksInvoked.has(e)) {
-                pendingHooks.add(e);
-            }
-            else {
-                alreadyRanHooks.add(e);
-            }
-            pendingFlush.add(e);
+function findPendingFlushEntities(maybePendingFlushEntities, hooksInvoked, pendingFlush, pendingHooks, alreadyRanHooks) {
+    for (const e of maybePendingFlushEntities) {
+        if ((0, BaseEntity_1.getInstanceData)(e).pendingOperation === "none") {
+            maybePendingFlushEntities.delete(e);
+            continue;
         }
+        if (!hooksInvoked.has(e)) {
+            pendingHooks.add(e);
+        }
+        else {
+            alreadyRanHooks.add(e);
+        }
+        pendingFlush.add(e);
     }
 }
+/** Returns true if the caller explicitly asked `find` to use SQL pagination. */
+function hasPaginationSettings(options) {
+    return "limit" in options || "offset" in options;
+}
 /** An error we throw to get knex to `ROLLBACK`, but then catch. */
 class InMemoryRollbackError extends Error {
 }