joist-core 2.3.0-next.4 → 2.3.0-next.40

package/build/EntityManager.js

@@ -23,10 +23,6 @@ exports.createRowFromEntityData = createRowFromEntityData;
 const dataloader_1 = __importDefault(require("dataloader"));
 const BaseEntity_1 = require("./BaseEntity");
 const BatchLoader_1 = require("./batchloaders/BatchLoader");
-const defaults_1 = require("./defaults");
-const fields_1 = require("./fields");
-const IndexManager_1 = require("./IndexManager");
-// We alias `Entity => EntityW` to denote "Entity wide" i.e. the non-narrowed Entity
 const loadBatchLoader_1 = require("./batchloaders/loadBatchLoader");
 const populateBatchLoader_1 = require("./batchloaders/populateBatchLoader");
 const config_1 = require("./config");
@@ -37,12 +33,17 @@ const findDataLoader_1 = require("./dataloaders/findDataLoader");
 const findIdsDataLoader_1 = require("./dataloaders/findIdsDataLoader");
 const findOrCreateDataLoader_1 = require("./dataloaders/findOrCreateDataLoader");
 const findPaginatedDataLoader_1 = require("./dataloaders/findPaginatedDataLoader");
+const defaults_1 = require("./defaults");
+// We alias `Entity => EntityW` to denote "Entity wide" i.e. the non-narrowed Entity
 const Entity_1 = require("./Entity");
+const fields_1 = require("./fields");
 const FlushLock_1 = require("./FlushLock");
 const index_1 = require("./index");
+const IndexManager_1 = require("./IndexManager");
 const IsLoadedCache_1 = require("./IsLoadedCache");
 const JoinRows_1 = require("./JoinRows");
 const loadHints_1 = require("./loadHints");
+const FieldLogger_1 = require("./logging/FieldLogger");
 const newEntity_1 = require("./newEntity");
 const newTestInstance_1 = require("./newTestInstance");
 const PluginManager_1 = require("./PluginManager");
@@ -51,7 +52,9 @@ const reactiveHints_1 = require("./reactiveHints");
 const relations_1 = require("./relations");
 const AsyncProperty_1 = require("./relations/AsyncProperty");
 const hasAsyncMethod_1 = require("./relations/hasAsyncMethod");
+const LazyField_1 = require("./relations/LazyField");
 const RecursiveCollection_1 = require("./relations/RecursiveCollection");
+const RowData_1 = require("./RowData");
 const scopes_1 = require("./scopes");
 const Todo_1 = require("./Todo");
 const trusted_1 = require("./trusted");
@@ -98,6 +101,8 @@ class EntityManager {
     // Provides field-based indexing for entity types with >1000 entities to optimize findWithNewOrChanged
     #indexManager = new IndexManager_1.IndexManager();
     #isValidating = false;
+    // Set while regular (pre-flush) validation rules run, so `em.find*` can fail fast (see #assertFindAllowed)
+    #findRestricted = false;
     #pendingPercolate = new Map();
     #preloadedRelations = new Map();
     /**
@@ -124,7 +129,7 @@ class EntityManager {
         afterCommit: [],
     };
     #preloader;
-    #fieldLogger;
+    #fieldLogger = FieldLogger_1.noopFieldLogger;
     #isLoadedCache = new IsLoadedCache_1.IsLoadedCache();
     #merging;
     /** Track `a#1`, `a#2`, etc indexes for `em.create`-d entities. */
@@ -265,6 +270,7 @@ class EntityManager {
         return this.#entitiesById.get(id);
     }
     async find(type, where, options) {
+        this.#assertFindAllowed("find");
         const { populate, ...rest } = options || {};
         const normalized = (0, index_1.mergeFindOptions)(where, rest);
         const settings = { where: normalized.where, ...normalized.options };
@@ -278,11 +284,24 @@ class EntityManager {
         }
         return result;
     }
+    /** Fails fast if `em.${method}` is called from a regular validation rule, i.e. steers to `config.addCommitRule`. */
+    #assertFindAllowed(method) {
+        if (this.#findRestricted) {
+            throw new Error(`em.${method} cannot be called from a validation rule (added via config.addRule), because rules run ` +
+                `before the flush and would query stale, pre-flush data. Use config.addCommitRule instead, which runs ` +
+                `after the INSERT/UPDATE/DELETEs are flushed (within the same transaction) and so sees the changed state.`);
+        }
+    }
     /** Runs the post-parse find pipeline: plugins mutate the logical AST, then Joist optimizes/prunes before SQL. */
     async executeFind(meta, operation, parsed, settings) {
         const { checkLimit, findSettings } = this.prepareFind(meta, operation, parsed, settings);
         return this.executePreparedFind(meta, operation, parsed, findSettings, checkLimit);
     }
+    /** Like {@link executeFind}, but returns a {@link RowData}; see {@link executePreparedFindRowData}. */
+    async executeFindRowData(meta, operation, parsed, settings) {
+        const { checkLimit, findSettings } = this.prepareFind(meta, operation, parsed, settings);
+        return this.executePreparedFindRowData(meta, operation, parsed, findSettings, checkLimit);
+    }
     /** Executes a query that has already had find hooks and optimizations applied. */
     async executePreparedFind(meta, operation, parsed, findSettings, checkLimit) {
         const { pluginManager } = getEmInternalApi(this);
@@ -292,9 +311,33 @@ class EntityManager {
         if (shouldCheck && rows.length >= this.entityLimit) {
             throw new Error(`Query returned more than ${this.entityLimit} entityLimit rows`);
         }
-        pluginManager.afterFind(meta, operation, rows);
+        if (pluginManager.hasHook("afterFind")) {
+            pluginManager.afterFind(meta, operation, new RowData_1.PojoRowData(rows));
+        }
         return rows;
     }
+    /**
+     * Like {@link executePreparedFind}, but returns a {@link RowData}: lazy wire rows when the
+     * driver has `lazyRows` enabled, else classic POJO rows wrapped in a `PojoRowData` — so
+     * loaders can hydrate from one shape without caring which the driver provides.
+     */
+    async executePreparedFindRowData(meta, operation, parsed, findSettings, checkLimit) {
+        const { executeFindRowData } = this.driver;
+        if (executeFindRowData === undefined) {
+            return new RowData_1.PojoRowData(await this.executePreparedFind(meta, operation, parsed, findSettings, checkLimit));
+        }
+        const { pluginManager } = getEmInternalApi(this);
+        const rowData = await executeFindRowData.call(this.driver, this, parsed, findSettings);
+        // Check by default unless explicitly disabled or the caller removed the LIMIT via `limit: undefined`
+        const shouldCheck = checkLimit ?? !("limit" in findSettings && findSettings.limit === undefined);
+        if (shouldCheck && rowData.rowCount >= this.entityLimit) {
+            throw new Error(`Query returned more than ${this.entityLimit} entityLimit rows`);
+        }
+        // The hook receives the lazy RowData view directly, so observe-only hooks (i.e. metrics
+        // reading `rowCount`) never materialize cells; hooks call `toRows()` if they want POJOs
+        pluginManager.afterFind(meta, operation, rowData);
+        return rowData;
+    }
     /**
      * Runs pre-SQL find hooks and optimizations against a parsed query.
      *
@@ -348,6 +391,7 @@ class EntityManager {
         return list[0];
     }
     async findByUnique(type, where, options = {}) {
+        this.#assertFindAllowed("findByUnique");
         const { populate, softDeletes = "exclude" } = options;
         const entries = Object.entries(where);
         if (entries.length !== 1) {
@@ -383,6 +427,7 @@ class EntityManager {
      * Note: this method is not currently auto-batched, so it will cause N+1s if called in a loop.
      */
     async findCount(type, where, options = {}) {
+        this.#assertFindAllowed("findCount");
         const normalized = (0, findCountDataLoader_1.mergeCountOptions)(where, options);
         const settings = { where: normalized.where, ...normalized.options };
         let count = await (0, findCountDataLoader_1.findCountDataLoader)(this, type, settings).catch(function findCount(err) {
@@ -416,6 +461,7 @@ class EntityManager {
      * as `em.find`.
      */
     async findIds(type, where, options = {}) {
+        this.#assertFindAllowed("findIds");
         const normalized = (0, findCountDataLoader_1.mergeCountOptions)(where, options);
         const settings = { where: normalized.where, ...normalized.options };
         return (0, findIdsDataLoader_1.findIdsDataLoader)(this, type, settings).catch(function findIds(err) {
@@ -709,14 +755,13 @@ class EntityManager {
     }
     async loadAll(type, _ids, hint) {
         const meta = (0, index_1.getMetadata)(type);
-        // Use pre-allocated arrays/for loops instead of `.filter`s since this can be a hot spot
-        const ids = new Array(_ids.length);
+        // Use pre-allocated arrays/for loops instead of `.filter`s since this can be a hot spot;
+        // misses remember their tagged id + position so the all-hit path skips the `ids` copy entirely
         const entities = new Array(_ids.length);
         let idsToLoad;
         let positionsToLoad;
         for (let i = 0; i < _ids.length; i++) {
             const id = (0, index_1.tagId)(meta, _ids[i]);
-            ids[i] = id;
             const entity = this.findExistingInstance(id);
             if (entity) {
                 entities[i] = entity;
@@ -726,25 +771,23 @@ class EntityManager {
                 (positionsToLoad ??= []).push(i);
             }
         }
-        if (idsToLoad && idsToLoad.length > 0) {
+        if (idsToLoad) {
             await (0, loadBatchLoader_1.loadBatchLoader)(this, meta)
                 .loadAll(idsToLoad.map((id) => ({ taggedId: id, hint })))
                 .catch(function loadAll(err) {
                 throw appendStack(err, new Error());
             });
-            for (const i of positionsToLoad) {
-                entities[i] = this.findExistingInstance(ids[i]);
+            let idsNotFound;
+            for (let j = 0; j < positionsToLoad.length; j++) {
+                const entity = this.findExistingInstance(idsToLoad[j]);
+                entities[positionsToLoad[j]] = entity;
+                if (entity === undefined)
+                    (idsNotFound ??= []).push(idsToLoad[j]);
             }
-        }
-        let idsNotFound;
-        for (let i = 0; i < entities.length; i++) {
-            if (entities[i] === undefined) {
-                (idsNotFound ??= []).push(ids[i]);
+            if (idsNotFound) {
+                throw new NotFoundError(`${idsNotFound.join(",")} were not found`);
             }
         }
-        if (idsNotFound) {
-            throw new NotFoundError(`${idsNotFound.join(",")} were not found`);
-        }
         const loadedEntities = entities;
         if (hint) {
             await this.populate(loadedEntities, hint);
@@ -759,36 +802,43 @@ class EntityManager {
     }
     async loadAllIfExists(type, _ids, hint) {
         const meta = (0, index_1.getMetadata)(type);
-        // Use pre-allocated arrays/for loops instead of `.filter`s since this can be a hot spot
-        const ids = new Array(_ids.length);
-        const entities = [];
+        // Use pre-allocated arrays/for loops instead of `.filter`s since this can be a hot spot;
+        // the all-hit path returns the single pre-sized array without any id copies or rebuilds
+        const maybeEntities = new Array(_ids.length);
         let idsToLoad;
+        let positionsToLoad;
         // Ensure the ids are tagged, and find any not-yet-loaded
         for (let i = 0; i < _ids.length; i++) {
             const id = (0, index_1.tagId)(meta, _ids[i]);
-            ids[i] = id;
             const entity = this.findExistingInstance(id);
             if (entity) {
-                entities.push(entity);
+                maybeEntities[i] = entity;
             }
             else {
                 (idsToLoad ??= []).push(id);
+                (positionsToLoad ??= []).push(i);
             }
         }
-        if (idsToLoad && idsToLoad.length > 0) {
+        let entities;
+        if (idsToLoad) {
             await (0, loadBatchLoader_1.loadBatchLoader)(this, meta)
                 .loadAll(idsToLoad.map((id) => ({ taggedId: id, hint })))
                 .catch(function loadAllIfExists(err) {
                 throw appendStack(err, new Error());
             });
-            // Now that everything is loaded, recalc `entities`
-            entities.length = 0;
-            for (const id of ids) {
-                const entity = this.findExistingInstance(id);
-                if (entity)
+            // Fill in whichever missed ids actually exist, then compact out the not-founds
+            for (let j = 0; j < positionsToLoad.length; j++) {
+                maybeEntities[positionsToLoad[j]] = this.findExistingInstance(idsToLoad[j]);
+            }
+            entities = [];
+            for (const entity of maybeEntities) {
+                if (entity !== undefined)
                     entities.push(entity);
             }
         }
+        else {
+            entities = maybeEntities;
+        }
         if (hint) {
             await this.populate(entities, hint);
         }
@@ -996,12 +1046,12 @@ class EntityManager {
                 // each time, so that for an INSERT-then-UPDATE the triggers don't think the
                 // UPDATE forgot to self-bump updatedAt, and then "helpfully" bump it for us.
                 if (alreadyRanHooks.size > 0) {
-                    maybeBumpUpdatedAt(this.#rm, (0, Todo_1.createTodos)([...alreadyRanHooks]), now);
+                    maybeBumpUpdatedAt(this.#rm, (0, Todo_1.createTodos)(alreadyRanHooks), now);
                 }
                 // Run hooks in a series of loops until things "settle down"
                 while (pendingHooks.size > 0) {
                     await this.#fl.allowWrites(async () => {
-                        let todos = (0, Todo_1.createTodos)([...pendingHooks]);
+                        let todos = (0, Todo_1.createTodos)(pendingHooks);
                         await (0, defaults_1.setAsyncDefaults)(suppressedDefaultTypeErrors, this.ctx, Todo_1.Todo.groupInsertsByTypeAndSubType(todos));
                         maybeBumpUpdatedAt(this.#rm, todos, now);
                         // Run our hooks
@@ -1059,16 +1109,25 @@ class EntityManager {
                     this.#isValidating = true;
                     await pluginManager.beforeValidate(changedEntities);
                     if (validate) {
-                        // Run simple rules first b/c it includes not-null/required rules, so that then when we run
-                        // `validateReactiveRules` next, the app's lambdas won't see fundamentally invalid entities & NPE.
-                        await validateSimpleRules(entityTodos);
-                        // After we've let any "author is not set" simple rules fail before prematurely throwing
-                        // the "of course that caused an NPE" `TypeError`s, if all the authors *were* valid/set,
-                        // and we still have TypeErrors (from derived valeus), they were real, unrelated errors
-                        // that the user should see.
-                        if (suppressedDefaultTypeErrors.length > 0)
-                            throw suppressedDefaultTypeErrors[0];
-                        await validateReactiveRules(this, this.#rm.logger, entityTodos, joinRowTodos);
+                        // Regular rules run pre-flush, so `em.find*` would query stale data; block it and steer
+                        // users to `config.addCommitRule` (which runs post-flush). Only rules are restricted —
+                        // `afterValidation` hooks & plugins below are left free to query.
+                        this.#findRestricted = true;
+                        try {
+                            // Run simple rules first b/c it includes not-null/required rules, so that then when we run
+                            // `validateReactiveRules` next, the app's lambdas won't see fundamentally invalid entities & NPE.
+                            await validateSimpleRules(entityTodos);
+                            // After we've let any "author is not set" simple rules fail before prematurely throwing
+                            // the "of course that caused an NPE" `TypeError`s, if all the authors *were* valid/set,
+                            // and we still have TypeErrors (from derived valeus), they were real, unrelated errors
+                            // that the user should see.
+                            if (suppressedDefaultTypeErrors.length > 0)
+                                throw suppressedDefaultTypeErrors[0];
+                            await validateReactiveRules(this, this.#rm.logger, entityTodos, joinRowTodos);
+                        }
+                        finally {
+                            this.#findRestricted = false;
+                        }
                         await afterValidation(this.ctx, entityTodos);
                     }
                     await pluginManager.afterValidate(changedEntities);
@@ -1077,6 +1136,61 @@ class EntityManager {
                     this.#isValidating = false;
                 }
             };
+            // Runs `addCommitRule`s, which look exactly like regular validation rules but fire here,
+            // after the INSERT/UPDATE/DELETEs have hit the db (but before COMMIT), so that any
+            // `em.find`s they make query the just-flushed/changed state within the transaction.
+            const runCommitValidation = async (entityTodos, joinRowTodos) => {
+                // Drop the pre-flush find/query caches so commit rules' `em.find`s re-query and see the
+                // just-flushed rows, instead of stale results cached earlier in this same flush.
+                this.#dataloaders = {};
+                this.#batchLoaders = {};
+                this.#preloadedRelations = new Map();
+                try {
+                    // `isValidating` keeps derived-field reads from creating (now-unflushable) dirty state.
+                    this.#isValidating = true;
+                    await validateSimpleRules(entityTodos, (config) => config.commitRules);
+                    await validateReactiveRules(this, this.#rm.logger, entityTodos, joinRowTodos, (meta) => meta.reactiveCommitRules);
+                }
+                finally {
+                    this.#isValidating = false;
+                }
+            };
+            // Commit rules run once, at the end, over just the flushed entities/join-rows whose type has
+            // commit rules (see `meta.hasCommitRules`). We accumulate that subset here as we flush — deduping
+            // across the RQF loop below (which resets entity dirty-state) — so we neither keep every todo
+            // nor re-scan/merge them afterwards; apps without commit rules leave these empty and pay ~nothing.
+            const commitRuleTodos = {};
+            const commitRuleJoinRowTodos = {};
+            const commitRuleSeen = new Set();
+            function collectCommitRuleTodos(entityTodos, joinRowTodos) {
+                if (skipValidation)
+                    return;
+                for (const todo of Object.values(entityTodos)) {
+                    if (!todo.metadata.hasCommitRules)
+                        continue;
+                    for (const key of ["inserts", "updates", "deletes"]) {
+                        for (const e of todo[key]) {
+                            if (!commitRuleSeen.has(e)) {
+                                commitRuleSeen.add(e);
+                                (0, Todo_1.getTodo)(commitRuleTodos, e)[key].push(e);
+                            }
+                        }
+                    }
+                }
+                for (const [table, todo] of Object.entries(joinRowTodos)) {
+                    if (!todo.m2m.meta.hasCommitRules && !todo.m2m.otherMeta?.hasCommitRules)
+                        continue;
+                    const into = (commitRuleJoinRowTodos[table] ??= { ...todo, newRows: [], deletedRows: [] });
+                    for (const key of ["newRows", "deletedRows"]) {
+                        for (const r of todo[key]) {
+                            if (!commitRuleSeen.has(r)) {
+                                commitRuleSeen.add(r);
+                                into[key].push(r);
+                            }
+                        }
+                    }
+                }
+            }
             // Run hooks (in iterative loops if hooks mutate new entities) on pending entities
             let entitiesToFlush = await runHooksOnPendingEntities();
             for (const e of entitiesToFlush)
@@ -1095,6 +1209,8 @@ class EntityManager {
                     do {
                         if (Object.keys(entityTodos).length > 0 || Object.keys(joinRowTodos).length > 0) {
                             await this.driver.flush(this, entityTodos, joinRowTodos);
+                            // Accumulate the commit-rule-relevant subset before the RQF loop clobbers/resets these
+                            collectCommitRuleTodos(entityTodos, joinRowTodos);
                         }
                         // Now that we've flushed, we can let plugins know what we've done.
                         pluginManager.afterWrite(entityTodos, joinRowTodos);
@@ -1130,6 +1246,18 @@ class EntityManager {
                             entityTodos = {};
                         }
                     } while (Object.keys(entityTodos).length > 0);
+                    // Now that all SQL has hit the db (but before COMMIT), run any `addCommitRule`s so their
+                    // `em.find`s see the changed state. `commitRuleTodos` is only non-empty if a flushed entity
+                    // actually has commit rules, so apps without them skip this (and its logging) entirely.
+                    if (Object.keys(commitRuleTodos).length > 0 || Object.keys(commitRuleJoinRowTodos).length > 0) {
+                        // Make just-inserted entities findable by id (like the RQF loop does) so commit rules'
+                        // `em.find`s return these same instances rather than hydrating duplicate ones.
+                        for (const e of allFlushedEntities) {
+                            if (e.isNewEntity && !e.isDeletedEntity)
+                                this.#entitiesById.set(e.idTagged, e);
+                        }
+                        await runCommitValidation(commitRuleTodos, commitRuleJoinRowTodos);
+                    }
                     // Run `beforeCommit once right before COMMIT
                     await beforeCommit(this.ctx, allFlushedEntities);
                     if (this.mode === "in-memory-writes") {
@@ -1141,13 +1269,17 @@ class EntityManager {
                 await afterCommit(this.ctx, allFlushedEntities);
                 // Update the `#orm` field to reflect the new state
                 for (const e of allFlushedEntities) {
+                    const instanceData = (0, BaseEntity_1.getInstanceData)(e);
                     if (e.isNewEntity && !e.isDeletedEntity)
                         this.#entitiesById.set(e.idTagged, e);
-                    (0, BaseEntity_1.getInstanceData)(e).resetAfterFlushed();
-                    // Reset AsyncQueryProperties since DB state may have changed
-                    for (const rel of Object.values((0, BaseEntity_1.getInstanceData)(e).relations)) {
-                        if (rel instanceof AsyncProperty_1.AsyncPropertyImpl)
-                            rel.resetAfterFlush();
+                    instanceData.resetAfterFlushed();
+                    // Reset AsyncQueryProperties & LazyFields since DB state may have changed
+                    const { relations } = instanceData;
+                    if (relations) {
+                        for (const rel of Object.values(relations)) {
+                            if (rel instanceof AsyncProperty_1.AsyncPropertyImpl || rel instanceof LazyField_1.LazyFieldImpl)
+                                rel.resetAfterFlush();
+                        }
                     }
                 }
                 // Update the joinRows refs to reflect the new state
@@ -1185,16 +1317,28 @@ class EntityManager {
                     }
                 }
             }
+            // node-pg keeps the DETAIL line (i.e. "Key (identity_id, final_id)=(102591, null) already exists")
+            // on a separate `detail` field, so keep it to suffix onto messages to help debugging.
+            const detail = e && typeof e === "object" && "detail" in e && typeof e.detail === "string" ? e.detail : undefined;
             if (e && typeof e === "object" && "constraint" in e && typeof e.constraint === "string") {
                 // node-pg errors use `constraint` to indicate the constraint name
                 const message = config_1.constraintNameToValidationError[e.constraint];
                 if (message) {
-                    throw new index_1.ValidationErrors(message);
+                    const errors = new index_1.ValidationErrors(message);
+                    // Keep the user-facing `errors[].message` pretty, but suffix the DETAIL onto the top-level
+                    // `message` so the debugging context survives into logs/stack traces.
+                    if (detail)
+                        errors.message = `${message} — ${detail}`;
+                    throw errors;
                 }
             }
             if (e instanceof InMemoryRollbackError) {
                 return [...allFlushedEntities].sort((a, b) => (0, BaseEntity_1.getInstanceData)(a).entityIndex - (0, BaseEntity_1.getInstanceData)(b).entityIndex);
             }
+            // For raw (non-pretty) pg errors, fold the DETAIL into `message` so it survives into logs.
+            if (detail && e instanceof Error) {
+                e.message = `${e.message} — ${detail}`;
+            }
             throw e;
         }
         finally {
@@ -1301,22 +1445,57 @@ class EntityManager {
      * the WIP entity state.
      */
     hydrate(type, rows, options) {
+        return this.hydrateFromRowData(type, new RowData_1.PojoRowData(rows), options);
+    }
+    /**
+     * Hydrates a query's `RowData`, running any sidecar reads, and always `finalize`s the result.
+     *
+     * This is the internal API for Joist's find/load loaders: `sidecars` is where reads of
+     * non-entity columns (preload aggregates, `_tags`, found-id checks) belong — they must run
+     * before `finalize`, because compaction drops unretained rows — and the `try/finally`
+     * guarantees a failed hydration or sidecar read still releases the result's unretained
+     * buffers instead of pinning them.
+     */
+    hydrateAndFinalize(type, rowData, opts) {
+        try {
+            const entities = this.hydrateFromRowData(type, rowData, opts);
+            opts?.sidecars?.(entities);
+            return entities;
+        }
+        finally {
+            rowData.finalize?.();
+        }
+    }
+    /**
+     * Like {@link hydrate}, but reads rows from a {@link RowData}, i.e. a driver-produced
+     * lazy query result, instead of an array of POJO rows.
+     *
+     * This is an internal API used by Joist's own find loaders (usually via
+     * {@link hydrateAndFinalize}); rows whose entities are kept (newly created or
+     * overwrite-refreshed) are `retain`-ed on the `RowData` so the caller can later `finalize`
+     * (trim/compact) the result.
+     */
+    hydrateFromRowData(type, rowData, options) {
         const maybeBaseMeta = (0, index_1.getMetadata)(type);
-        const taggedIdPrefix = `${maybeBaseMeta.tagName}:`;
         const overwriteExisting = options?.overwriteExisting === true;
-        let i = 0;
-        const entities = new Array(rows.length);
-        for (const row of rows) {
-            const id = row["id"];
-            const taggedId = id === undefined || id === null ? (0, utils_1.fail)("No id column was available") : `${taggedIdPrefix}${id}`;
+        const count = rowData.rowCount;
+        const entities = new Array(count);
+        for (let i = 0; i < count; i++) {
+            const id = rowData.get(i, "id");
+            const taggedId = id === undefined || id === null ? (0, utils_1.fail)("No id column was available") : (0, index_1.keyToTaggedId)(maybeBaseMeta, id);
             // See if this is already in our UoW
             let entity = this.findExistingInstance(taggedId);
             if (!entity) {
                 // Look for __class from the driver telling us which subtype to instantiate
-                const meta = findConcreteMeta(maybeBaseMeta, row);
+                const meta = findConcreteMeta(maybeBaseMeta, rowData, i);
                 // Pass id as a hint that we're in hydrate mode
                 entity = (0, newEntity_1.newEntity)(this, (0, index_1.asConcreteCstr)(meta.cstr), false);
-                (0, BaseEntity_1.getInstanceData)(entity).row = row;
+                const instanceData = (0, BaseEntity_1.getInstanceData)(entity);
+                instanceData.rowData = rowData;
+                instanceData.rowIndex = i;
+                rowData.retain?.(i);
+                // Seed the id to share the identity-map key string and skip the id serde on first read
+                instanceData.data["id"] = taggedId;
                 this.#doRegister(entity, taggedId, meta, true);
             }
             else if (overwriteExisting) {
@@ -1324,24 +1503,34 @@ class EntityManager {
                 // `EntityManager.refresh` is telling us to explicitly load the latest data.
                 // First swap out the old row with the new row
                 const instanceData = (0, BaseEntity_1.getInstanceData)(entity);
-                instanceData.row = row;
+                instanceData.rowData = rowData;
+                instanceData.rowIndex = i;
+                rowData.retain?.(i);
                 // And then only refresh the data keys that have already been serde-d from rows
                 // (this keeps us from deserializing data out of rows that we don't need).
                 const { data } = instanceData;
                 const dataKeys = Object.keys(data);
-                if (dataKeys.length > 0) {
+                // `id` is seeded at hydrate and can never change, so it never needs refreshing
+                if (dataKeys.length > ("id" in data ? 1 : 0)) {
                     const allFields = (0, index_1.getMetadata)(entity).allFields;
-                    const changedFields = entity.changes.fieldsWithoutRelations;
+                    // For existing entities this is exactly `changes.fieldsWithoutRelations`, minus the proxy allocation
+                    const changedFields = instanceData.isNewEntity
+                        ? entity.changes.fieldsWithoutRelations
+                        : Object.keys(instanceData.originalData);
                     if (changedFields.length === 0) {
                         for (const fieldName of dataKeys) {
+                            if (fieldName === "id")
+                                continue;
                             const serde = allFields[fieldName].serde ?? (0, utils_1.fail)(`Missing serde for ${fieldName}`);
-                            serde.setOnEntity(data, row);
+                            serde.setOnEntityFromRowData(data, rowData, i);
                         }
                     }
                     else {
                         for (const fieldName of dataKeys) {
+                            if (fieldName === "id")
+                                continue;
                             const serde = allFields[fieldName].serde ?? (0, utils_1.fail)(`Missing serde for ${fieldName}`);
-                            serde.setOnEntity(data, row);
+                            serde.setOnEntityFromRowData(data, rowData, i);
                             // Make the field look not-dirty
                             if (changedFields.includes(fieldName)) {
                                 instanceData.markFieldClean(fieldName);
@@ -1350,7 +1539,7 @@ class EntityManager {
                     }
                 }
             }
-            entities[i++] = entity;
+            entities[i] = entity;
         }
         return entities;
     }
@@ -1359,15 +1548,28 @@ class EntityManager {
             (0, BaseEntity_1.getInstanceData)(entity).markTouched();
         }
     }
-    async recalc(entityOrEntities) {
+    async recalc(entityOrEntities, reactionName) {
+        const entities = (0, utils_1.toArray)(entityOrEntities);
+        if (reactionName !== undefined) {
+            // Resolve every reaction before invoking any of them so a mixed list cannot partially run.
+            const reactions = entities.map((entity) => {
+                const reaction = (0, index_1.getBaseAndSelfMetas)((0, index_1.getMetadata)(entity))
+                    .flatMap((meta) => meta.config.__data.reactions)
+                    .find((reaction) => reaction.name === reactionName);
+                return reaction ?? (0, utils_1.fail)(`Reaction '${reactionName}' not found on ${(0, index_1.getMetadata)(entity).type}`);
+            });
+            await (0, trusted_1.runInTrustedContext)(() => Promise.allSettled(reactions.map((reaction, i) => reaction.fn(entities[i], this.ctx))).then(utils_1.failIfAnyRejected));
+            await this.#rm.recalcPendingReactables("reactables");
+            return;
+        }
         // Look for async reactive fields
-        const relations = (0, utils_1.toArray)(entityOrEntities).flatMap((entity) => Object.values((0, index_1.getMetadata)(entity).allFields)
+        const relations = entities.flatMap((entity) => Object.values((0, index_1.getMetadata)(entity).allFields)
             .filter((f) => "derived" in f && f.derived === "async")
             .map((field) => entity[field.fieldName]));
         // Use forceReload: true to tell ReactiveReferences to recalc against their full graph
         await Promise.all(relations.map((r) => r.load({ forceReload: true })));
         // And also sync reactive fields
-        (0, utils_1.toArray)(entityOrEntities).flatMap((entity) => Object.values((0, index_1.getMetadata)(entity).allFields)
+        entities.flatMap((entity) => Object.values((0, index_1.getMetadata)(entity).allFields)
             .filter((f) => "derived" in f && f.derived === "sync")
             .forEach((field) => {
             // This will be a noop if the value is the same
@@ -1460,7 +1662,7 @@ class EntityManager {
         }
         const writeFn = getDefaultWriteFn(this.ctx);
         if (typeof arg === "boolean") {
-            this.#fieldLogger = arg ? new index_1.FieldLogger([], writeFn) : undefined;
+            this.#fieldLogger = arg ? new index_1.FieldLogger([], writeFn) : FieldLogger_1.noopFieldLogger;
         }
         else if (typeof arg === "string" || Array.isArray(arg)) {
             const specs = Array.isArray(arg) ? arg : [arg];
@@ -1495,15 +1697,18 @@ class EntityManager {
         const entities = this.#entitiesByTag.get(meta.tagName) ?? [];
         // Don't bother filtering if there's no where clause (particularly b/c IndexManager.findMatching
         // really expects there to be at least 1 condition)
-        if (Object.entries(where).length === 0) {
+        if (!(0, utils_1.hasAnyKey)(where)) {
             return entities.filter((e) => e instanceof cstr && !e.isDeletedEntity);
         }
         if (this.#indexManager.shouldIndexType(entities.length)) {
             this.#indexManager.enableIndexingForType(meta, entities, where);
-            return (this.#indexManager
-                .findMatching(meta, entities, where)
-                // Still filter by `instanceof cstr` to handle subtyping
-                .filter((e) => e instanceof cstr && !e.isDeletedEntity));
+            // Build the final array in one pass, still filtering `instanceof cstr` to handle subtyping
+            const result = [];
+            for (const e of this.#indexManager.findMatching(meta, entities, where)) {
+                if (e instanceof cstr && !e.isDeletedEntity)
+                    result.push(e);
+            }
+            return result;
         }
         else {
             return entities
@@ -1626,6 +1831,8 @@ class EntityManager {
                 continue;
             }
             const { relations } = oldEntity.__data;
+            if (!relations)
+                continue;
             for (const [field, relation] of Object.entries(relations)) {
                 // With lazyRelation, custom relations are inserted into the `relations` map. Custom relations don't
                 // store any data, so we can ignore them by checking if the relation implements `import`
@@ -1742,7 +1949,7 @@ class EntityManager {
         }
         // api will be undefined during getFakeInstance
         const api = getEmInternalApi(this);
-        api?.fieldLogger?.logCreate(entity);
+        api?.fieldLogger.logCreate(entity);
         (0, index_1.setOpts)(entity, opts, { partial, calledFromConstructor: true });
         // Apply any synchronous defaults, after the opts have been applied
         if (!this.fakeInstance) {
@@ -1851,8 +2058,10 @@ exports.TooManyError = TooManyError;
  * from the currently-changed entities back to each rule's originally-defined-in entity,
  * and ensure those entities are added to `todos`.
  */
-async function validateReactiveRules(em, logger, todos, joinRowTodos) {
-    logger?.logStartingValidate(em, todos);
+async function validateReactiveRules(em, logger, todos, joinRowTodos, 
+// Which list of reactive rules to run, i.e. the pre-flush `reactiveRules` or the post-flush `reactiveCommitRules`
+getRules = (meta) => meta.reactiveRules) {
+    logger.logStartingValidate(em, todos);
     // Use a map of rule -> Set<Entity> so that we only invoke a rule once per entity,
     // even if it was triggered by multiple changed fields.
     const fns = new Map();
@@ -1869,15 +2078,18 @@ async function validateReactiveRules(em, logger, todos, joinRowTodos) {
             entities = new Set();
             fns.set(rule.fn, entities);
         }
-        logger?.logWalked(triggered, rule, found, "validate");
+        logger.logWalked(triggered, rule, found, "validate");
         found.forEach((entity) => {
             entities.add(entity);
         });
     }
     const p1 = Object.values(todos).flatMap((todo) => {
-        const entities = [...todo.inserts, ...todo.updates, ...todo.deletes];
         // Find each statically-declared reactive rule for the given entity type
-        const rules = todo.metadata.reactiveRules;
+        const rules = getRules(todo.metadata);
+        // Skip the entities spread for rule-less types
+        if (rules.length === 0)
+            return [];
+        const entities = [...todo.inserts, ...todo.updates, ...todo.deletes];
         return rules.map((rule) => {
             // Of all changed entities of this type, how many specifically trigger this rule?
             const triggered = entities.filter((e) => {
@@ -1887,10 +2099,12 @@ async function validateReactiveRules(em, logger, todos, joinRowTodos) {
                 // Any new-or-deleted entity fires every rule (reactiveRules has already filtered out read-only)
                 if (e.isNewEntity || e.isDeletedEntity)
                     return true;
-                // Otherwise see if the changed fields overlaps with the rule's fields
-                const changedFields = e.changes.fieldsWithoutRelations;
-                for (const field of changedFields) {
-                    if (rule.fields.includes(field))
+                // Otherwise see if the changed fields overlap with the rule's fields; for existing entities
+                // `changes.fieldsWithoutRelations` is exactly `Object.keys(originalData)`, so probe originalData
+                // directly instead of allocating a proxy + keys array per entity per rule.
+                const { originalData } = (0, BaseEntity_1.getInstanceData)(e);
+                for (const field of rule.fields) {
+                    if (field in originalData)
                         return true;
                 }
                 return false;
@@ -1903,8 +2117,8 @@ async function validateReactiveRules(em, logger, todos, joinRowTodos) {
         // For enum m2ms the columns include the enum's numeric id, so keep only the entity values.
         const entities = [...todo.newRows, ...todo.deletedRows].flatMap((jr) => Object.values(jr.columns)).filter(Entity_1.isEntity);
         // The owning-entity side (applies to both entity-to-entity and entity-to-enum m2ms).
-        const a = todo.m2m.meta
-            .reactiveRules.filter((rule) => rule.fields.includes(todo.m2m.fieldName))
+        const a = getRules(todo.m2m.meta)
+            .filter((rule) => rule.fields.includes(todo.m2m.fieldName))
             .map((rule) => {
             const triggered = entities.filter((e) => e instanceof todo.m2m.meta.cstr);
             return followAndQueue(triggered, rule);
@@ -1912,8 +2126,8 @@ async function validateReactiveRules(em, logger, todos, joinRowTodos) {
         // The "other" side only exists for entity-to-entity m2ms; enums have no reverse field.
         const { otherMeta, otherFieldName } = todo.m2m;
         const b = otherMeta
-            ? otherMeta
-                .reactiveRules.filter((rule) => rule.fields.includes(otherFieldName))
+            ? getRules(otherMeta)
+                .filter((rule) => rule.fields.includes(otherFieldName))
                 .map((rule) => {
                 const triggered = entities.filter((e) => e instanceof otherMeta.cstr);
                 return followAndQueue(triggered, rule);
@@ -1932,22 +2146,37 @@ async function validateReactiveRules(em, logger, todos, joinRowTodos) {
 // Run *non-reactive* (those with `fields: undefined`) rules of explicitly mutated entities,
 // because even for reactive validations on mutated entities, we defer to addReactiveValidations
 // to mark only the rules that need to run.
-async function validateSimpleRules(todos) {
-    const p = Object.values(todos).flatMap(({ inserts, updates }) => {
-        return [...inserts, ...updates]
-            .filter((e) => !e.isDeletedEntity)
-            .flatMap((entity) => {
-            const rules = (0, index_1.getBaseAndSelfMetas)((0, index_1.getMetadata)(entity)).flatMap((m) => m.config.__data.rules);
-            return rules
-                .filter((rule) => rule.hint === undefined)
-                .flatMap(async ({ fn }) => coerceError(entity, await fn(entity)));
-        });
-    });
+async function validateSimpleRules(todos, 
+// Which config list to pull rules from, i.e. the pre-flush `rules` or the post-flush `commitRules`
+getRules = (config) => config.rules) {
+    let p = undefined;
+    for (const todoKey in todos) {
+        const { inserts, updates } = todos[todoKey];
+        for (const entities of [inserts, updates]) {
+            for (const entity of entities) {
+                if (entity.isDeletedEntity)
+                    continue;
+                for (const m of (0, index_1.getBaseAndSelfMetas)((0, index_1.getMetadata)(entity))) {
+                    for (const rule of getRules(m.config.__data)) {
+                        if (rule.hint !== undefined)
+                            continue;
+                        (p ??= []).push(invokeRule(entity, rule.fn));
+                    }
+                }
+            }
+        }
+    }
+    if (!p)
+        return;
     const errors = (0, utils_1.failIfAnyRejected)(await Promise.allSettled(p)).flat();
     if (errors.length > 0) {
         throw new index_1.ValidationErrors(errors);
     }
 }
+/** Invokes a single validation rule, coercing its result/sync throws into a promise. */
+async function invokeRule(entity, fn) {
+    return coerceError(entity, await fn(entity));
+}
 function driverBeforeBegin(em, txn) {
     return Promise.all(getEmInternalApi(em).hooks.beforeBegin.map((fn) => fn(em, txn)));
 }
@@ -1961,23 +2190,54 @@ function driverAfterCommit(em, txn) {
     return Promise.all(getEmInternalApi(em).hooks.afterCommit.map((fn) => fn(em, txn)));
 }
 async function runHookOnTodos(ctx, hook, todos, keys) {
-    const entities = Object.values(todos).flatMap((todo) => {
-        return keys.flatMap((k) => todo[k].filter((e) => k === "deletes" || !e.isDeletedEntity));
-    });
-    return runHook(ctx, hook, entities);
+    // Only collect entities for todos whose type hierarchy actually has this hook, so that
+    // hook-less bulk flushes skip the per-entity array building entirely.
+    let entities = undefined;
+    for (const todoKey in todos) {
+        const todo = todos[todoKey];
+        if (!metaHasHook(todo.metadata, hook))
+            continue;
+        for (const k of keys) {
+            for (const e of todo[k]) {
+                if (k !== "deletes" && e.isDeletedEntity)
+                    continue;
+                (entities ??= []).push(e);
+            }
+        }
+    }
+    if (entities)
+        return runHook(ctx, hook, entities);
+}
+/** Returns whether `meta`'s base/self/sub hierarchy has any `hook` functions registered. */
+function metaHasHook(meta, hook) {
+    for (const m of (0, index_1.getBaseSelfAndSubMetas)(meta)) {
+        if (m.config.__data.hooks[hook].length > 0)
+            return true;
+    }
+    return false;
 }
 async function runHook(ctx, hook, entities) {
-    const p = entities.flatMap((entity) => {
-        const hookFns = (0, index_1.getBaseAndSelfMetas)((0, index_1.getMetadata)(entity)).flatMap((m) => m.config.__data.hooks[hook]);
-        // Use an explicit `async` here to ensure all hooks are promises, i.e. so that a non-promise
-        // hook blowing up doesn't orphan the others .
-        return hookFns.map(async (fn) => fn(entity, ctx));
-    });
+    let p = undefined;
+    for (const entity of entities) {
+        for (const m of (0, index_1.getBaseAndSelfMetas)((0, index_1.getMetadata)(entity))) {
+            for (const fn of m.config.__data.hooks[hook]) {
+                // Use an explicit `async` invoke to ensure all hooks are promises, i.e. so that a
+                // non-promise hook blowing up doesn't orphan the others.
+                (p ??= []).push(invokeHook(fn, entity, ctx));
+            }
+        }
+    }
+    if (!p)
+        return;
     // Use `allSettled` so that even if 1 hook blows up, we don't orphan other hooks mid-flush
     // (causes weird errors when/if they try to access the EntityManager that has "moved on")
     const results = await Promise.allSettled(p);
     (0, utils_1.failIfAnyRejected)(results);
 }
+/** Invokes a single hook fn, coercing sync throws into rejections. */
+async function invokeHook(fn, entity, ctx) {
+    return fn(entity, ctx);
+}
 function beforeDelete(ctx, todos) {
     return runHookOnTodos(ctx, "beforeDelete", todos, ["deletes"]);
 }
@@ -1997,16 +2257,21 @@ function afterValidation(ctx, todos) {
 function entitiesFromTodos(entityTodos, joinRowTodos) {
     const entities = new Set();
     for (const todo of Object.values(entityTodos)) {
-        [...todo.inserts, ...todo.updates, ...todo.deletes].forEach((entity) => entities.add(entity));
+        for (const list of [todo.inserts, todo.updates, todo.deletes]) {
+            for (const entity of list)
+                entities.add(entity);
+        }
     }
     for (const todo of Object.values(joinRowTodos)) {
-        [...todo.newRows, ...todo.deletedRows].forEach((row) => {
-            // For enum m2ms, a column value is the enum's numeric id rather than an entity.
-            Object.values(row.columns).forEach((value) => {
-                if ((0, Entity_1.isEntity)(value))
-                    entities.add(value);
-            });
-        });
+        for (const rows of [todo.newRows, todo.deletedRows]) {
+            for (const row of rows) {
+                // For enum m2ms, a column value is the enum's numeric id rather than an entity.
+                for (const value of Object.values(row.columns)) {
+                    if ((0, Entity_1.isEntity)(value))
+                        entities.add(value);
+                }
+            }
+        }
     }
     return [...entities];
 }
@@ -2040,23 +2305,19 @@ function coerceError(entity, maybeError) {
 }
 /** Evaluates each (non-async) derived field to see if it's value has changed. */
 function recalcSynchronousDerivedFields(todos) {
-    const entities = Object.values(todos)
-        .flatMap((todo) => [...todo.inserts, ...todo.updates])
-        .filter((e) => !e.isDeletedEntity);
-    const derivedFieldsByMeta = new Map([...new Set(entities.map(index_1.getMetadata))].map((m) => {
-        return [
-            m,
-            Object.values(m.allFields)
-                .filter((f) => (f.kind === "primitive" || f.kind === "enum") && f.derived === "sync")
-                .map((f) => f.fieldName),
-        ];
-    }));
-    for (const entity of entities) {
-        const derivedFields = derivedFieldsByMeta.get((0, index_1.getMetadata)(entity)) || [];
-        derivedFields.forEach((fieldName) => {
-            // setField will intelligently mark/not mark the field as dirty.
-            (0, fields_1.setField)(entity, fieldName, entity[fieldName]);
-        });
+    for (const todoKey in todos) {
+        const todo = todos[todoKey];
+        for (const entities of [todo.inserts, todo.updates]) {
+            for (const entity of entities) {
+                if (entity.isDeletedEntity)
+                    continue;
+                // Use the per-meta cached field list instead of rebuilding a map per hook loop
+                for (const fieldName of (0, index_1.getMetadata)(entity).syncDerivedFields) {
+                    // setField will intelligently mark/not mark the field as dirty.
+                    (0, fields_1.setField)(entity, fieldName, entity[fieldName]);
+                }
+            }
+        }
     }
 }
 /** Recursively crawls through `entity`, with the given populate `deep` hint, and adds anything found to `found`. */
@@ -2174,22 +2435,28 @@ function getNow() {
     return now;
 }
 /** Given a `row` from the db, resolves the CTI/STI subtype, if applicable. */
-function findConcreteMeta(maybeBaseMeta, row) {
-    // Common case of no CTI or STI inheritance
-    if (!row.__class && maybeBaseMeta.inheritanceType !== "sti") {
+function findConcreteMeta(maybeBaseMeta, rowData, rowIndex) {
+    // Common case of no CTI or STI inheritance; only probe __class for CTI metas so that
+    // lazy/columnar stores don't pay a missing-column lookup per row for regular entities
+    if (maybeBaseMeta.inheritanceType === undefined) {
+        return maybeBaseMeta;
+    }
+    const __class = rowData.get(rowIndex, "__class");
+    if (!__class && maybeBaseMeta.inheritanceType !== "sti") {
         return maybeBaseMeta;
     }
-    if (row.__class) {
-        if (row.__class === "_" && maybeBaseMeta.ctiAbstract) {
-            throw new Error(`${maybeBaseMeta.type} ${(0, index_1.tagId)(maybeBaseMeta, row.id)} must be instantiated via a subtype`);
+    if (__class) {
+        if (__class === "_" && maybeBaseMeta.ctiAbstract) {
+            const id = rowData.get(rowIndex, "id");
+            throw new Error(`${maybeBaseMeta.type} ${(0, index_1.tagId)(maybeBaseMeta, id)} must be instantiated via a subtype`);
         }
         // Look for the CTI __class from the driver telling us which subtype to instantiate
-        return maybeBaseMeta.subTypesByType.get(row.__class) ?? maybeBaseMeta;
+        return maybeBaseMeta.subTypesByType.get(__class) ?? maybeBaseMeta;
     }
     else if (maybeBaseMeta.inheritanceType === "sti") {
         // Look for the STI discriminator value
         const baseMeta = (0, index_1.getBaseMeta)(maybeBaseMeta);
-        const value = row[baseMeta.stiDiscriminatorColumnName];
+        const value = rowData.get(rowIndex, baseMeta.stiDiscriminatorColumnName);
         return baseMeta.subTypesByStiValue.get(value) ?? baseMeta;
     }
     else {
@@ -2278,7 +2545,7 @@ const fieldMap = {};
 // Generates what a row from the db would look like for a given entity
 function createRowFromEntityData(e, opts = {}) {
     const { preferOriginalData = true } = opts;
-    const { row: oldRow, data, originalData } = e.__data;
+    const { rowData, rowIndex, data, originalData } = e.__data;
     const __class = e.constructor.name;
     const { metadata: meta } = e.__data;
     if (!fieldMap[__class]) {
@@ -2303,8 +2570,8 @@ function createRowFromEntityData(e, opts = {}) {
                 // reflect what would come from the db if we queried it right now, so use originalData when present
                 column.rowValue(preferOriginalData && field.fieldName in originalData ? originalData : data)
             : // `data` is lazy and isn't set until it's accessed, so if the field isn't present there, then we should
-                // be safe to pull the raw data out of `row`
-                oldRow[column.columnName];
+                // be safe to pull the raw data out of the as-loaded RowData
+                rowData.get(rowIndex, column.columnName);
         row[column.columnName] = value ?? null;
     }
     return row;