@incoqnito.io/bajadab 1.2.2 → 1.3.0

package/dist/basics.js

@@ -1,17 +1,15 @@
-import logger from "winston";
+import { AsyncLocalStorage } from "node:async_hooks";
 import { uuidv7, } from "uuidv7";
-import { runMultiReentrantProtected, runReentrantProtected } from "@incoqnito.io/ts-iq-core";
+import { runReentrantProtected } from "@incoqnito.io/ts-iq-core";
 import { EBArrayDeletionStrategy, EBAutoFlushStrategy, EBAutoIDStrategy, EBDirtyUnloadStrategy, EBDuplicateIDStrategy, EBHandlerMode, EBMemStorageStrategy, EBModificationType, } from "./base.js";
 import { canonicalName } from "./utils.js";
 /** Default {@link TGetIDFunction}: reads `v.id`, `undefined` if absent. */
-export const DEFAULT_GET_ID = (v) => ("id" in v) ? v.id : undefined;
+export const DEFAULT_GET_ID = (v) => (v !== null && typeof v === 'object' && "id" in v) ? v.id : undefined;
 /** Default {@link TSetIDFunction}: assigns to `v.id`. */
 export const DEFAULT_SET_ID = async (v, id) => { v.id = id; };
-var MutexKeys;
-(function (MutexKeys) {
-    // because of the spaces this will never be a collection name and as such is safe to use as a mutex key
-    MutexKeys["DATABASE"] = " database ";
-})(MutexKeys || (MutexKeys = {}));
+function normalizeID(id) {
+    return id === undefined || id === null || id === "" ? undefined : String(id);
+}
 /** {@link IBCollectionConfig} used when a collection is created without an explicit one. */
 export const DEFAULT_COLLECTION_CONFIG = {
     arrayDeletionStrategy: EBArrayDeletionStrategy.IN_PLACE,
@@ -30,16 +28,24 @@ export class BDatabase {
     collections = new Map();
     io;
     functionRegistry;
+    iid = uuidv7();
+    transactionScope = new AsyncLocalStorage();
     constructor(io, config, functionRegistry) {
-        this.config = config ?? DEFAULT_DATABASE_CONFIG;
+        this.config = { ...(config ?? DEFAULT_DATABASE_CONFIG) };
         this.io = io;
         this.functionRegistry = functionRegistry;
     }
+    async runExclusive(fn) {
+        return runReentrantProtected(this.iid, fn);
+    }
+    isInTransaction(collectionName) {
+        return this.transactionScope.getStore()?.has(collectionName) ?? false;
+    }
     addCollection(collection) {
         this.collections.set(collection.collectionName, collection);
     }
     async getCollection(collectionName, collectionConfig) {
-        return runReentrantProtected(MutexKeys.DATABASE, async () => {
+        return this.runExclusive(async () => {
             const cname = canonicalName(collectionName);
             let ret = this.collections.get(cname);
             if (!ret) {
@@ -49,25 +55,26 @@ export class BDatabase {
         });
     }
     async createCollection(collectionName, collectionConfig) {
-        return runReentrantProtected(MutexKeys.DATABASE, async () => {
+        return this.runExclusive(async () => {
             const cname = canonicalName(collectionName);
             if (this.collections.has(cname)) {
                 throw new Error(`cannot create already existing collection ${collectionName}`);
             }
             const id = uuidv7();
-            const ret = new BCollection(this, id, collectionName, collectionConfig ?? this.config.defaultCollectionConfig);
+            const ret = new BCollection(this, id, collectionName, { ...(collectionConfig ?? this.config.defaultCollectionConfig) });
+            await this.io.createCollection(ret);
             this.addCollection(ret);
-            return this.io.createCollection(ret);
+            return ret;
         });
     }
     async hasCollection(collectionName) {
-        return runReentrantProtected(MutexKeys.DATABASE, async () => {
+        return this.runExclusive(async () => {
             return this.collections.has(canonicalName(collectionName));
         });
     }
     async dropCollection(collectionName) {
         const cname = canonicalName(collectionName);
-        return runMultiReentrantProtected([MutexKeys.DATABASE, cname], async () => {
+        return this.runExclusive(async () => {
             const coll = this.collections.get(cname);
             if (coll) {
                 coll.markDropped();
@@ -80,8 +87,7 @@ export class BDatabase {
         });
     }
     async snapshot(targetDir, options) {
-        const collectionNames = Array.from(this.collections.keys());
-        return runMultiReentrantProtected([MutexKeys.DATABASE, ...collectionNames], async () => {
+        return this.runExclusive(async () => {
             for (const collection of this.collections.values()) {
                 await collection.flush();
             }
@@ -89,13 +95,21 @@ export class BDatabase {
         });
     }
     async transaction(collectionNames, fn) {
+        if (this.transactionScope.getStore() !== undefined) {
+            throw new Error("nested transactions are not supported");
+        }
         const canonicalNames = Array.from(new Set(collectionNames.map(canonicalName)));
-        return runMultiReentrantProtected(canonicalNames, async () => {
+        return this.runExclusive(async () => {
             const originals = new Map();
             const clones = new Map();
             for (const name of canonicalNames) {
-                const original = await this.getCollection(name);
+                const original = this.collections.get(name);
+                if (!original) {
+                    throw new Error(`no such collection ${name}`);
+                }
                 originals.set(name, original);
+            }
+            for (const [name, original] of originals) {
                 clones.set(name, await original.cloneForTransaction());
             }
             const tx = {
@@ -107,9 +121,36 @@ export class BDatabase {
                     return clone;
                 },
             };
-            await fn(tx);
-            for (const name of canonicalNames) {
-                await originals.get(name).commitTransactionClone(clones.get(name));
+            try {
+                await this.transactionScope.run(new Set(canonicalNames), () => fn(tx));
+                const prepared = [];
+                try {
+                    for (const name of canonicalNames) {
+                        prepared.push(await originals.get(name).prepareTransactionCommit(clones.get(name)));
+                    }
+                }
+                catch (err) {
+                    await Promise.all(prepared.map(p => p.write.discard()));
+                    throw err;
+                }
+                let renamed = 0;
+                try {
+                    for (const p of prepared) {
+                        await p.write.commit();
+                        renamed++;
+                    }
+                }
+                finally {
+                    await Promise.all(prepared.slice(renamed).map(p => p.write.discard()));
+                    const applied = prepared.slice(0, renamed);
+                    applied.forEach(p => p.apply());
+                    for (const p of applied) {
+                        await p.fire();
+                    }
+                }
+            }
+            finally {
+                clones.forEach(clone => clone.markDropped());
             }
         });
     }
@@ -124,26 +165,19 @@ export class BCollection {
     loaded = false;
     dirty = false;
     dropped = false;
+    transactionClone = false;
     addHandlers = [];
     updateHandlers = [];
     deleteHandlers = [];
     indexDefinitions = new Map();
     indexRuntime = new Map();
+    indexedKeys = new Map();
+    txChanges;
     constructor(database, collectionID, collectionName, collectionConfig) {
         this.database = database;
         this.collectionID = collectionID;
         this.collectionName = canonicalName(collectionName);
         this.collectionConfig = collectionConfig;
-        switch (collectionConfig.memStorageStrategy) {
-            case EBMemStorageStrategy.MAP:
-                // if (!collectionConfig.getID || !collectionConfig.setID) {
-                //     throw new Error("need id accessors configured when using map mem storage");
-                // }
-                break;
-            default:
-            case EBMemStorageStrategy.ARRAY:
-                break;
-        }
     }
     async checkLoaded() {
         if (!this.loaded) {
@@ -151,7 +185,7 @@ export class BCollection {
         }
     }
     async runExclusive(fn) {
-        return runReentrantProtected(this.collectionName, fn);
+        return this.database.runExclusive(fn);
     }
     async runProtected(fn) {
         return this.runExclusive(async () => {
@@ -161,6 +195,14 @@ export class BCollection {
             return fn();
         });
     }
+    async runMutation(fn) {
+        return this.runProtected(async () => {
+            if (!this.transactionClone && this.database.isInTransaction(this.collectionName)) {
+                throw new Error(`collection "${this.collectionName}" is part of a running transaction; write through tx.getCollection() instead`);
+            }
+            return fn();
+        });
+    }
     /**
      * call this only within the protecting mutex
      */
@@ -206,7 +248,7 @@ export class BCollection {
             if (!byKey) {
                 throw new Error(`index "${indexKey}" is not configured`);
             }
-            return byKey.get(key)?.slice() ?? [];
+            return Array.from(byKey.get(key) ?? []);
         });
     }
     async invokeHandler(fn, modificationType) {
@@ -214,7 +256,8 @@ export class BCollection {
             await fn();
         }
         catch (error) {
-            logger.error(`bajadab: collection "${this.collectionName}" ${modificationType} handler failed`, error);
+            // TODO switch logging to ILogging where necessary
+            console.error(`bajadab: collection "${this.collectionName}" ${modificationType} handler failed`, error);
         }
     }
     async fireAdd(added) {
@@ -271,10 +314,11 @@ export class BCollection {
         });
     }
     async delete(predicate) {
-        return this.runProtected(async () => {
+        return this.runMutation(async () => {
             await this.checkLoaded();
             let ret = 0;
             const trackDeletes = this.deleteHandlers.length > 0;
+            const collectDeleted = trackDeletes || this.txChanges !== undefined;
             const indexDeletes = this.indexDefinitions.size > 0;
             const deleted = [];
             switch (this.collectionConfig.memStorageStrategy) {
@@ -283,10 +327,10 @@ export class BCollection {
                         if (predicate(value)) {
                             ret++;
                             if (indexDeletes) {
-                                await this.removeFromIndexes(value);
+                                this.removeFromIndexes(value);
                             }
                             this.mapData.delete(key);
-                            if (trackDeletes) {
+                            if (collectDeleted) {
                                 deleted.push(value);
                             }
                         }
@@ -306,9 +350,9 @@ export class BCollection {
                                 else {
                                     ret++;
                                     if (indexDeletes) {
-                                        await this.removeFromIndexes(item);
+                                        this.removeFromIndexes(item);
                                     }
-                                    if (trackDeletes) {
+                                    if (collectDeleted) {
                                         deleted.push(item);
                                     }
                                 }
@@ -317,20 +361,29 @@ export class BCollection {
                             break;
                         default:
                         case EBArrayDeletionStrategy.NEW_ARRAY:
-                            const ol = this.arrayData.length;
-                            if (trackDeletes || indexDeletes) {
-                                const removed = this.arrayData.filter(predicate);
-                                if (trackDeletes) {
-                                    deleted.push(...removed);
-                                }
-                                if (indexDeletes) {
-                                    for (const v of removed) {
-                                        await this.removeFromIndexes(v);
+                            if (collectDeleted || indexDeletes) {
+                                const kept = [];
+                                for (const v of this.arrayData) {
+                                    if (predicate(v)) {
+                                        ret++;
+                                        if (indexDeletes) {
+                                            this.removeFromIndexes(v);
+                                        }
+                                        if (collectDeleted) {
+                                            deleted.push(v);
+                                        }
+                                    }
+                                    else {
+                                        kept.push(v);
                                     }
                                 }
+                                this.arrayData = kept;
+                            }
+                            else {
+                                const ol = this.arrayData.length;
+                                this.arrayData = this.arrayData.filter(v => !predicate(v));
+                                ret += (ol - this.arrayData.length);
                             }
-                            this.arrayData = this.arrayData.filter(v => !predicate(v));
-                            ret += (ol - this.arrayData.length);
                             break;
                     }
                     break;
@@ -338,7 +391,8 @@ export class BCollection {
             if (ret > 0) {
                 await this.doOnDirty();
             }
-            if (deleted.length > 0) {
+            await this.recordTxChanges(EBModificationType.DELETE, deleted);
+            if (trackDeletes && deleted.length > 0) {
                 await this.fireDelete(deleted);
             }
             return ret;
@@ -349,10 +403,10 @@ export class BCollection {
             await this.checkLoaded();
             switch (this.collectionConfig.memStorageStrategy) {
                 case EBMemStorageStrategy.MAP:
-                    return this.mapData.values().find(predicate) !== undefined;
+                    return this.mapData.values().some(predicate);
                 default:
                 case EBMemStorageStrategy.ARRAY:
-                    return this.arrayData.find(predicate) !== undefined;
+                    return this.arrayData.some(predicate);
             }
         });
     }
@@ -381,31 +435,46 @@ export class BCollection {
         return result;
     }
     async addToIndexes(v) {
-        for (const [indexKey, keys] of await this.indexKeysFor(v)) {
+        this.addKeysToIndexes(v, await this.indexKeysFor(v));
+    }
+    async indexKeysForAll(vs) {
+        const ret = [];
+        for (const v of vs) {
+            ret.push(await this.indexKeysFor(v));
+        }
+        return ret;
+    }
+    addKeysToIndexes(v, indexKeys) {
+        for (const [indexKey, keys] of indexKeys) {
             const byKey = this.indexRuntime.get(indexKey);
             for (const key of keys) {
                 let bucket = byKey.get(key);
                 if (!bucket) {
-                    bucket = [];
+                    bucket = new Set();
                     byKey.set(key, bucket);
                 }
-                bucket.push(v);
+                bucket.add(v);
             }
         }
+        if (indexKeys.length > 0) {
+            this.indexedKeys.set(v, indexKeys);
+        }
     }
-    async removeFromIndexes(v) {
-        for (const [indexKey, keys] of await this.indexKeysFor(v)) {
+    removeFromIndexes(v) {
+        const indexKeys = this.indexedKeys.get(v);
+        if (!indexKeys) {
+            return;
+        }
+        this.indexedKeys.delete(v);
+        for (const [indexKey, keys] of indexKeys) {
             const byKey = this.indexRuntime.get(indexKey);
             for (const key of keys) {
                 const bucket = byKey.get(key);
                 if (!bucket) {
                     continue;
                 }
-                const at = bucket.indexOf(v);
-                if (at !== -1) {
-                    bucket.splice(at, 1);
-                }
-                if (bucket.length === 0) {
+                bucket.delete(v);
+                if (bucket.size === 0) {
                     byKey.delete(key);
                 }
             }
@@ -419,6 +488,7 @@ export class BCollection {
         // every configured index gets an entry up front, even if empty, so getByIndex()/
         // addToIndexes()/removeFromIndexes() can assume it's there instead of checking twice
         this.indexRuntime = new Map(Array.from(this.indexDefinitions.keys(), (indexKey) => [indexKey, new Map()]));
+        this.indexedKeys = new Map();
         if (this.indexDefinitions.size === 0) {
             return;
         }
@@ -441,18 +511,18 @@ export class BCollection {
     /**
      * id -> value for every stored item, resolved via the plain id getter (no auto-id assignment - every
      * stored item already has one). Used to diff a transaction's working copy against the pre-transaction
-     * state in {@link commitTransactionClone}.
+     * state in {@link prepareTransactionCommit}.
      */
     async idKeyedSnapshot() {
         switch (this.collectionConfig.memStorageStrategy) {
             case EBMemStorageStrategy.MAP:
-                return new Map(this.mapData);
+                return this.mapData;
             default:
             case EBMemStorageStrategy.ARRAY:
                 const getter = this.resolveIDGetter();
                 const ret = new Map();
                 for (const v of this.arrayData) {
-                    const id = await getter(v);
+                    const id = normalizeID(await getter(v));
                     if (id) {
                         ret.set(id, v);
                     }
@@ -460,43 +530,90 @@ export class BCollection {
                 return ret;
         }
     }
-    async getID(v) {
-        let id = await this.resolveIDGetter()(v);
-        if (!id) {
-            switch (this.collectionConfig.autoIDStrategy) {
-                case EBAutoIDStrategy.NO_AUTO_ID:
-                    throw new Error("cannot omit auto-ids and than trying to add a value without an id");
-                default:
-                case EBAutoIDStrategy.AUTO_ID_UUID:
-                    id = uuidv7();
-                    let setter;
-                    if (this.collectionConfig.setID) {
-                        const found = this.database.functionRegistry?.fetch(this.collectionConfig.setID);
-                        if (!found) {
-                            throw new Error(`setID function "${this.collectionConfig.setID}" is not registered`);
-                        }
-                        setter = found;
-                    }
-                    else {
-                        setter = DEFAULT_SET_ID;
-                    }
-                    await setter(v, id);
-                    break;
+    resolveIDSetter() {
+        if (this.collectionConfig.setID) {
+            const found = this.database.functionRegistry?.fetch(this.collectionConfig.setID);
+            if (!found) {
+                throw new Error(`setID function "${this.collectionConfig.setID}" is not registered`);
+            }
+            return found;
+        }
+        return DEFAULT_SET_ID;
+    }
+    async resolveIncomingIDs(vs) {
+        const getter = this.resolveIDGetter();
+        const ids = [];
+        for (const v of vs) {
+            const id = normalizeID(await getter(v));
+            if (!id && this.collectionConfig.autoIDStrategy === EBAutoIDStrategy.NO_AUTO_ID) {
+                throw new Error("cannot omit auto-ids and than trying to add a value without an id");
+            }
+            ids.push(id);
+        }
+        return ids;
+    }
+    async assignMissingIDs(vs, ids) {
+        let setter;
+        const ret = [];
+        for (let i = 0; i < vs.length; i++) {
+            let id = ids[i];
+            if (!id) {
+                setter ??= this.resolveIDSetter();
+                id = uuidv7();
+                await setter(vs[i], id);
+            }
+            ret.push(id);
+        }
+        return ret;
+    }
+    async storedArrayIDs() {
+        const getter = this.resolveIDGetter();
+        const ids = [];
+        for (const v of this.arrayData) {
+            ids.push(normalizeID(await getter(v)));
+        }
+        return ids;
+    }
+    async recordTxChanges(modificationType, values) {
+        if (!this.txChanges || values.length === 0) {
+            return;
+        }
+        const getter = this.resolveIDGetter();
+        for (const v of values) {
+            const id = normalizeID(await getter(v));
+            if (id) {
+                this.txChanges.set(id, modificationType);
             }
         }
-        return id;
     }
     async add(...vs) {
-        return this.runProtected(async () => {
+        return this.runMutation(async () => {
             await this.checkLoaded();
+            const trackAdds = this.addHandlers.length > 0;
+            const collectAdded = trackAdds || this.txChanges !== undefined;
+            const indexAdds = this.indexDefinitions.size > 0;
+            const incomingIDs = await this.resolveIncomingIDs(vs);
+            const added = [];
+            let ret = 0;
             switch (this.collectionConfig.memStorageStrategy) {
-                case EBMemStorageStrategy.MAP:
-                    let ret = 0;
-                    const trackAddsMap = this.addHandlers.length > 0;
-                    const indexAddsMap = this.indexDefinitions.size > 0;
-                    const addedMap = [];
-                    for (const v of vs) {
-                        const id = await this.getID(v);
+                case EBMemStorageStrategy.MAP: {
+                    if (this.collectionConfig.duplicateIDStrategy === EBDuplicateIDStrategy.ERROR) {
+                        const batchIDs = new Set();
+                        for (const id of incomingIDs) {
+                            if (id === undefined) {
+                                continue;
+                            }
+                            if (this.mapData.has(id) || batchIDs.has(id)) {
+                                throw new Error(`duplicate id ${id}`);
+                            }
+                            batchIDs.add(id);
+                        }
+                    }
+                    const ids = await this.assignMissingIDs(vs, incomingIDs);
+                    const indexKeys = indexAdds ? await this.indexKeysForAll(vs) : [];
+                    for (let i = 0; i < vs.length; i++) {
+                        const id = ids[i];
+                        const v = vs[i];
                         if (this.mapData.has(id)) {
                             switch (this.collectionConfig.duplicateIDStrategy) {
                                 case EBDuplicateIDStrategy.ERROR:
@@ -508,15 +625,15 @@ export class BCollection {
                                 case EBDuplicateIDStrategy.OVERWRITE:
                                     // replace the new value
                                     ret++;
-                                    if (indexAddsMap) {
-                                        await this.removeFromIndexes(this.mapData.get(id));
+                                    if (indexAdds) {
+                                        this.removeFromIndexes(this.mapData.get(id));
                                     }
                                     this.mapData.set(id, v);
-                                    if (indexAddsMap) {
-                                        await this.addToIndexes(v);
+                                    if (indexAdds) {
+                                        this.addKeysToIndexes(v, indexKeys[i]);
                                     }
-                                    if (trackAddsMap) {
-                                        addedMap.push(v);
+                                    if (collectAdded) {
+                                        added.push(v);
                                     }
                                     break;
                             }
@@ -524,169 +641,164 @@ export class BCollection {
                         else {
                             ret++;
                             this.mapData.set(id, v);
-                            if (indexAddsMap) {
-                                await this.addToIndexes(v);
+                            if (indexAdds) {
+                                this.addKeysToIndexes(v, indexKeys[i]);
                             }
-                            if (trackAddsMap) {
-                                addedMap.push(v);
+                            if (collectAdded) {
+                                added.push(v);
                             }
                         }
                     }
-                    if (ret > 0) {
-                        await this.doOnDirty();
-                    }
-                    if (addedMap.length > 0) {
-                        await this.fireAdd(addedMap);
-                    }
-                    return ret;
+                    break;
+                }
                 default:
-                case EBMemStorageStrategy.ARRAY:
-                    const trackAddsArray = this.addHandlers.length > 0;
-                    const indexAddsArray = this.indexDefinitions.size > 0;
-                    const addedArray = [];
+                case EBMemStorageStrategy.ARRAY: {
+                    const storedIDs = await this.storedArrayIDs();
                     switch (this.collectionConfig.duplicateIDStrategy) {
-                        case EBDuplicateIDStrategy.ERROR:
-                            const haveIDs = new Set();
-                            for (const v of this.arrayData) {
-                                haveIDs.add(await this.getID(v));
-                            }
-                            for (const v of vs) {
-                                const id = await this.getID(v);
+                        case EBDuplicateIDStrategy.ERROR: {
+                            const haveIDs = new Set(storedIDs.filter((id) => id !== undefined));
+                            for (const id of incomingIDs) {
+                                if (id === undefined) {
+                                    continue;
+                                }
                                 if (haveIDs.has(id)) {
                                     throw new Error(`duplicate id ${id}`);
                                 }
                                 haveIDs.add(id);
+                            }
+                            await this.assignMissingIDs(vs, incomingIDs);
+                            const indexKeys = indexAdds ? await this.indexKeysForAll(vs) : [];
+                            vs.forEach((v, i) => {
+                                ret++;
                                 this.arrayData.push(v);
-                                if (indexAddsArray) {
-                                    await this.addToIndexes(v);
+                                if (indexAdds) {
+                                    this.addKeysToIndexes(v, indexKeys[i]);
                                 }
-                                if (trackAddsArray) {
-                                    addedArray.push(v);
+                                if (collectAdded) {
+                                    added.push(v);
                                 }
-                            }
+                            });
                             break;
-                        case EBDuplicateIDStrategy.IGNORE:
-                            const seenIDs = new Set();
-                            for (const v of this.arrayData) {
-                                seenIDs.add(await this.getID(v));
-                            }
-                            for (const v of vs) {
-                                const id = await this.getID(v);
+                        }
+                        case EBDuplicateIDStrategy.IGNORE: {
+                            const ids = await this.assignMissingIDs(vs, incomingIDs);
+                            const seenIDs = new Set(storedIDs.filter((id) => id !== undefined));
+                            const toAdd = [];
+                            ids.forEach((id, i) => {
                                 if (!seenIDs.has(id)) {
                                     seenIDs.add(id);
-                                    this.arrayData.push(v);
-                                    if (indexAddsArray) {
-                                        await this.addToIndexes(v);
-                                    }
-                                    if (trackAddsArray) {
-                                        addedArray.push(v);
-                                    }
+                                    toAdd.push(vs[i]);
                                 }
-                            }
+                            });
+                            const indexKeys = indexAdds ? await this.indexKeysForAll(toAdd) : [];
+                            toAdd.forEach((v, i) => {
+                                ret++;
+                                this.arrayData.push(v);
+                                if (indexAdds) {
+                                    this.addKeysToIndexes(v, indexKeys[i]);
+                                }
+                                if (collectAdded) {
+                                    added.push(v);
+                                }
+                            });
                             break;
+                        }
                         default:
-                        case EBDuplicateIDStrategy.OVERWRITE:
-                            const indexMap = new Map();
-                            for (let i = 0; i < this.arrayData.length; i++) {
-                                indexMap.set(await this.getID(this.arrayData[i]), i);
-                            }
-                            for (const v of vs) {
-                                const id = await this.getID(v);
-                                const existingIndex = indexMap.get(id);
+                        case EBDuplicateIDStrategy.OVERWRITE: {
+                            const ids = await this.assignMissingIDs(vs, incomingIDs);
+                            const indexKeys = indexAdds ? await this.indexKeysForAll(vs) : [];
+                            const positions = new Map();
+                            storedIDs.forEach((id, i) => {
+                                if (id !== undefined) {
+                                    positions.set(id, i);
+                                }
+                            });
+                            vs.forEach((v, i) => {
+                                const id = ids[i];
+                                const existingIndex = positions.get(id);
+                                ret++;
                                 if (existingIndex !== undefined) {
-                                    if (indexAddsArray) {
-                                        await this.removeFromIndexes(this.arrayData[existingIndex]);
+                                    if (indexAdds) {
+                                        this.removeFromIndexes(this.arrayData[existingIndex]);
                                     }
                                     this.arrayData[existingIndex] = v;
                                 }
                                 else {
-                                    // store new index in indexCache in the case this id appears again
-                                    indexMap.set(id, this.arrayData.length);
+                                    positions.set(id, this.arrayData.length);
                                     this.arrayData.push(v);
                                 }
-                                if (indexAddsArray) {
-                                    await this.addToIndexes(v);
+                                if (indexAdds) {
+                                    this.addKeysToIndexes(v, indexKeys[i]);
                                 }
-                                if (trackAddsArray) {
-                                    addedArray.push(v);
+                                if (collectAdded) {
+                                    added.push(v);
                                 }
-                            }
+                            });
                             break;
+                        }
                     }
-                    if (vs.length > 0) {
-                        await this.doOnDirty();
-                    }
-                    if (addedArray.length > 0) {
-                        await this.fireAdd(addedArray);
-                    }
-                    return vs.length;
+                    break;
+                }
+            }
+            if (ret > 0) {
+                await this.doOnDirty();
             }
+            await this.recordTxChanges(EBModificationType.ADD, added);
+            if (trackAdds && added.length > 0) {
+                await this.fireAdd(added);
+            }
+            return ret;
         });
     }
     async update(predicate, updater) {
-        return this.runProtected(async () => {
+        return this.runMutation(async () => {
             await this.checkLoaded();
-            let ret = 0;
             const trackUpdates = this.updateHandlers.length > 0;
             const indexUpdates = this.indexDefinitions.size > 0;
-            const updated = [];
+            const planned = [];
+            const plan = async (previous, store) => {
+                const replacement = await updater(previous);
+                if (replacement === undefined) {
+                    throw new Error("updater gave undefined");
+                }
+                const replacementKeys = indexUpdates ? await this.indexKeysFor(replacement) : [];
+                planned.push({ previous, replacement, replacementKeys, store });
+            };
             switch (this.collectionConfig.memStorageStrategy) {
                 case EBMemStorageStrategy.MAP:
                     for (const [key, value] of this.mapData) {
                         if (predicate(value)) {
-                            const replacement = await updater(value);
-                            if (replacement !== undefined) {
-                                ret++;
-                                if (indexUpdates) {
-                                    await this.removeFromIndexes(value);
-                                }
-                                this.mapData.set(key, replacement);
-                                if (indexUpdates) {
-                                    await this.addToIndexes(replacement);
-                                }
-                                if (trackUpdates) {
-                                    updated.push([value, replacement]);
-                                }
-                            }
-                            else {
-                                throw new Error("updater gave undefined");
-                            }
+                            await plan(value, (replacement) => { this.mapData.set(key, replacement); });
                         }
                     }
                     break;
                 default:
                 case EBMemStorageStrategy.ARRAY:
                     for (let i = 0; i < this.arrayData.length; i++) {
-                        if (predicate(this.arrayData[i])) {
-                            const previous = this.arrayData[i];
-                            const replacement = await updater(previous);
-                            if (replacement !== undefined) {
-                                ret++;
-                                if (indexUpdates) {
-                                    await this.removeFromIndexes(previous);
-                                }
-                                this.arrayData[i] = replacement;
-                                if (indexUpdates) {
-                                    await this.addToIndexes(replacement);
-                                }
-                                if (trackUpdates) {
-                                    updated.push([previous, replacement]);
-                                }
-                            }
-                            else {
-                                throw new Error("updater gave undefined");
-                            }
+                        const previous = this.arrayData[i];
+                        if (predicate(previous)) {
+                            await plan(previous, (replacement) => { this.arrayData[i] = replacement; });
                         }
                     }
                     break;
             }
-            if (ret > 0) {
+            for (const { previous, replacement, replacementKeys, store } of planned) {
+                if (indexUpdates) {
+                    this.removeFromIndexes(previous);
+                }
+                store(replacement);
+                if (indexUpdates) {
+                    this.addKeysToIndexes(replacement, replacementKeys);
+                }
+            }
+            if (planned.length > 0) {
                 await this.doOnDirty();
             }
-            if (updated.length > 0) {
-                await this.fireUpdate(updated);
+            await this.recordTxChanges(EBModificationType.UPDATE, planned.map(({ replacement }) => replacement));
+            if (trackUpdates && planned.length > 0) {
+                await this.fireUpdate(planned.map(({ previous, replacement }) => [previous, replacement]));
             }
-            return ret;
+            return planned.length;
         });
     }
     async doOnDirty() {
@@ -722,10 +834,10 @@ export class BCollection {
                     this.arrayData = await this.database.io.loadArrayData(this.collectionID);
                     break;
             }
-            this.loaded = true;
-            this.dirty = false;
             this.resolveIndexDefinitions();
             await this.rebuildIndexRuntime();
+            this.loaded = true;
+            this.dirty = false;
         });
     }
     async unload() {
@@ -752,6 +864,8 @@ export class BCollection {
                     this.arrayData = [];
                     break;
             }
+            this.indexRuntime.clear();
+            this.indexedKeys.clear();
             this.loaded = false;
             this.dirty = false;
         });
@@ -770,62 +884,75 @@ export class BCollection {
             clone.mapData = new Map(this.mapData);
             clone.arrayData = this.arrayData.slice();
             clone.loaded = true;
+            clone.transactionClone = true;
+            if (this.addHandlers.length > 0 || this.updateHandlers.length > 0 || this.deleteHandlers.length > 0) {
+                clone.txChanges = new Map();
+            }
             clone.resolveIndexDefinitions();
             await clone.rebuildIndexRuntime();
             return clone;
         });
     }
     /**
-     * Applies a working copy from {@link cloneForTransaction} back onto this collection: replaces the live
-     * state, flushes once, then fires `ADD`/`UPDATE`/`DELETE` handlers for the net difference between this
-     * collection's state before the transaction and `clone`'s state - not for each intermediate operation the
-     * transaction performed. The before/after diff itself (and the id resolution it needs) only runs when at
-     * least one handler is registered on this collection - a transaction on a collection nobody listens to
-     * skips it entirely. Call only while already holding this collection's mutex for the whole transaction.
+     * Writes `clone`'s state to a temp file (see {@link IPreparedWrite}) and computes the handler events for the
+     * net changes against this collection's current state: one per id the transaction touched, typed by the last
+     * operation on it (`DELETE` if gone, `ADD` if new or last stored via `add()`, `UPDATE` otherwise). Nothing is
+     * applied until `apply()`; call only while already holding this collection's mutex for the whole transaction.
      */
-    async commitTransactionClone(clone) {
+    async prepareTransactionCommit(clone) {
         return this.runProtected(async () => {
             await this.checkLoaded();
             const added = [];
             const updated = [];
             const deleted = [];
-            // the before/after diff only feeds fireAdd()/fireUpdate()/fireDelete() below - skip it
-            // entirely (incl. idKeyedSnapshot()'s id-getter calls) when nothing is listening
-            const trackChanges = this.addHandlers.length > 0 || this.updateHandlers.length > 0 || this.deleteHandlers.length > 0;
-            if (trackChanges) {
+            if (clone.txChanges && clone.txChanges.size > 0) {
                 const before = await this.idKeyedSnapshot();
                 const after = await clone.idKeyedSnapshot();
-                for (const [id, value] of after) {
-                    if (!before.has(id)) {
-                        added.push(value);
-                    }
-                    else {
-                        const previous = before.get(id);
-                        if (JSON.stringify(previous) !== JSON.stringify(value)) {
-                            updated.push([previous, value]);
+                for (const [id, modificationType] of clone.txChanges) {
+                    if (!after.has(id)) {
+                        if (before.has(id)) {
+                            deleted.push(before.get(id));
                         }
                     }
-                }
-                for (const [id, value] of before) {
-                    if (!after.has(id)) {
-                        deleted.push(value);
+                    else if (!before.has(id) || modificationType === EBModificationType.ADD) {
+                        added.push(after.get(id));
+                    }
+                    else {
+                        updated.push([before.get(id), after.get(id)]);
                     }
                 }
             }
-            this.mapData = clone.mapData;
-            this.arrayData = clone.arrayData;
-            this.indexRuntime = clone.indexRuntime;
-            this.dirty = true;
-            await this.flush();
-            if (added.length > 0) {
-                await this.fireAdd(added);
-            }
-            if (updated.length > 0) {
-                await this.fireUpdate(updated);
-            }
-            if (deleted.length > 0) {
-                await this.fireDelete(deleted);
+            let write;
+            switch (this.collectionConfig.memStorageStrategy) {
+                case EBMemStorageStrategy.MAP:
+                    write = await this.database.io.prepareMapData(this.collectionID, clone.mapData);
+                    break;
+                default:
+                case EBMemStorageStrategy.ARRAY:
+                    write = await this.database.io.prepareArrayData(this.collectionID, clone.arrayData);
+                    break;
             }
+            return {
+                write,
+                apply: () => {
+                    this.mapData = clone.mapData;
+                    this.arrayData = clone.arrayData;
+                    this.indexRuntime = clone.indexRuntime;
+                    this.indexedKeys = clone.indexedKeys;
+                    this.dirty = false;
+                },
+                fire: () => this.runProtected(async () => {
+                    if (added.length > 0) {
+                        await this.fireAdd(added);
+                    }
+                    if (updated.length > 0) {
+                        await this.fireUpdate(updated);
+                    }
+                    if (deleted.length > 0) {
+                        await this.fireDelete(deleted);
+                    }
+                }),
+            };
         });
     }
 }