@incoqnito.io/bajadab 1.2.2 → 1.3.0

package/src/main/basics.ts

@@ -1,913 +1,1059 @@
-import logger from "winston";
-
-import { uuidv7, } from "uuidv7";
-import { runMultiReentrantProtected, runReentrantProtected } from "@incoqnito.io/ts-iq-core";
-
-import type { IODatabase } from "./fileio.js";
-
-import {
-    EBArrayDeletionStrategy,
-    EBAutoFlushStrategy,
-    EBAutoIDStrategy,
-    EBDirtyUnloadStrategy,
-    EBDuplicateIDStrategy,
-    EBHandlerMode,
-    EBMemStorageStrategy,
-    EBModificationType,
-    IBCollection,
-    IBCollectionConfig,
-    IBDatabase,
-    IBDatabaseConfig,
-    IBFunctionRegistry,
-    IBSnapshotOptions,
-    IBTransaction,
-    IBTransactionCollection,
-    IHandlerRegistration,
-    TAddHandler,
-    TDeleteHandler,
-    TGetIDFunction,
-    TIndexKeyFunction,
-    TSetIDFunction,
-    TUpdateHandler,
-} from "./base.js";
-import { canonicalName } from "./utils.js";
-
-/** Default {@link TGetIDFunction}: reads `v.id`, `undefined` if absent. */
-export const DEFAULT_GET_ID: TGetIDFunction = (v: any) => ("id" in v) ? v.id : undefined;
-/** Default {@link TSetIDFunction}: assigns to `v.id`. */
-export const DEFAULT_SET_ID: TSetIDFunction = async (v: any, id: string) => { v.id = id; };
-
-const enum MutexKeys {
-    // because of the spaces this will never be a collection name and as such is safe to use as a mutex key
-    DATABASE = " database ",
-}
-
-/** {@link IBCollectionConfig} used when a collection is created without an explicit one. */
-export const DEFAULT_COLLECTION_CONFIG: IBCollectionConfig = {
-    arrayDeletionStrategy: EBArrayDeletionStrategy.IN_PLACE,
-    autoIDStrategy: EBAutoIDStrategy.AUTO_ID_UUID,
-    duplicateIDStrategy: EBDuplicateIDStrategy.OVERWRITE,
-    memStorageStrategy: EBMemStorageStrategy.MAP,
-    dirtyUnloadStrategy: EBDirtyUnloadStrategy.FLUSH,
-    autoFlushStrategy: EBAutoFlushStrategy.ALWAYS_AUTO_FLUSH,
-};
-
-/** {@link IBDatabaseConfig} used when `getDatabase()` creates a fresh database. */
-export const DEFAULT_DATABASE_CONFIG: IBDatabaseConfig = {
-    defaultCollectionConfig: DEFAULT_COLLECTION_CONFIG,
-};
-
-export class BDatabase implements IBDatabase {
-    public readonly config: IBDatabaseConfig;
-    private collections: Map<string, BCollection> = new Map();
-    readonly io: IODatabase;
-    readonly functionRegistry?: IBFunctionRegistry;
-
-    public constructor(io: IODatabase, config?: IBDatabaseConfig, functionRegistry?: IBFunctionRegistry) {
-        this.config = config ?? DEFAULT_DATABASE_CONFIG;
-        this.io = io;
-        this.functionRegistry = functionRegistry;
-    }
-
-    public addCollection(collection: BCollection): void {
-        this.collections.set(collection.collectionName, collection);
-    }
-
-    public async getCollection<T>(collectionName: string, collectionConfig?: IBCollectionConfig): Promise<IBCollection<T>> {
-        return runReentrantProtected(MutexKeys.DATABASE, async () => {
-            const cname = canonicalName(collectionName);
-            let ret: BCollection<T> | undefined = this.collections.get(cname);
-            if (!ret) {
-                ret = await this.createCollection(collectionName, collectionConfig);
-            }
-            return ret;
-        });
-    }
-
-    public async createCollection<T = any>(collectionName: string, collectionConfig?: IBCollectionConfig): Promise<BCollection<T>> {
-        return runReentrantProtected(MutexKeys.DATABASE, async () => {
-            const cname = canonicalName(collectionName);
-            if (this.collections.has(cname)) {
-                throw new Error(`cannot create already existing collection ${collectionName}`);
-            }
-            const id = uuidv7();
-            const ret: BCollection<T> = new BCollection<T>(this, id, collectionName, collectionConfig ?? this.config.defaultCollectionConfig);
-            this.addCollection(ret);
-            return this.io.createCollection<T>(ret);
-        });
-    }
-
-    public async hasCollection(collectionName: string): Promise<boolean> {
-        return runReentrantProtected(MutexKeys.DATABASE, async () => {
-            return this.collections.has(canonicalName(collectionName));
-        });
-    }
-
-    public async dropCollection(collectionName: string): Promise<void> {
-        const cname = canonicalName(collectionName);
-        return runMultiReentrantProtected([MutexKeys.DATABASE, cname], async () => {
-            const coll: BCollection = this.collections.get(cname) as BCollection;
-            if (coll) {
-                coll.markDropped();
-                this.collections.delete(cname);
-                return this.io.deleteCollection(cname);
-            }
-            else {
-                throw new Error(`no such collection ${collectionName}`);
-            }
-        });
-    }
-
-    public async snapshot(targetDir: string, options?: IBSnapshotOptions): Promise<string> {
-        const collectionNames = Array.from(this.collections.keys());
-        return runMultiReentrantProtected([MutexKeys.DATABASE, ...collectionNames], async () => {
-            for (const collection of this.collections.values()) {
-                await collection.flush();
-            }
-            return this.io.snapshot(targetDir, options);
-        });
-    }
-
-    public async transaction(collectionNames: string[], fn: (tx: IBTransaction) => Promise<void>): Promise<void> {
-        const canonicalNames = Array.from(new Set(collectionNames.map(canonicalName)));
-        return runMultiReentrantProtected(canonicalNames, async () => {
-            const originals = new Map<string, BCollection>();
-            const clones = new Map<string, BCollection>();
-            for (const name of canonicalNames) {
-                const original = await this.getCollection(name) as BCollection;
-                originals.set(name, original);
-                clones.set(name, await original.cloneForTransaction());
-            }
-            const tx: IBTransaction = {
-                getCollection: async <T>(collectionName: string): Promise<IBTransactionCollection<T>> => {
-                    const clone = clones.get(canonicalName(collectionName));
-                    if (!clone) {
-                        throw new Error(`collection "${collectionName}" was not declared to this transaction`);
-                    }
-                    return clone as IBTransactionCollection<T>;
-                },
-            };
-            await fn(tx);
-            for (const name of canonicalNames) {
-                await originals.get(name)!.commitTransactionClone(clones.get(name)!);
-            }
-        });
-    }
-
-}
-
-export class BCollection<T = any> implements IBCollection<T> {
-    readonly database: BDatabase;
-    readonly collectionConfig: IBCollectionConfig;
-    readonly collectionName: string;
-    readonly collectionID: string;
-
-    private mapData: Map<string, T> = new Map();
-    private arrayData: Array<T> = [];
-    private loaded: boolean = false;
-    private dirty: boolean = false;
-    private dropped: boolean = false;
-
-    private addHandlers: Array<{ handler: TAddHandler<T>; mode: EBHandlerMode }> = [];
-    private updateHandlers: Array<{ handler: TUpdateHandler<T>; mode: EBHandlerMode }> = [];
-    private deleteHandlers: Array<{ handler: TDeleteHandler<T>; mode: EBHandlerMode }> = [];
-
-    private indexDefinitions: Map<string, TIndexKeyFunction<T>> = new Map();
-    private indexRuntime: Map<string, Map<string, T[]>> = new Map();
-
-    public constructor(database: BDatabase, collectionID: string, collectionName: string, collectionConfig: IBCollectionConfig) {
-        this.database = database;
-        this.collectionID = collectionID;
-        this.collectionName = canonicalName(collectionName);
-        this.collectionConfig = collectionConfig;
-        switch (collectionConfig.memStorageStrategy) {
-            case EBMemStorageStrategy.MAP:
-                // if (!collectionConfig.getID || !collectionConfig.setID) {
-                //     throw new Error("need id accessors configured when using map mem storage");
-                // }
-                break;
-            default:
-            case EBMemStorageStrategy.ARRAY:
-                break;
-        }
-    }
-
-    private async checkLoaded(): Promise<void> {
-        if (!this.loaded) {
-            await this.reload();
-        }
-    }
-
-    private async runExclusive<R>(fn: () => Promise<R>): Promise<R> {
-        return runReentrantProtected(this.collectionName, fn);
-    }
-
-    private async runProtected<R>(fn: () => Promise<R>): Promise<R> {
-        return this.runExclusive(async () => {
-            if (this.dropped) {
-                throw new Error(`collection "${this.collectionName}" has been dropped`);
-            }
-            return fn();
-        });
-    }
-
-    /**
-     * call this only within the protecting mutex
-     */
-    public markDropped(): void {
-        this.dropped = true;
-    }
-
-    public async get(predicate?: (v: T) => boolean): Promise<T[]> {
-        return this.runProtected(async () => {
-            await this.checkLoaded();
-            switch (this.collectionConfig.memStorageStrategy) {
-                case EBMemStorageStrategy.MAP:
-                    return !predicate ?
-                        this.mapData.values().toArray() :
-                        this.mapData.values().filter(predicate).toArray();
-                default:
-                case EBMemStorageStrategy.ARRAY:
-                    return !predicate ?
-                        this.arrayData.slice() :
-                        this.arrayData.filter(predicate);
-            }
-        });
-    }
-
-    public async count(predicate?: (v: T) => boolean): Promise<number> {
-        return this.runProtected(async () => {
-            await this.checkLoaded();
-            switch (this.collectionConfig.memStorageStrategy) {
-                case EBMemStorageStrategy.MAP:
-                    return !predicate ?
-                        this.mapData.size :
-                        this.mapData.values().filter(predicate).reduce((n) => n + 1, 0);
-                default:
-                case EBMemStorageStrategy.ARRAY:
-                    return !predicate ?
-                        this.arrayData.length :
-                        this.arrayData.filter(predicate).length;
-            }
-        });
-    }
-
-    public async getByIndex(indexKey: string, key: string): Promise<T[]> {
-        return this.runProtected(async () => {
-            await this.checkLoaded();
-            const byKey = this.indexRuntime.get(indexKey);
-            if (!byKey) {
-                throw new Error(`index "${indexKey}" is not configured`);
-            }
-            return byKey.get(key)?.slice() ?? [];
-        });
-    }
-
-    private async invokeHandler(fn: () => void | Promise<void>, modificationType: EBModificationType): Promise<void> {
-        try {
-            await fn();
-        }
-        catch (error) {
-            logger.error(`bajadab: collection "${this.collectionName}" ${modificationType} handler failed`, error);
-        }
-    }
-
-    private async fireAdd(added: T[]): Promise<void> {
-        for (const v of added) {
-            for (const { handler, mode } of this.addHandlers) {
-                const call = this.invokeHandler(() => handler(v), EBModificationType.ADD);
-                if (mode === EBHandlerMode.SYNC) {
-                    await call;
-                }
-            }
-        }
-    }
-
-    private async fireUpdate(pairs: Array<[T, T]>): Promise<void> {
-        for (const [previous, updated] of pairs) {
-            for (const { handler, mode } of this.updateHandlers) {
-                const call = this.invokeHandler(() => handler(previous, updated), EBModificationType.UPDATE);
-                if (mode === EBHandlerMode.SYNC) {
-                    await call;
-                }
-            }
-        }
-    }
-
-    private async fireDelete(deleted: T[]): Promise<void> {
-        for (const v of deleted) {
-            for (const { handler, mode } of this.deleteHandlers) {
-                const call = this.invokeHandler(() => handler(v), EBModificationType.DELETE);
-                if (mode === EBHandlerMode.SYNC) {
-                    await call;
-                }
-            }
-        }
-    }
-
-    public async on(modificationType: EBModificationType.ADD, handler: TAddHandler<T>, mode?: EBHandlerMode): Promise<IHandlerRegistration>;
-    public async on(modificationType: EBModificationType.UPDATE, handler: TUpdateHandler<T>, mode?: EBHandlerMode): Promise<IHandlerRegistration>;
-    public async on(modificationType: EBModificationType.DELETE, handler: TDeleteHandler<T>, mode?: EBHandlerMode): Promise<IHandlerRegistration>;
-    public async on(modificationType: EBModificationType, handler: TAddHandler<T> | TUpdateHandler<T> | TDeleteHandler<T>, mode: EBHandlerMode = EBHandlerMode.SYNC): Promise<IHandlerRegistration> {
-        return this.runProtected(async () => {
-            switch (modificationType) {
-                case EBModificationType.ADD: {
-                    const h = handler as TAddHandler<T>;
-                    this.addHandlers.push({ handler: h, mode });
-                    return { cancel: async () => this.runExclusive(async () => { this.addHandlers = this.addHandlers.filter(x => x.handler !== h); }) };
-                }
-                case EBModificationType.UPDATE: {
-                    const h = handler as TUpdateHandler<T>;
-                    this.updateHandlers.push({ handler: h, mode });
-                    return { cancel: async () => this.runExclusive(async () => { this.updateHandlers = this.updateHandlers.filter(x => x.handler !== h); }) };
-                }
-                case EBModificationType.DELETE: {
-                    const h = handler as TDeleteHandler<T>;
-                    this.deleteHandlers.push({ handler: h, mode });
-                    return { cancel: async () => this.runExclusive(async () => { this.deleteHandlers = this.deleteHandlers.filter(x => x.handler !== h); }) };
-                }
-                default:
-                    throw new Error(`unknown modification type ${modificationType}`);
-            }
-        });
-    }
-
-    public async delete(predicate: (v: T) => boolean): Promise<number> {
-        return this.runProtected(async () => {
-            await this.checkLoaded();
-            let ret = 0;
-            const trackDeletes = this.deleteHandlers.length > 0;
-            const indexDeletes = this.indexDefinitions.size > 0;
-            const deleted: T[] = [];
-            switch (this.collectionConfig.memStorageStrategy) {
-                case EBMemStorageStrategy.MAP:
-                    for (const [key, value] of this.mapData) {
-                        if (predicate(value)) {
-                            ret++;
-                            if (indexDeletes) {
-                                await this.removeFromIndexes(value);
-                            }
-                            this.mapData.delete(key);
-                            if (trackDeletes) {
-                                deleted.push(value);
-                            }
-                        }
-                    }
-                    break;
-                default:
-                case EBMemStorageStrategy.ARRAY:
-                    switch (this.collectionConfig.arrayDeletionStrategy) {
-                        case EBArrayDeletionStrategy.IN_PLACE:
-                            let writeIndex = 0;
-                            for (let readIndex = 0; readIndex < this.arrayData.length; readIndex++) {
-                                const item = this.arrayData[readIndex];
-                                if (!predicate(item!)) {
-                                    this.arrayData[writeIndex] = item!;
-                                    writeIndex++;
-                                }
-                                else {
-                                    ret++;
-                                    if (indexDeletes) {
-                                        await this.removeFromIndexes(item!);
-                                    }
-                                    if (trackDeletes) {
-                                        deleted.push(item!);
-                                    }
-                                }
-                            }
-                            this.arrayData.length = writeIndex;
-                            break;
-                        default:
-                        case EBArrayDeletionStrategy.NEW_ARRAY:
-                            const ol = this.arrayData.length;
-                            if (trackDeletes || indexDeletes) {
-                                const removed = this.arrayData.filter(predicate);
-                                if (trackDeletes) {
-                                    deleted.push(...removed);
-                                }
-                                if (indexDeletes) {
-                                    for (const v of removed) {
-                                        await this.removeFromIndexes(v);
-                                    }
-                                }
-                            }
-                            this.arrayData = this.arrayData.filter(v => !predicate(v));
-                            ret += (ol - this.arrayData.length);
-                            break;
-                    }
-                    break;
-            }
-            if (ret > 0) {
-                await this.doOnDirty();
-            }
-            if (deleted.length > 0) {
-                await this.fireDelete(deleted);
-            }
-            return ret;
-        });
-    }
-
-    public async has(predicate: (v: T) => boolean): Promise<boolean> {
-        return this.runProtected(async () => {
-            await this.checkLoaded();
-            switch (this.collectionConfig.memStorageStrategy) {
-                case EBMemStorageStrategy.MAP:
-                    return this.mapData.values().find(predicate) !== undefined;
-                default:
-                case EBMemStorageStrategy.ARRAY:
-                    return this.arrayData.find(predicate) !== undefined;
-            }
-        });
-    }
-
-    private resolveIndexDefinitions(): void {
-        this.indexDefinitions = new Map();
-        if (!this.collectionConfig.indexes) {
-            return;
-        }
-        for (const [indexKey, registryKey] of Object.entries(this.collectionConfig.indexes)) {
-            const extractor = this.database.functionRegistry?.fetch(registryKey) as TIndexKeyFunction<T> | undefined;
-            if (!extractor) {
-                throw new Error(`index key function "${registryKey}" is not registered`);
-            }
-            this.indexDefinitions.set(indexKey, extractor);
-        }
-    }
-
-    private async indexKeysFor(v: T): Promise<Array<[string, string[]]>> {
-        const result: Array<[string, string[]]> = [];
-        for (const [indexKey, extractor] of this.indexDefinitions) {
-            const extracted = await extractor(v);
-            if (extracted === undefined) {
-                continue;
-            }
-            result.push([indexKey, Array.isArray(extracted) ? extracted : [extracted]]);
-        }
-        return result;
-    }
-
-    private async addToIndexes(v: T): Promise<void> {
-        for (const [indexKey, keys] of await this.indexKeysFor(v)) {
-            const byKey = this.indexRuntime.get(indexKey)!;
-            for (const key of keys) {
-                let bucket = byKey.get(key);
-                if (!bucket) {
-                    bucket = [];
-                    byKey.set(key, bucket);
-                }
-                bucket.push(v);
-            }
-        }
-    }
-
-    private async removeFromIndexes(v: T): Promise<void> {
-        for (const [indexKey, keys] of await this.indexKeysFor(v)) {
-            const byKey = this.indexRuntime.get(indexKey)!;
-            for (const key of keys) {
-                const bucket = byKey.get(key);
-                if (!bucket) {
-                    continue;
-                }
-                const at = bucket.indexOf(v);
-                if (at !== -1) {
-                    bucket.splice(at, 1);
-                }
-                if (bucket.length === 0) {
-                    byKey.delete(key);
-                }
-            }
-        }
-    }
-
-    /**
-     * full rebuild from currently loaded data; only needed right after (re)load, since every
-     * value then is a freshly deserialized object the incremental add/update/delete hooks never saw
-     */
-    private async rebuildIndexRuntime(): Promise<void> {
-        // every configured index gets an entry up front, even if empty, so getByIndex()/
-        // addToIndexes()/removeFromIndexes() can assume it's there instead of checking twice
-        this.indexRuntime = new Map(Array.from(this.indexDefinitions.keys(), (indexKey) => [indexKey, new Map()]));
-        if (this.indexDefinitions.size === 0) {
-            return;
-        }
-        const values = this.collectionConfig.memStorageStrategy === EBMemStorageStrategy.MAP ?
-            [...this.mapData.values()] : this.arrayData;
-        for (const v of values) {
-            await this.addToIndexes(v);
-        }
-    }
-
-    private resolveIDGetter(): TGetIDFunction {
-        if (this.collectionConfig.getID) {
-            const found = this.database.functionRegistry?.fetch(this.collectionConfig.getID) as TGetIDFunction | undefined;
-            if (!found) {
-                throw new Error(`getID function "${this.collectionConfig.getID}" is not registered`);
-            }
-            return found;
-        }
-        return DEFAULT_GET_ID;
-    }
-
-    /**
-     * id -> value for every stored item, resolved via the plain id getter (no auto-id assignment - every
-     * stored item already has one). Used to diff a transaction's working copy against the pre-transaction
-     * state in {@link commitTransactionClone}.
-     */
-    private async idKeyedSnapshot(): Promise<Map<string, T>> {
-        switch (this.collectionConfig.memStorageStrategy) {
-            case EBMemStorageStrategy.MAP:
-                return new Map(this.mapData);
-            default:
-            case EBMemStorageStrategy.ARRAY:
-                const getter = this.resolveIDGetter();
-                const ret = new Map<string, T>();
-                for (const v of this.arrayData) {
-                    const id = await getter(v);
-                    if (id) {
-                        ret.set(id, v);
-                    }
-                }
-                return ret;
-        }
-    }
-
-    private async getID(v: T): Promise<string> {
-        let id = await this.resolveIDGetter()(v);
-        if (!id) {
-            switch (this.collectionConfig.autoIDStrategy) {
-                case EBAutoIDStrategy.NO_AUTO_ID:
-                    throw new Error("cannot omit auto-ids and than trying to add a value without an id");
-                default:
-                case EBAutoIDStrategy.AUTO_ID_UUID:
-                    id = uuidv7();
-                    let setter: TSetIDFunction;
-                    if (this.collectionConfig.setID) {
-                        const found = this.database.functionRegistry?.fetch(this.collectionConfig.setID) as TSetIDFunction | undefined;
-                        if (!found) {
-                            throw new Error(`setID function "${this.collectionConfig.setID}" is not registered`);
-                        }
-                        setter = found;
-                    }
-                    else {
-                        setter = DEFAULT_SET_ID;
-                    }
-                    await setter(v, id);
-                    break;
-            }
-        }
-        return id;
-    }
-
-    public async add(...vs: T[]): Promise<number> {
-        return this.runProtected(async () => {
-            await this.checkLoaded();
-            switch (this.collectionConfig.memStorageStrategy) {
-                case EBMemStorageStrategy.MAP:
-                    let ret = 0;
-                    const trackAddsMap = this.addHandlers.length > 0;
-                    const indexAddsMap = this.indexDefinitions.size > 0;
-                    const addedMap: T[] = [];
-                    for (const v of vs) {
-                        const id = await this.getID(v);
-                        if (this.mapData.has(id)) {
-                            switch (this.collectionConfig.duplicateIDStrategy) {
-                                case EBDuplicateIDStrategy.ERROR:
-                                    throw new Error(`duplicate id ${id}`);
-                                case EBDuplicateIDStrategy.IGNORE:
-                                    // ignore the new value
-                                    break;
-                                default:
-                                case EBDuplicateIDStrategy.OVERWRITE:
-                                    // replace the new value
-                                    ret++;
-                                    if (indexAddsMap) {
-                                        await this.removeFromIndexes(this.mapData.get(id)!);
-                                    }
-                                    this.mapData.set(id, v);
-                                    if (indexAddsMap) {
-                                        await this.addToIndexes(v);
-                                    }
-                                    if (trackAddsMap) {
-                                        addedMap.push(v);
-                                    }
-                                    break;
-                            }
-                        }
-                        else {
-                            ret++;
-                            this.mapData.set(id, v);
-                            if (indexAddsMap) {
-                                await this.addToIndexes(v);
-                            }
-                            if (trackAddsMap) {
-                                addedMap.push(v);
-                            }
-                        }
-                    }
-                    if (ret > 0) {
-                        await this.doOnDirty();
-                    }
-                    if (addedMap.length > 0) {
-                        await this.fireAdd(addedMap);
-                    }
-                    return ret;
-                default:
-                case EBMemStorageStrategy.ARRAY:
-                    const trackAddsArray = this.addHandlers.length > 0;
-                    const indexAddsArray = this.indexDefinitions.size > 0;
-                    const addedArray: T[] = [];
-                    switch (this.collectionConfig.duplicateIDStrategy) {
-                        case EBDuplicateIDStrategy.ERROR:
-                            const haveIDs = new Set<string>();
-                            for (const v of this.arrayData) {
-                                haveIDs.add(await this.getID(v));
-                            }
-                            for (const v of vs) {
-                                const id = await this.getID(v);
-                                if (haveIDs.has(id)) {
-                                    throw new Error(`duplicate id ${id}`);
-                                }
-                                haveIDs.add(id);
-                                this.arrayData.push(v);
-                                if (indexAddsArray) {
-                                    await this.addToIndexes(v);
-                                }
-                                if (trackAddsArray) {
-                                    addedArray.push(v);
-                                }
-                            }
-                            break;
-                        case EBDuplicateIDStrategy.IGNORE:
-                            const seenIDs = new Set<string>();
-                            for (const v of this.arrayData) {
-                                seenIDs.add(await this.getID(v));
-                            }
-                            for (const v of vs) {
-                                const id = await this.getID(v);
-                                if (!seenIDs.has(id)) {
-                                    seenIDs.add(id);
-                                    this.arrayData.push(v);
-                                    if (indexAddsArray) {
-                                        await this.addToIndexes(v);
-                                    }
-                                    if (trackAddsArray) {
-                                        addedArray.push(v);
-                                    }
-                                }
-                            }
-                            break;
-                        default:
-                        case EBDuplicateIDStrategy.OVERWRITE:
-                            const indexMap = new Map<string | number, number>();
-                            for (let i = 0; i < this.arrayData.length; i++) {
-                                indexMap.set(await this.getID(this.arrayData[i]!), i);
-                            }
-                            for (const v of vs) {
-                                const id = await this.getID(v);
-                                const existingIndex = indexMap.get(id);
-
-                                if (existingIndex !== undefined) {
-                                    if (indexAddsArray) {
-                                        await this.removeFromIndexes(this.arrayData[existingIndex]!);
-                                    }
-                                    this.arrayData[existingIndex] = v;
-                                }
-                                else {
-                                    // store new index in indexCache in the case this id appears again
-                                    indexMap.set(id, this.arrayData.length);
-                                    this.arrayData.push(v);
-                                }
-                                if (indexAddsArray) {
-                                    await this.addToIndexes(v);
-                                }
-                                if (trackAddsArray) {
-                                    addedArray.push(v);
-                                }
-                            }
-                            break;
-                    }
-                    if (vs.length > 0) {
-                        await this.doOnDirty();
-                    }
-                    if (addedArray.length > 0) {
-                        await this.fireAdd(addedArray);
-                    }
-                    return vs.length;
-            }
-        });
-    }
-
-    public async update(predicate: (v: T) => boolean, updater: (v: T) => Promise<T>): Promise<number> {
-        return this.runProtected(async () => {
-            await this.checkLoaded();
-            let ret = 0;
-            const trackUpdates = this.updateHandlers.length > 0;
-            const indexUpdates = this.indexDefinitions.size > 0;
-            const updated: Array<[T, T]> = [];
-            switch (this.collectionConfig.memStorageStrategy) {
-                case EBMemStorageStrategy.MAP:
-                    for (const [key, value] of this.mapData) {
-                        if (predicate(value)) {
-                            const replacement = await updater(value);
-                            if (replacement !== undefined) {
-                                ret++;
-                                if (indexUpdates) {
-                                    await this.removeFromIndexes(value);
-                                }
-                                this.mapData.set(key, replacement);
-                                if (indexUpdates) {
-                                    await this.addToIndexes(replacement);
-                                }
-                                if (trackUpdates) {
-                                    updated.push([value, replacement]);
-                                }
-                            }
-                            else {
-                                throw new Error("updater gave undefined");
-                            }
-                        }
-                    }
-                    break;
-                default:
-                case EBMemStorageStrategy.ARRAY:
-                    for (let i = 0; i < this.arrayData.length; i++) {
-                        if (predicate(this.arrayData[i]!)) {
-                            const previous = this.arrayData[i]!;
-                            const replacement = await updater(previous);
-                            if (replacement !== undefined) {
-                                ret++;
-                                if (indexUpdates) {
-                                    await this.removeFromIndexes(previous);
-                                }
-                                this.arrayData[i] = replacement;
-                                if (indexUpdates) {
-                                    await this.addToIndexes(replacement);
-                                }
-                                if (trackUpdates) {
-                                    updated.push([previous, replacement]);
-                                }
-                            }
-                            else {
-                                throw new Error("updater gave undefined");
-                            }
-                        }
-                    }
-                    break;
-            }
-            if (ret > 0) {
-                await this.doOnDirty();
-            }
-            if (updated.length > 0) {
-                await this.fireUpdate(updated);
-            }
-            return ret;
-        });
-    }
-
-    private async doOnDirty(): Promise<void> {
-        this.dirty = true;
-        if (this.collectionConfig.autoFlushStrategy == EBAutoFlushStrategy.ALWAYS_AUTO_FLUSH) {
-            await this.flush();
-        }
-    }
-
-    public async flush(): Promise<void> {
-        return this.runProtected(async () => {
-            if (this.loaded) {
-                switch (this.collectionConfig.memStorageStrategy) {
-                    case EBMemStorageStrategy.MAP:
-                        await this.database.io.writeMapData(this.collectionID, this.mapData);
-                        break;
-                    default:
-                    case EBMemStorageStrategy.ARRAY:
-                        await this.database.io.writeArrayData(this.collectionID, this.arrayData);
-                        break;
-                }
-                this.dirty = false;
-            }
-        });
-    }
-
-    public async reload(): Promise<void> {
-        return this.runProtected(async () => {
-            switch (this.collectionConfig.memStorageStrategy) {
-                case EBMemStorageStrategy.MAP:
-                    this.mapData = await this.database.io.loadMapData<T>(this.collectionID);
-                    break;
-                default:
-                case EBMemStorageStrategy.ARRAY:
-                    this.arrayData = await this.database.io.loadArrayData<T>(this.collectionID);
-                    break;
-            }
-            this.loaded = true;
-            this.dirty = false;
-            this.resolveIndexDefinitions();
-            await this.rebuildIndexRuntime();
-        });
-    }
-
-    public async unload(): Promise<void> {
-        return this.runProtected(async () => {
-            if (this.dirty) {
-                switch (this.collectionConfig.dirtyUnloadStrategy) {
-                    case EBDirtyUnloadStrategy.FLUSH:
-                        await this.flush();
-                        break;
-                    case EBDirtyUnloadStrategy.ERROR:
-                        throw new Error("cannot unload unwritten collection");
-                    default:
-                    case EBDirtyUnloadStrategy.IGNORE:
-                        // do nothing
-                        break;
-
-                }
-            }
-            switch (this.collectionConfig.memStorageStrategy) {
-                case EBMemStorageStrategy.MAP:
-                    this.mapData = new Map();
-                    break;
-                default:
-                case EBMemStorageStrategy.ARRAY:
-                    this.arrayData = [];
-                    break;
-            }
-            this.loaded = false;
-            this.dirty = false;
-        });
-    }
-
-    /**
-     * Creates a transaction-scoped working copy: same identity/config as this collection, but with
-     * independently mutable in-memory state (auto-flush forced off, regardless of this collection's own
-     * `autoFlushStrategy`) and no change-event handlers of its own. Call only while already holding this
-     * collection's mutex for the whole transaction (see {@link BDatabase.transaction}).
-     */
-    public async cloneForTransaction(): Promise<BCollection<T>> {
-        return this.runProtected(async () => {
-            await this.checkLoaded();
-            const txConfig: IBCollectionConfig = { ...this.collectionConfig, autoFlushStrategy: EBAutoFlushStrategy.NO_AUTO_FLUSH };
-            const clone = new BCollection<T>(this.database, this.collectionID, this.collectionName, txConfig);
-            clone.mapData = new Map(this.mapData);
-            clone.arrayData = this.arrayData.slice();
-            clone.loaded = true;
-            clone.resolveIndexDefinitions();
-            await clone.rebuildIndexRuntime();
-            return clone;
-        });
-    }
-
-    /**
-     * Applies a working copy from {@link cloneForTransaction} back onto this collection: replaces the live
-     * state, flushes once, then fires `ADD`/`UPDATE`/`DELETE` handlers for the net difference between this
-     * collection's state before the transaction and `clone`'s state - not for each intermediate operation the
-     * transaction performed. The before/after diff itself (and the id resolution it needs) only runs when at
-     * least one handler is registered on this collection - a transaction on a collection nobody listens to
-     * skips it entirely. Call only while already holding this collection's mutex for the whole transaction.
-     */
-    public async commitTransactionClone(clone: BCollection<T>): Promise<void> {
-        return this.runProtected(async () => {
-            await this.checkLoaded();
-            const added: T[] = [];
-            const updated: Array<[T, T]> = [];
-            const deleted: T[] = [];
-            // the before/after diff only feeds fireAdd()/fireUpdate()/fireDelete() below - skip it
-            // entirely (incl. idKeyedSnapshot()'s id-getter calls) when nothing is listening
-            const trackChanges = this.addHandlers.length > 0 || this.updateHandlers.length > 0 || this.deleteHandlers.length > 0;
-            if (trackChanges) {
-                const before = await this.idKeyedSnapshot();
-                const after = await clone.idKeyedSnapshot();
-                for (const [id, value] of after) {
-                    if (!before.has(id)) {
-                        added.push(value);
-                    }
-                    else {
-                        const previous = before.get(id) as T;
-                        if (JSON.stringify(previous) !== JSON.stringify(value)) {
-                            updated.push([previous, value]);
-                        }
-                    }
-                }
-                for (const [id, value] of before) {
-                    if (!after.has(id)) {
-                        deleted.push(value);
-                    }
-                }
-            }
-            this.mapData = clone.mapData;
-            this.arrayData = clone.arrayData;
-            this.indexRuntime = clone.indexRuntime;
-            this.dirty = true;
-            await this.flush();
-            if (added.length > 0) {
-                await this.fireAdd(added);
-            }
-            if (updated.length > 0) {
-                await this.fireUpdate(updated);
-            }
-            if (deleted.length > 0) {
-                await this.fireDelete(deleted);
-            }
-        });
-    }
-
+import { AsyncLocalStorage } from "node:async_hooks";
+import { uuidv7, } from "uuidv7";
+import { runReentrantProtected } from "@incoqnito.io/ts-iq-core";
+
+import type { IODatabase, IPreparedWrite } from "./fileio.js";
+
+import {
+    EBArrayDeletionStrategy,
+    EBAutoFlushStrategy,
+    EBAutoIDStrategy,
+    EBDirtyUnloadStrategy,
+    EBDuplicateIDStrategy,
+    EBHandlerMode,
+    EBMemStorageStrategy,
+    EBModificationType,
+    IBCollection,
+    IBCollectionConfig,
+    IBDatabase,
+    IBDatabaseConfig,
+    IBFunctionRegistry,
+    IBSnapshotOptions,
+    IBTransaction,
+    IBTransactionCollection,
+    IHandlerRegistration,
+    TAddHandler,
+    TDeleteHandler,
+    TGetIDFunction,
+    TIndexKeyFunction,
+    TSetIDFunction,
+    TUpdateHandler,
+} from "./base.js";
+import { canonicalName } from "./utils.js";
+
+/** Default {@link TGetIDFunction}: reads `v.id`, `undefined` if absent. */
+export const DEFAULT_GET_ID: TGetIDFunction = (v: any) => (v !== null && typeof v === 'object' && "id" in v) ? v.id : undefined;
+/** Default {@link TSetIDFunction}: assigns to `v.id`. */
+export const DEFAULT_SET_ID: TSetIDFunction = async (v: any, id: string) => { v.id = id; };
+
+function normalizeID(id: unknown): string | undefined {
+    return id === undefined || id === null || id === "" ? undefined : String(id);
+}
+
+export interface IPreparedTransactionCommit {
+    readonly write: IPreparedWrite;
+    apply(): void;
+    fire(): Promise<void>;
+}
+
+/** {@link IBCollectionConfig} used when a collection is created without an explicit one. */
+export const DEFAULT_COLLECTION_CONFIG: IBCollectionConfig = {
+    arrayDeletionStrategy: EBArrayDeletionStrategy.IN_PLACE,
+    autoIDStrategy: EBAutoIDStrategy.AUTO_ID_UUID,
+    duplicateIDStrategy: EBDuplicateIDStrategy.OVERWRITE,
+    memStorageStrategy: EBMemStorageStrategy.MAP,
+    dirtyUnloadStrategy: EBDirtyUnloadStrategy.FLUSH,
+    autoFlushStrategy: EBAutoFlushStrategy.ALWAYS_AUTO_FLUSH,
+};
+
+/** {@link IBDatabaseConfig} used when `getDatabase()` creates a fresh database. */
+export const DEFAULT_DATABASE_CONFIG: IBDatabaseConfig = {
+    defaultCollectionConfig: DEFAULT_COLLECTION_CONFIG,
+};
+
+export class BDatabase implements IBDatabase {
+    public readonly config: IBDatabaseConfig;
+    private collections: Map<string, BCollection> = new Map();
+    readonly io: IODatabase;
+    readonly functionRegistry?: IBFunctionRegistry;
+    readonly iid = uuidv7();
+    private readonly transactionScope = new AsyncLocalStorage<ReadonlySet<string>>();
+
+    public constructor(io: IODatabase, config?: IBDatabaseConfig, functionRegistry?: IBFunctionRegistry) {
+        this.config = { ...(config ?? DEFAULT_DATABASE_CONFIG) };
+        this.io = io;
+        this.functionRegistry = functionRegistry;
+    }
+
+    public async runExclusive<R>(fn: () => Promise<R>): Promise<R> {
+        return runReentrantProtected(this.iid, fn);
+    }
+
+    public isInTransaction(collectionName: string): boolean {
+        return this.transactionScope.getStore()?.has(collectionName) ?? false;
+    }
+
+    public addCollection(collection: BCollection): void {
+        this.collections.set(collection.collectionName, collection);
+    }
+
+    public async getCollection<T>(collectionName: string, collectionConfig?: IBCollectionConfig): Promise<IBCollection<T>> {
+        return this.runExclusive(async () => {
+            const cname = canonicalName(collectionName);
+            let ret: BCollection<T> | undefined = this.collections.get(cname);
+            if (!ret) {
+                ret = await this.createCollection(collectionName, collectionConfig);
+            }
+            return ret;
+        });
+    }
+
+    public async createCollection<T = any>(collectionName: string, collectionConfig?: IBCollectionConfig): Promise<BCollection<T>> {
+        return this.runExclusive(async () => {
+            const cname = canonicalName(collectionName);
+            if (this.collections.has(cname)) {
+                throw new Error(`cannot create already existing collection ${collectionName}`);
+            }
+            const id = uuidv7();
+            const ret: BCollection<T> = new BCollection<T>(this, id, collectionName, { ...(collectionConfig ?? this.config.defaultCollectionConfig) });
+            await this.io.createCollection<T>(ret);
+            this.addCollection(ret);
+            return ret;
+        });
+    }
+
+    public async hasCollection(collectionName: string): Promise<boolean> {
+        return this.runExclusive(async () => {
+            return this.collections.has(canonicalName(collectionName));
+        });
+    }
+
+    public async dropCollection(collectionName: string): Promise<void> {
+        const cname = canonicalName(collectionName);
+        return this.runExclusive(async () => {
+            const coll: BCollection = this.collections.get(cname) as BCollection;
+            if (coll) {
+                coll.markDropped();
+                this.collections.delete(cname);
+                return this.io.deleteCollection(cname);
+            }
+            else {
+                throw new Error(`no such collection ${collectionName}`);
+            }
+        });
+    }
+
+    public async snapshot(targetDir: string, options?: IBSnapshotOptions): Promise<string> {
+        return this.runExclusive(async () => {
+            for (const collection of this.collections.values()) {
+                await collection.flush();
+            }
+            return this.io.snapshot(targetDir, options);
+        });
+    }
+
+    public async transaction(collectionNames: string[], fn: (tx: IBTransaction) => Promise<void>): Promise<void> {
+        if (this.transactionScope.getStore() !== undefined) {
+            throw new Error("nested transactions are not supported");
+        }
+        const canonicalNames = Array.from(new Set(collectionNames.map(canonicalName)));
+        return this.runExclusive(async () => {
+            const originals = new Map<string, BCollection>();
+            const clones = new Map<string, BCollection>();
+            for (const name of canonicalNames) {
+                const original = this.collections.get(name);
+                if (!original) {
+                    throw new Error(`no such collection ${name}`);
+                }
+                originals.set(name, original);
+            }
+            for (const [name, original] of originals) {
+                clones.set(name, await original.cloneForTransaction());
+            }
+            const tx: IBTransaction = {
+                getCollection: async <T>(collectionName: string): Promise<IBTransactionCollection<T>> => {
+                    const clone = clones.get(canonicalName(collectionName));
+                    if (!clone) {
+                        throw new Error(`collection "${collectionName}" was not declared to this transaction`);
+                    }
+                    return clone as IBTransactionCollection<T>;
+                },
+            };
+            try {
+                await this.transactionScope.run(new Set(canonicalNames), () => fn(tx));
+                const prepared: IPreparedTransactionCommit[] = [];
+                try {
+                    for (const name of canonicalNames) {
+                        prepared.push(await originals.get(name)!.prepareTransactionCommit(clones.get(name)!));
+                    }
+                }
+                catch (err) {
+                    await Promise.all(prepared.map(p => p.write.discard()));
+                    throw err;
+                }
+                let renamed = 0;
+                try {
+                    for (const p of prepared) {
+                        await p.write.commit();
+                        renamed++;
+                    }
+                }
+                finally {
+                    await Promise.all(prepared.slice(renamed).map(p => p.write.discard()));
+                    const applied = prepared.slice(0, renamed);
+                    applied.forEach(p => p.apply());
+                    for (const p of applied) {
+                        await p.fire();
+                    }
+                }
+            }
+            finally {
+                clones.forEach(clone => clone.markDropped());
+            }
+        });
+    }
+
+}
+
+export class BCollection<T = any> implements IBCollection<T> {
+    readonly database: BDatabase;
+    readonly collectionConfig: IBCollectionConfig;
+    readonly collectionName: string;
+    readonly collectionID: string;
+
+    private mapData: Map<string, T> = new Map();
+    private arrayData: Array<T> = [];
+    private loaded: boolean = false;
+    private dirty: boolean = false;
+    private dropped: boolean = false;
+    private transactionClone: boolean = false;
+
+    private addHandlers: Array<{ handler: TAddHandler<T>; mode: EBHandlerMode }> = [];
+    private updateHandlers: Array<{ handler: TUpdateHandler<T>; mode: EBHandlerMode }> = [];
+    private deleteHandlers: Array<{ handler: TDeleteHandler<T>; mode: EBHandlerMode }> = [];
+
+    private indexDefinitions: Map<string, TIndexKeyFunction<T>> = new Map();
+    private indexRuntime: Map<string, Map<string, Set<T>>> = new Map();
+    private indexedKeys: Map<T, Array<[string, string[]]>> = new Map();
+    private txChanges?: Map<string, EBModificationType>;
+
+    public constructor(database: BDatabase, collectionID: string, collectionName: string, collectionConfig: IBCollectionConfig) {
+        this.database = database;
+        this.collectionID = collectionID;
+        this.collectionName = canonicalName(collectionName);
+        this.collectionConfig = collectionConfig;
+    }
+
+    private async checkLoaded(): Promise<void> {
+        if (!this.loaded) {
+            await this.reload();
+        }
+    }
+
+    private async runExclusive<R>(fn: () => Promise<R>): Promise<R> {
+        return this.database.runExclusive(fn);
+    }
+
+    private async runProtected<R>(fn: () => Promise<R>): Promise<R> {
+        return this.runExclusive(async () => {
+            if (this.dropped) {
+                throw new Error(`collection "${this.collectionName}" has been dropped`);
+            }
+            return fn();
+        });
+    }
+
+    private async runMutation<R>(fn: () => Promise<R>): Promise<R> {
+        return this.runProtected(async () => {
+            if (!this.transactionClone && this.database.isInTransaction(this.collectionName)) {
+                throw new Error(`collection "${this.collectionName}" is part of a running transaction; write through tx.getCollection() instead`);
+            }
+            return fn();
+        });
+    }
+
+    /**
+     * call this only within the protecting mutex
+     */
+    public markDropped(): void {
+        this.dropped = true;
+    }
+
+    public async get(predicate?: (v: T) => boolean): Promise<T[]> {
+        return this.runProtected(async () => {
+            await this.checkLoaded();
+            switch (this.collectionConfig.memStorageStrategy) {
+                case EBMemStorageStrategy.MAP:
+                    return !predicate ?
+                        this.mapData.values().toArray() :
+                        this.mapData.values().filter(predicate).toArray();
+                default:
+                case EBMemStorageStrategy.ARRAY:
+                    return !predicate ?
+                        this.arrayData.slice() :
+                        this.arrayData.filter(predicate);
+            }
+        });
+    }
+
+    public async count(predicate?: (v: T) => boolean): Promise<number> {
+        return this.runProtected(async () => {
+            await this.checkLoaded();
+            switch (this.collectionConfig.memStorageStrategy) {
+                case EBMemStorageStrategy.MAP:
+                    return !predicate ?
+                        this.mapData.size :
+                        this.mapData.values().filter(predicate).reduce((n) => n + 1, 0);
+                default:
+                case EBMemStorageStrategy.ARRAY:
+                    return !predicate ?
+                        this.arrayData.length :
+                        this.arrayData.filter(predicate).length;
+            }
+        });
+    }
+
+    public async getByIndex(indexKey: string, key: string): Promise<T[]> {
+        return this.runProtected(async () => {
+            await this.checkLoaded();
+            const byKey = this.indexRuntime.get(indexKey);
+            if (!byKey) {
+                throw new Error(`index "${indexKey}" is not configured`);
+            }
+            return Array.from(byKey.get(key) ?? []);
+        });
+    }
+
+    private async invokeHandler(fn: () => void | Promise<void>, modificationType: EBModificationType): Promise<void> {
+        try {
+            await fn();
+        }
+        catch (error) {
+            // TODO switch logging to ILogging where necessary
+            console.error(`bajadab: collection "${this.collectionName}" ${modificationType} handler failed`, error);
+        }
+    }
+
+    private async fireAdd(added: T[]): Promise<void> {
+        for (const v of added) {
+            for (const { handler, mode } of this.addHandlers) {
+                const call = this.invokeHandler(() => handler(v), EBModificationType.ADD);
+                if (mode === EBHandlerMode.SYNC) {
+                    await call;
+                }
+            }
+        }
+    }
+
+    private async fireUpdate(pairs: Array<[T, T]>): Promise<void> {
+        for (const [previous, updated] of pairs) {
+            for (const { handler, mode } of this.updateHandlers) {
+                const call = this.invokeHandler(() => handler(previous, updated), EBModificationType.UPDATE);
+                if (mode === EBHandlerMode.SYNC) {
+                    await call;
+                }
+            }
+        }
+    }
+
+    private async fireDelete(deleted: T[]): Promise<void> {
+        for (const v of deleted) {
+            for (const { handler, mode } of this.deleteHandlers) {
+                const call = this.invokeHandler(() => handler(v), EBModificationType.DELETE);
+                if (mode === EBHandlerMode.SYNC) {
+                    await call;
+                }
+            }
+        }
+    }
+
+    public async on(modificationType: EBModificationType.ADD, handler: TAddHandler<T>, mode?: EBHandlerMode): Promise<IHandlerRegistration>;
+    public async on(modificationType: EBModificationType.UPDATE, handler: TUpdateHandler<T>, mode?: EBHandlerMode): Promise<IHandlerRegistration>;
+    public async on(modificationType: EBModificationType.DELETE, handler: TDeleteHandler<T>, mode?: EBHandlerMode): Promise<IHandlerRegistration>;
+    public async on(modificationType: EBModificationType, handler: TAddHandler<T> | TUpdateHandler<T> | TDeleteHandler<T>, mode: EBHandlerMode = EBHandlerMode.SYNC): Promise<IHandlerRegistration> {
+        return this.runProtected(async () => {
+            switch (modificationType) {
+                case EBModificationType.ADD: {
+                    const h = handler as TAddHandler<T>;
+                    this.addHandlers.push({ handler: h, mode });
+                    return { cancel: async () => this.runExclusive(async () => { this.addHandlers = this.addHandlers.filter(x => x.handler !== h); }) };
+                }
+                case EBModificationType.UPDATE: {
+                    const h = handler as TUpdateHandler<T>;
+                    this.updateHandlers.push({ handler: h, mode });
+                    return { cancel: async () => this.runExclusive(async () => { this.updateHandlers = this.updateHandlers.filter(x => x.handler !== h); }) };
+                }
+                case EBModificationType.DELETE: {
+                    const h = handler as TDeleteHandler<T>;
+                    this.deleteHandlers.push({ handler: h, mode });
+                    return { cancel: async () => this.runExclusive(async () => { this.deleteHandlers = this.deleteHandlers.filter(x => x.handler !== h); }) };
+                }
+                default:
+                    throw new Error(`unknown modification type ${modificationType}`);
+            }
+        });
+    }
+
+    public async delete(predicate: (v: T) => boolean): Promise<number> {
+        return this.runMutation(async () => {
+            await this.checkLoaded();
+            let ret = 0;
+            const trackDeletes = this.deleteHandlers.length > 0;
+            const collectDeleted = trackDeletes || this.txChanges !== undefined;
+            const indexDeletes = this.indexDefinitions.size > 0;
+            const deleted: T[] = [];
+            switch (this.collectionConfig.memStorageStrategy) {
+                case EBMemStorageStrategy.MAP:
+                    for (const [key, value] of this.mapData) {
+                        if (predicate(value)) {
+                            ret++;
+                            if (indexDeletes) {
+                                this.removeFromIndexes(value);
+                            }
+                            this.mapData.delete(key);
+                            if (collectDeleted) {
+                                deleted.push(value);
+                            }
+                        }
+                    }
+                    break;
+                default:
+                case EBMemStorageStrategy.ARRAY:
+                    switch (this.collectionConfig.arrayDeletionStrategy) {
+                        case EBArrayDeletionStrategy.IN_PLACE:
+                            let writeIndex = 0;
+                            for (let readIndex = 0; readIndex < this.arrayData.length; readIndex++) {
+                                const item = this.arrayData[readIndex];
+                                if (!predicate(item!)) {
+                                    this.arrayData[writeIndex] = item!;
+                                    writeIndex++;
+                                }
+                                else {
+                                    ret++;
+                                    if (indexDeletes) {
+                                        this.removeFromIndexes(item!);
+                                    }
+                                    if (collectDeleted) {
+                                        deleted.push(item!);
+                                    }
+                                }
+                            }
+                            this.arrayData.length = writeIndex;
+                            break;
+                        default:
+                        case EBArrayDeletionStrategy.NEW_ARRAY:
+                            if (collectDeleted || indexDeletes) {
+                                const kept: T[] = [];
+                                for (const v of this.arrayData) {
+                                    if (predicate(v)) {
+                                        ret++;
+                                        if (indexDeletes) {
+                                            this.removeFromIndexes(v);
+                                        }
+                                        if (collectDeleted) {
+                                            deleted.push(v);
+                                        }
+                                    }
+                                    else {
+                                        kept.push(v);
+                                    }
+                                }
+                                this.arrayData = kept;
+                            }
+                            else {
+                                const ol = this.arrayData.length;
+                                this.arrayData = this.arrayData.filter(v => !predicate(v));
+                                ret += (ol - this.arrayData.length);
+                            }
+                            break;
+                    }
+                    break;
+            }
+            if (ret > 0) {
+                await this.doOnDirty();
+            }
+            await this.recordTxChanges(EBModificationType.DELETE, deleted);
+            if (trackDeletes && deleted.length > 0) {
+                await this.fireDelete(deleted);
+            }
+            return ret;
+        });
+    }
+
+    public async has(predicate: (v: T) => boolean): Promise<boolean> {
+        return this.runProtected(async () => {
+            await this.checkLoaded();
+            switch (this.collectionConfig.memStorageStrategy) {
+                case EBMemStorageStrategy.MAP:
+                    return this.mapData.values().some(predicate);
+                default:
+                case EBMemStorageStrategy.ARRAY:
+                    return this.arrayData.some(predicate);
+            }
+        });
+    }
+
+    private resolveIndexDefinitions(): void {
+        this.indexDefinitions = new Map();
+        if (!this.collectionConfig.indexes) {
+            return;
+        }
+        for (const [indexKey, registryKey] of Object.entries(this.collectionConfig.indexes)) {
+            const extractor = this.database.functionRegistry?.fetch(registryKey) as TIndexKeyFunction<T> | undefined;
+            if (!extractor) {
+                throw new Error(`index key function "${registryKey}" is not registered`);
+            }
+            this.indexDefinitions.set(indexKey, extractor);
+        }
+    }
+
+    private async indexKeysFor(v: T): Promise<Array<[string, string[]]>> {
+        const result: Array<[string, string[]]> = [];
+        for (const [indexKey, extractor] of this.indexDefinitions) {
+            const extracted = await extractor(v);
+            if (extracted === undefined) {
+                continue;
+            }
+            result.push([indexKey, Array.isArray(extracted) ? extracted : [extracted]]);
+        }
+        return result;
+    }
+
+    private async addToIndexes(v: T): Promise<void> {
+        this.addKeysToIndexes(v, await this.indexKeysFor(v));
+    }
+
+    private async indexKeysForAll(vs: T[]): Promise<Array<Array<[string, string[]]>>> {
+        const ret: Array<Array<[string, string[]]>> = [];
+        for (const v of vs) {
+            ret.push(await this.indexKeysFor(v));
+        }
+        return ret;
+    }
+
+    private addKeysToIndexes(v: T, indexKeys: Array<[string, string[]]>): void {
+        for (const [indexKey, keys] of indexKeys) {
+            const byKey = this.indexRuntime.get(indexKey)!;
+            for (const key of keys) {
+                let bucket = byKey.get(key);
+                if (!bucket) {
+                    bucket = new Set();
+                    byKey.set(key, bucket);
+                }
+                bucket.add(v);
+            }
+        }
+        if (indexKeys.length > 0) {
+            this.indexedKeys.set(v, indexKeys);
+        }
+    }
+
+    private removeFromIndexes(v: T): void {
+        const indexKeys = this.indexedKeys.get(v);
+        if (!indexKeys) {
+            return;
+        }
+        this.indexedKeys.delete(v);
+        for (const [indexKey, keys] of indexKeys) {
+            const byKey = this.indexRuntime.get(indexKey)!;
+            for (const key of keys) {
+                const bucket = byKey.get(key);
+                if (!bucket) {
+                    continue;
+                }
+                bucket.delete(v);
+                if (bucket.size === 0) {
+                    byKey.delete(key);
+                }
+            }
+        }
+    }
+
+    /**
+     * full rebuild from currently loaded data; only needed right after (re)load, since every
+     * value then is a freshly deserialized object the incremental add/update/delete hooks never saw
+     */
+    private async rebuildIndexRuntime(): Promise<void> {
+        // every configured index gets an entry up front, even if empty, so getByIndex()/
+        // addToIndexes()/removeFromIndexes() can assume it's there instead of checking twice
+        this.indexRuntime = new Map(Array.from(this.indexDefinitions.keys(), (indexKey) => [indexKey, new Map()]));
+        this.indexedKeys = new Map();
+        if (this.indexDefinitions.size === 0) {
+            return;
+        }
+        const values = this.collectionConfig.memStorageStrategy === EBMemStorageStrategy.MAP ?
+            [...this.mapData.values()] : this.arrayData;
+        for (const v of values) {
+            await this.addToIndexes(v);
+        }
+    }
+
+    private resolveIDGetter(): TGetIDFunction {
+        if (this.collectionConfig.getID) {
+            const found = this.database.functionRegistry?.fetch(this.collectionConfig.getID) as TGetIDFunction | undefined;
+            if (!found) {
+                throw new Error(`getID function "${this.collectionConfig.getID}" is not registered`);
+            }
+            return found;
+        }
+        return DEFAULT_GET_ID;
+    }
+
+    /**
+     * id -> value for every stored item, resolved via the plain id getter (no auto-id assignment - every
+     * stored item already has one). Used to diff a transaction's working copy against the pre-transaction
+     * state in {@link prepareTransactionCommit}.
+     */
+    private async idKeyedSnapshot(): Promise<Map<string, T>> {
+        switch (this.collectionConfig.memStorageStrategy) {
+            case EBMemStorageStrategy.MAP:
+                return this.mapData;
+            default:
+            case EBMemStorageStrategy.ARRAY:
+                const getter = this.resolveIDGetter();
+                const ret = new Map<string, T>();
+                for (const v of this.arrayData) {
+                    const id = normalizeID(await getter(v));
+                    if (id) {
+                        ret.set(id, v);
+                    }
+                }
+                return ret;
+        }
+    }
+
+    private resolveIDSetter(): TSetIDFunction {
+        if (this.collectionConfig.setID) {
+            const found = this.database.functionRegistry?.fetch(this.collectionConfig.setID) as TSetIDFunction | undefined;
+            if (!found) {
+                throw new Error(`setID function "${this.collectionConfig.setID}" is not registered`);
+            }
+            return found;
+        }
+        return DEFAULT_SET_ID;
+    }
+
+    private async resolveIncomingIDs(vs: T[]): Promise<Array<string | undefined>> {
+        const getter = this.resolveIDGetter();
+        const ids: Array<string | undefined> = [];
+        for (const v of vs) {
+            const id = normalizeID(await getter(v));
+            if (!id && this.collectionConfig.autoIDStrategy === EBAutoIDStrategy.NO_AUTO_ID) {
+                throw new Error("cannot omit auto-ids and than trying to add a value without an id");
+            }
+            ids.push(id);
+        }
+        return ids;
+    }
+
+    private async assignMissingIDs(vs: T[], ids: Array<string | undefined>): Promise<string[]> {
+        let setter: TSetIDFunction | undefined;
+        const ret: string[] = [];
+        for (let i = 0; i < vs.length; i++) {
+            let id = ids[i];
+            if (!id) {
+                setter ??= this.resolveIDSetter();
+                id = uuidv7();
+                await setter(vs[i], id);
+            }
+            ret.push(id);
+        }
+        return ret;
+    }
+
+    private async storedArrayIDs(): Promise<Array<string | undefined>> {
+        const getter = this.resolveIDGetter();
+        const ids: Array<string | undefined> = [];
+        for (const v of this.arrayData) {
+            ids.push(normalizeID(await getter(v)));
+        }
+        return ids;
+    }
+
+    private async recordTxChanges(modificationType: EBModificationType, values: T[]): Promise<void> {
+        if (!this.txChanges || values.length === 0) {
+            return;
+        }
+        const getter = this.resolveIDGetter();
+        for (const v of values) {
+            const id = normalizeID(await getter(v));
+            if (id) {
+                this.txChanges.set(id, modificationType);
+            }
+        }
+    }
+
+    public async add(...vs: T[]): Promise<number> {
+        return this.runMutation(async () => {
+            await this.checkLoaded();
+            const trackAdds = this.addHandlers.length > 0;
+            const collectAdded = trackAdds || this.txChanges !== undefined;
+            const indexAdds = this.indexDefinitions.size > 0;
+            const incomingIDs = await this.resolveIncomingIDs(vs);
+            const added: T[] = [];
+            let ret = 0;
+            switch (this.collectionConfig.memStorageStrategy) {
+                case EBMemStorageStrategy.MAP: {
+                    if (this.collectionConfig.duplicateIDStrategy === EBDuplicateIDStrategy.ERROR) {
+                        const batchIDs = new Set<string>();
+                        for (const id of incomingIDs) {
+                            if (id === undefined) {
+                                continue;
+                            }
+                            if (this.mapData.has(id) || batchIDs.has(id)) {
+                                throw new Error(`duplicate id ${id}`);
+                            }
+                            batchIDs.add(id);
+                        }
+                    }
+                    const ids = await this.assignMissingIDs(vs, incomingIDs);
+                    const indexKeys = indexAdds ? await this.indexKeysForAll(vs) : [];
+                    for (let i = 0; i < vs.length; i++) {
+                        const id = ids[i]!;
+                        const v = vs[i]!;
+                        if (this.mapData.has(id)) {
+                            switch (this.collectionConfig.duplicateIDStrategy) {
+                                case EBDuplicateIDStrategy.ERROR:
+                                    throw new Error(`duplicate id ${id}`);
+                                case EBDuplicateIDStrategy.IGNORE:
+                                    // ignore the new value
+                                    break;
+                                default:
+                                case EBDuplicateIDStrategy.OVERWRITE:
+                                    // replace the new value
+                                    ret++;
+                                    if (indexAdds) {
+                                        this.removeFromIndexes(this.mapData.get(id)!);
+                                    }
+                                    this.mapData.set(id, v);
+                                    if (indexAdds) {
+                                        this.addKeysToIndexes(v, indexKeys[i]!);
+                                    }
+                                    if (collectAdded) {
+                                        added.push(v);
+                                    }
+                                    break;
+                            }
+                        }
+                        else {
+                            ret++;
+                            this.mapData.set(id, v);
+                            if (indexAdds) {
+                                this.addKeysToIndexes(v, indexKeys[i]!);
+                            }
+                            if (collectAdded) {
+                                added.push(v);
+                            }
+                        }
+                    }
+                    break;
+                }
+                default:
+                case EBMemStorageStrategy.ARRAY: {
+                    const storedIDs = await this.storedArrayIDs();
+                    switch (this.collectionConfig.duplicateIDStrategy) {
+                        case EBDuplicateIDStrategy.ERROR: {
+                            const haveIDs = new Set(storedIDs.filter((id): id is string => id !== undefined));
+                            for (const id of incomingIDs) {
+                                if (id === undefined) {
+                                    continue;
+                                }
+                                if (haveIDs.has(id)) {
+                                    throw new Error(`duplicate id ${id}`);
+                                }
+                                haveIDs.add(id);
+                            }
+                            await this.assignMissingIDs(vs, incomingIDs);
+                            const indexKeys = indexAdds ? await this.indexKeysForAll(vs) : [];
+                            vs.forEach((v, i) => {
+                                ret++;
+                                this.arrayData.push(v);
+                                if (indexAdds) {
+                                    this.addKeysToIndexes(v, indexKeys[i]!);
+                                }
+                                if (collectAdded) {
+                                    added.push(v);
+                                }
+                            });
+                            break;
+                        }
+                        case EBDuplicateIDStrategy.IGNORE: {
+                            const ids = await this.assignMissingIDs(vs, incomingIDs);
+                            const seenIDs = new Set(storedIDs.filter((id): id is string => id !== undefined));
+                            const toAdd: T[] = [];
+                            ids.forEach((id, i) => {
+                                if (!seenIDs.has(id)) {
+                                    seenIDs.add(id);
+                                    toAdd.push(vs[i]!);
+                                }
+                            });
+                            const indexKeys = indexAdds ? await this.indexKeysForAll(toAdd) : [];
+                            toAdd.forEach((v, i) => {
+                                ret++;
+                                this.arrayData.push(v);
+                                if (indexAdds) {
+                                    this.addKeysToIndexes(v, indexKeys[i]!);
+                                }
+                                if (collectAdded) {
+                                    added.push(v);
+                                }
+                            });
+                            break;
+                        }
+                        default:
+                        case EBDuplicateIDStrategy.OVERWRITE: {
+                            const ids = await this.assignMissingIDs(vs, incomingIDs);
+                            const indexKeys = indexAdds ? await this.indexKeysForAll(vs) : [];
+                            const positions = new Map<string, number>();
+                            storedIDs.forEach((id, i) => {
+                                if (id !== undefined) {
+                                    positions.set(id, i);
+                                }
+                            });
+                            vs.forEach((v, i) => {
+                                const id = ids[i]!;
+                                const existingIndex = positions.get(id);
+                                ret++;
+                                if (existingIndex !== undefined) {
+                                    if (indexAdds) {
+                                        this.removeFromIndexes(this.arrayData[existingIndex]!);
+                                    }
+                                    this.arrayData[existingIndex] = v;
+                                }
+                                else {
+                                    positions.set(id, this.arrayData.length);
+                                    this.arrayData.push(v);
+                                }
+                                if (indexAdds) {
+                                    this.addKeysToIndexes(v, indexKeys[i]!);
+                                }
+                                if (collectAdded) {
+                                    added.push(v);
+                                }
+                            });
+                            break;
+                        }
+                    }
+                    break;
+                }
+            }
+            if (ret > 0) {
+                await this.doOnDirty();
+            }
+            await this.recordTxChanges(EBModificationType.ADD, added);
+            if (trackAdds && added.length > 0) {
+                await this.fireAdd(added);
+            }
+            return ret;
+        });
+    }
+
+    public async update(predicate: (v: T) => boolean, updater: (v: T) => Promise<T>): Promise<number> {
+        return this.runMutation(async () => {
+            await this.checkLoaded();
+            const trackUpdates = this.updateHandlers.length > 0;
+            const indexUpdates = this.indexDefinitions.size > 0;
+            const planned: Array<{
+                previous: T;
+                replacement: T;
+                replacementKeys: Array<[string, string[]]>;
+                store: (replacement: T) => void;
+            }> = [];
+            const plan = async (previous: T, store: (replacement: T) => void): Promise<void> => {
+                const replacement = await updater(previous);
+                if (replacement === undefined) {
+                    throw new Error("updater gave undefined");
+                }
+                const replacementKeys = indexUpdates ? await this.indexKeysFor(replacement) : [];
+                planned.push({ previous, replacement, replacementKeys, store });
+            };
+            switch (this.collectionConfig.memStorageStrategy) {
+                case EBMemStorageStrategy.MAP:
+                    for (const [key, value] of this.mapData) {
+                        if (predicate(value)) {
+                            await plan(value, (replacement) => { this.mapData.set(key, replacement); });
+                        }
+                    }
+                    break;
+                default:
+                case EBMemStorageStrategy.ARRAY:
+                    for (let i = 0; i < this.arrayData.length; i++) {
+                        const previous = this.arrayData[i]!;
+                        if (predicate(previous)) {
+                            await plan(previous, (replacement) => { this.arrayData[i] = replacement; });
+                        }
+                    }
+                    break;
+            }
+            for (const { previous, replacement, replacementKeys, store } of planned) {
+                if (indexUpdates) {
+                    this.removeFromIndexes(previous);
+                }
+                store(replacement);
+                if (indexUpdates) {
+                    this.addKeysToIndexes(replacement, replacementKeys);
+                }
+            }
+            if (planned.length > 0) {
+                await this.doOnDirty();
+            }
+            await this.recordTxChanges(EBModificationType.UPDATE, planned.map(({ replacement }) => replacement));
+            if (trackUpdates && planned.length > 0) {
+                await this.fireUpdate(planned.map(({ previous, replacement }) => [previous, replacement]));
+            }
+            return planned.length;
+        });
+    }
+
+    private async doOnDirty(): Promise<void> {
+        this.dirty = true;
+        if (this.collectionConfig.autoFlushStrategy == EBAutoFlushStrategy.ALWAYS_AUTO_FLUSH) {
+            await this.flush();
+        }
+    }
+
+    public async flush(): Promise<void> {
+        return this.runProtected(async () => {
+            if (this.loaded) {
+                switch (this.collectionConfig.memStorageStrategy) {
+                    case EBMemStorageStrategy.MAP:
+                        await this.database.io.writeMapData(this.collectionID, this.mapData);
+                        break;
+                    default:
+                    case EBMemStorageStrategy.ARRAY:
+                        await this.database.io.writeArrayData(this.collectionID, this.arrayData);
+                        break;
+                }
+                this.dirty = false;
+            }
+        });
+    }
+
+    public async reload(): Promise<void> {
+        return this.runProtected(async () => {
+            switch (this.collectionConfig.memStorageStrategy) {
+                case EBMemStorageStrategy.MAP:
+                    this.mapData = await this.database.io.loadMapData<T>(this.collectionID);
+                    break;
+                default:
+                case EBMemStorageStrategy.ARRAY:
+                    this.arrayData = await this.database.io.loadArrayData<T>(this.collectionID);
+                    break;
+            }
+            this.resolveIndexDefinitions();
+            await this.rebuildIndexRuntime();
+            this.loaded = true;
+            this.dirty = false;
+        });
+    }
+
+    public async unload(): Promise<void> {
+        return this.runProtected(async () => {
+            if (this.dirty) {
+                switch (this.collectionConfig.dirtyUnloadStrategy) {
+                    case EBDirtyUnloadStrategy.FLUSH:
+                        await this.flush();
+                        break;
+                    case EBDirtyUnloadStrategy.ERROR:
+                        throw new Error("cannot unload unwritten collection");
+                    default:
+                    case EBDirtyUnloadStrategy.IGNORE:
+                        // do nothing
+                        break;
+
+                }
+            }
+            switch (this.collectionConfig.memStorageStrategy) {
+                case EBMemStorageStrategy.MAP:
+                    this.mapData = new Map();
+                    break;
+                default:
+                case EBMemStorageStrategy.ARRAY:
+                    this.arrayData = [];
+                    break;
+            }
+            this.indexRuntime.clear();
+            this.indexedKeys.clear();
+            this.loaded = false;
+            this.dirty = false;
+        });
+    }
+
+    /**
+     * Creates a transaction-scoped working copy: same identity/config as this collection, but with
+     * independently mutable in-memory state (auto-flush forced off, regardless of this collection's own
+     * `autoFlushStrategy`) and no change-event handlers of its own. Call only while already holding this
+     * collection's mutex for the whole transaction (see {@link BDatabase.transaction}).
+     */
+    public async cloneForTransaction(): Promise<BCollection<T>> {
+        return this.runProtected(async () => {
+            await this.checkLoaded();
+            const txConfig: IBCollectionConfig = { ...this.collectionConfig, autoFlushStrategy: EBAutoFlushStrategy.NO_AUTO_FLUSH };
+            const clone = new BCollection<T>(this.database, this.collectionID, this.collectionName, txConfig);
+            clone.mapData = new Map(this.mapData);
+            clone.arrayData = this.arrayData.slice();
+            clone.loaded = true;
+            clone.transactionClone = true;
+            if (this.addHandlers.length > 0 || this.updateHandlers.length > 0 || this.deleteHandlers.length > 0) {
+                clone.txChanges = new Map();
+            }
+            clone.resolveIndexDefinitions();
+            await clone.rebuildIndexRuntime();
+            return clone;
+        });
+    }
+
+    /**
+     * Writes `clone`'s state to a temp file (see {@link IPreparedWrite}) and computes the handler events for the
+     * net changes against this collection's current state: one per id the transaction touched, typed by the last
+     * operation on it (`DELETE` if gone, `ADD` if new or last stored via `add()`, `UPDATE` otherwise). Nothing is
+     * applied until `apply()`; call only while already holding this collection's mutex for the whole transaction.
+     */
+    public async prepareTransactionCommit(clone: BCollection<T>): Promise<IPreparedTransactionCommit> {
+        return this.runProtected(async () => {
+            await this.checkLoaded();
+            const added: T[] = [];
+            const updated: Array<[T, T]> = [];
+            const deleted: T[] = [];
+            if (clone.txChanges && clone.txChanges.size > 0) {
+                const before = await this.idKeyedSnapshot();
+                const after = await clone.idKeyedSnapshot();
+                for (const [id, modificationType] of clone.txChanges) {
+                    if (!after.has(id)) {
+                        if (before.has(id)) {
+                            deleted.push(before.get(id)!);
+                        }
+                    }
+                    else if (!before.has(id) || modificationType === EBModificationType.ADD) {
+                        added.push(after.get(id)!);
+                    }
+                    else {
+                        updated.push([before.get(id)!, after.get(id)!]);
+                    }
+                }
+            }
+            let write: IPreparedWrite;
+            switch (this.collectionConfig.memStorageStrategy) {
+                case EBMemStorageStrategy.MAP:
+                    write = await this.database.io.prepareMapData(this.collectionID, clone.mapData);
+                    break;
+                default:
+                case EBMemStorageStrategy.ARRAY:
+                    write = await this.database.io.prepareArrayData(this.collectionID, clone.arrayData);
+                    break;
+            }
+            return {
+                write,
+                apply: () => {
+                    this.mapData = clone.mapData;
+                    this.arrayData = clone.arrayData;
+                    this.indexRuntime = clone.indexRuntime;
+                    this.indexedKeys = clone.indexedKeys;
+                    this.dirty = false;
+                },
+                fire: () => this.runProtected(async () => {
+                    if (added.length > 0) {
+                        await this.fireAdd(added);
+                    }
+                    if (updated.length > 0) {
+                        await this.fireUpdate(updated);
+                    }
+                    if (deleted.length > 0) {
+                        await this.fireDelete(deleted);
+                    }
+                }),
+            };
+        });
+    }
+
 }