@incoqnito.io/bajadab 1.1.0 → 1.2.0

package/src/main/basics.ts

@@ -1,482 +1,777 @@
-import logger from "winston";
-
-import { uuidv7, } from "uuidv7";
-import { runMultiReentrantProtected, runReentrantProtected } from "@incoqnito.io/ts-iq-core";
-
-import type { IODatabase } from "./fileio.js";
-
-import {
-    EBArrayDeletionStrategy,
-    EBAutoFlushStrategy,
-    EBAutoIDStrategy,
-    EBDirtyUnloadStrategy,
-    EBDuplicateIDStrategy,
-    EBMemStorageStrategy,
-    IBCollection,
-    IBCollectionConfig,
-    IBDatabase,
-    IBDatabaseConfig,
-    IBFunctionRegistry,
-    TGetIDFunction,
-    TSetIDFunction,
-} from "./base.js";
-import { canonicalName } from "./utils.js";
-
-/** Default {@link TGetIDFunction}: reads `v.id`, `undefined` if absent. */
-export const DEFAULT_GET_ID: TGetIDFunction = (v: any) => ("id" in v) ? v.id : undefined;
-/** Default {@link TSetIDFunction}: assigns to `v.id`. */
-export const DEFAULT_SET_ID: TSetIDFunction = async (v: any, id: string) => { v.id = id; };
-
-const enum MutexKeys {
-    // because of the spaces this will never be a collection name and as such is safe to use as a mutex key
-    DATABASE = " database ",
-}
-
-/** {@link IBCollectionConfig} used when a collection is created without an explicit one. */
-export const DEFAULT_COLLECTION_CONFIG: IBCollectionConfig = {
-    arrayDeletionStrategy: EBArrayDeletionStrategy.IN_PLACE,
-    autoIDStrategy: EBAutoIDStrategy.AUTO_ID_UUID,
-    duplicateIDStrategy: EBDuplicateIDStrategy.OVERWRITE,
-    memStorageStrategy: EBMemStorageStrategy.MAP,
-    dirtyUnloadStrategy: EBDirtyUnloadStrategy.FLUSH,
-    autoFlushStrategy: EBAutoFlushStrategy.ALWAYS_AUTO_FLUSH,
-};
-
-/** {@link IBDatabaseConfig} used when `getDatabase()` creates a fresh database. */
-export const DEFAULT_DATABASE_CONFIG: IBDatabaseConfig = {
-    defaultCollectionConfig: DEFAULT_COLLECTION_CONFIG,
-};
-
-export class BDatabase implements IBDatabase {
-    public readonly config: IBDatabaseConfig;
-    private collections: Map<string, BCollection> = new Map();
-    readonly io: IODatabase;
-    readonly functionRegistry?: IBFunctionRegistry;
-
-    public constructor(io: IODatabase, config?: IBDatabaseConfig, functionRegistry?: IBFunctionRegistry) {
-        this.config = config ?? DEFAULT_DATABASE_CONFIG;
-        this.io = io;
-        this.functionRegistry = functionRegistry;
-    }
-
-    public addCollection(collection: BCollection): void {
-        this.collections.set(collection.collectionName, collection);
-    }
-
-    public async getCollection<T>(collectionName: string, collectionConfig?: IBCollectionConfig): Promise<IBCollection<T>> {
-        return runReentrantProtected(MutexKeys.DATABASE, async () => {
-            const cname = canonicalName(collectionName);
-            let ret: BCollection<T> | undefined = this.collections.get(cname);
-            if (!ret) {
-                ret = await this.createCollection(collectionName, collectionConfig);
-            }
-            return ret;
-        });
-    }
-
-    public async createCollection<T = any>(collectionName: string, collectionConfig?: IBCollectionConfig): Promise<BCollection<T>> {
-        return runReentrantProtected(MutexKeys.DATABASE, async () => {
-            const cname = canonicalName(collectionName);
-            if (this.collections.has(cname)) {
-                throw new Error(`cannot create already existing collection ${collectionName}`);
-            }
-            const id = uuidv7();
-            const ret: BCollection<T> = new BCollection<T>(this, id, collectionName, collectionConfig ?? this.config.defaultCollectionConfig);
-            this.addCollection(ret);
-            return this.io.createCollection<T>(ret);
-        });
-    }
-
-    public async hasCollection(collectionName: string): Promise<boolean> {
-        return runReentrantProtected(MutexKeys.DATABASE, async () => {
-            return this.collections.has(canonicalName(collectionName));
-        });
-    }
-
-    public async dropCollection(collectionName: string): Promise<void> {
-        const cname = canonicalName(collectionName);
-        return runMultiReentrantProtected([MutexKeys.DATABASE, cname], async () => {
-            const coll: BCollection = this.collections.get(cname) as BCollection;
-            if (coll) {
-                coll.markDropped();
-                this.collections.delete(cname);
-                return this.io.deleteCollection(cname);
-            }
-            else {
-                throw new Error(`no such collection ${collectionName}`);
-            }
-        });
-    }
-
-}
-
-export class BCollection<T = any> implements IBCollection<T> {
-    readonly database: BDatabase;
-    readonly collectionConfig: IBCollectionConfig;
-    readonly collectionName: string;
-    readonly collectionID: string;
-
-    private mapData: Map<string, T> = new Map();
-    private arrayData: Array<T> = [];
-    private loaded: boolean = false;
-    private dirty: boolean = false;
-    private dropped: boolean = false;
-
-    public constructor(database: BDatabase, collectionID: string, collectionName: string, collectionConfig: IBCollectionConfig) {
-        this.database = database;
-        this.collectionID = collectionID;
-        this.collectionName = canonicalName(collectionName);
-        this.collectionConfig = collectionConfig;
-        switch (collectionConfig.memStorageStrategy) {
-            case EBMemStorageStrategy.MAP:
-                // if (!collectionConfig.getID || !collectionConfig.setID) {
-                //     throw new Error("need id accessors configured when using map mem storage");
-                // }
-                break;
-            default:
-            case EBMemStorageStrategy.ARRAY:
-                break;
-        }
-    }
-
-    private async checkLoaded(): Promise<void> {
-        if (!this.loaded) {
-            await this.reload();
-        }
-    }
-
-    private async runProtected<R>(fn: () => Promise<R>): Promise<R> {
-        return runReentrantProtected(this.collectionName, async () => {
-            if (this.dropped) {
-                throw new Error(`collection "${this.collectionName}" has been dropped`);
-            }
-            return fn();
-        });
-    }
-
-    /**
-     * call this only within the proteing mutex
-     */
-    public markDropped(): void {
-        this.dropped = true;
-    }
-
-    public async get(predicate?: (v: T) => boolean): Promise<T[]> {
-        return this.runProtected(async () => {
-            await this.checkLoaded();
-            switch (this.collectionConfig.memStorageStrategy) {
-                case EBMemStorageStrategy.MAP:
-                    return !predicate ?
-                        this.mapData.values().toArray() :
-                        this.mapData.values().filter(predicate).toArray();
-                default:
-                case EBMemStorageStrategy.ARRAY:
-                    return !predicate ?
-                        // MARK this may be a bit slow for bigger collections    
-                        [...this.arrayData] :
-                        this.arrayData.filter(predicate);
-            }
-        });
-    }
-
-    public async delete(predicate: (v: T) => boolean): Promise<number> {
-        return this.runProtected(async () => {
-            await this.checkLoaded();
-            let ret = 0;
-            switch (this.collectionConfig.memStorageStrategy) {
-                case EBMemStorageStrategy.MAP:
-                    for (const [key, value] of this.mapData) {
-                        if (predicate(value)) {
-                            ret++;
-                            this.mapData.delete(key);
-                        }
-                    }
-                    break;
-                default:
-                case EBMemStorageStrategy.ARRAY:
-                    switch (this.collectionConfig.arrayDeletionStrategy) {
-                        case EBArrayDeletionStrategy.IN_PLACE:
-                            let writeIndex = 0;
-                            for (let readIndex = 0; readIndex < this.arrayData.length; readIndex++) {
-                                const item = this.arrayData[readIndex];
-                                if (!predicate(item!)) {
-                                    this.arrayData[writeIndex] = item!;
-                                    writeIndex++;
-                                }
-                                else {
-                                    ret++;
-                                }
-                            }
-                            this.arrayData.length = writeIndex;
-                            break;
-                        default:
-                        case EBArrayDeletionStrategy.NEW_ARRAY:
-                            const ol = this.arrayData.length;
-                            this.arrayData = this.arrayData.filter(v => !predicate(v));
-                            ret += (ol - this.arrayData.length);
-                            break;
-                    }
-                    break;
-            }
-            if (ret > 0) {
-                await this.doOnDirty();
-            }
-            return ret;
-        });
-    }
-
-    public async has(predicate: (v: T) => boolean): Promise<boolean> {
-        return this.runProtected(async () => {
-            await this.checkLoaded();
-            switch (this.collectionConfig.memStorageStrategy) {
-                case EBMemStorageStrategy.MAP:
-                    return this.mapData.values().find(predicate) !== undefined;
-                default:
-                case EBMemStorageStrategy.ARRAY:
-                    return this.arrayData.find(predicate) !== undefined;
-            }
-        });
-    }
-
-    private async getID(v: T): Promise<string> {
-        let getter: TGetIDFunction;
-        if (this.collectionConfig.getID) {
-            const found = this.database.functionRegistry?.fetch(this.collectionConfig.getID) as TGetIDFunction | undefined;
-            if (!found) {
-                throw new Error(`getID function "${this.collectionConfig.getID}" is not registered`);
-            }
-            getter = found;
-        }
-        else {
-            getter = DEFAULT_GET_ID;
-        }
-        let id = await getter(v);
-        if (!id) {
-            switch (this.collectionConfig.autoIDStrategy) {
-                case EBAutoIDStrategy.NO_AUTO_ID:
-                    throw new Error("cannot omit auto-ids and than trying to add a value without an id");
-                default:
-                case EBAutoIDStrategy.AUTO_ID_UUID:
-                    id = uuidv7();
-                    let setter: TSetIDFunction;
-                    if (this.collectionConfig.setID) {
-                        const found = this.database.functionRegistry?.fetch(this.collectionConfig.setID) as TSetIDFunction | undefined;
-                        if (!found) {
-                            throw new Error(`setID function "${this.collectionConfig.setID}" is not registered`);
-                        }
-                        setter = found;
-                    }
-                    else {
-                        setter = DEFAULT_SET_ID;
-                    }
-                    await setter(v, id);
-                    break;
-            }
-        }
-        return id;
-    }
-
-    public async add(...vs: T[]): Promise<number> {
-        return this.runProtected(async () => {
-            await this.checkLoaded();
-            switch (this.collectionConfig.memStorageStrategy) {
-                case EBMemStorageStrategy.MAP:
-                    let ret = 0;
-                    for (const v of vs) {
-                        const id = await this.getID(v);
-                        if (this.mapData.has(id)) {
-                            switch (this.collectionConfig.duplicateIDStrategy) {
-                                case EBDuplicateIDStrategy.ERROR:
-                                    throw new Error(`duplicate id ${id}`);
-                                case EBDuplicateIDStrategy.IGNORE:
-                                    // ignore the new value
-                                    break;
-                                default:
-                                case EBDuplicateIDStrategy.OVERWRITE:
-                                    // replace the new value
-                                    ret++;
-                                    this.mapData.set(id, v);
-                                    break;
-                            }
-                        }
-                        else {
-                            ret++;
-                            this.mapData.set(id, v);
-                        }
-                    }
-                    if (ret > 0) {
-                        await this.doOnDirty();
-                    }
-                    return ret;
-                default:
-                case EBMemStorageStrategy.ARRAY:
-                    switch (this.collectionConfig.duplicateIDStrategy) {
-                        case EBDuplicateIDStrategy.ERROR:
-                            const haveIDs = new Set<string>();
-                            for (const v of this.arrayData) {
-                                haveIDs.add(await this.getID(v));
-                            }
-                            for (const v of vs) {
-                                const id = await this.getID(v);
-                                if (haveIDs.has(id)) {
-                                    throw new Error(`duplicate id ${id}`);
-                                }
-                                haveIDs.add(id);
-                                this.arrayData.push(v);
-                            }
-                            break;
-                        case EBDuplicateIDStrategy.IGNORE:
-                            const seenIDs = new Set<string>();
-                            for (const v of this.arrayData) {
-                                seenIDs.add(await this.getID(v));
-                            }
-                            for (const v of vs) {
-                                const id = await this.getID(v);
-                                if (!seenIDs.has(id)) {
-                                    seenIDs.add(id);
-                                    this.arrayData.push(v);
-                                }
-                            }
-                            break;
-                        default:
-                        case EBDuplicateIDStrategy.OVERWRITE:
-                            const indexMap = new Map<string | number, number>();
-                            for (let i = 0; i < this.arrayData.length; i++) {
-                                indexMap.set(await this.getID(this.arrayData[i]!), i);
-                            }
-                            for (const v of vs) {
-                                const id = await this.getID(v);
-                                const existingIndex = indexMap.get(id);
-
-                                if (existingIndex !== undefined) {
-                                    this.arrayData[existingIndex] = v;
-                                }
-                                else {
-                                    // store new index in indexCache in the case this id appears again
-                                    indexMap.set(id, this.arrayData.length);
-                                    this.arrayData.push(v);
-                                }
-                            }
-                            break;
-                    }
-                    if (vs.length > 0) {
-                        await this.doOnDirty();
-                    }
-                    return vs.length;
-            }
-        });
-    }
-
-    public async update(predicate: (v: T) => boolean, updater: (v: T) => Promise<T>): Promise<number> {
-        return this.runProtected(async () => {
-            await this.checkLoaded();
-            let ret = 0;
-            switch (this.collectionConfig.memStorageStrategy) {
-                case EBMemStorageStrategy.MAP:
-                    for (const [key, value] of this.mapData) {
-                        if (predicate(value)) {
-                            const replacement = await updater(value);
-                            if (replacement !== undefined) {
-                                ret++;
-                                this.mapData.set(key, replacement);
-                            }
-                            else {
-                                throw new Error("updater gave undefined");
-                            }
-                        }
-                    }
-                    break;
-                default:
-                case EBMemStorageStrategy.ARRAY:
-                    for (let i = 0; i < this.arrayData.length; i++) {
-                        if (predicate(this.arrayData[i]!)) {
-                            const replacement = await updater(this.arrayData[i]!);
-                            if (replacement !== undefined) {
-                                ret++;
-                                this.arrayData[i] = replacement;
-                            }
-                            else {
-                                throw new Error("updater gave undefined");
-                            }
-                        }
-                    }
-                    break;
-            }
-            if (ret > 0) {
-                await this.doOnDirty();
-            }
-            return ret;
-        });
-    }
-
-    private async doOnDirty(): Promise<void> {
-        this.dirty = true;
-        if (this.collectionConfig.autoFlushStrategy == EBAutoFlushStrategy.ALWAYS_AUTO_FLUSH) {
-            await this.flush();
-        }
-    }
-
-    public async flush(): Promise<void> {
-        return this.runProtected(async () => {
-            if (this.loaded) {
-                switch (this.collectionConfig.memStorageStrategy) {
-                    case EBMemStorageStrategy.MAP:
-                        await this.database.io.writeMapData(this.collectionID, this.mapData);
-                        break;
-                    default:
-                    case EBMemStorageStrategy.ARRAY:
-                        await this.database.io.writeArrayData(this.collectionID, this.arrayData);
-                        break;
-                }
-                this.dirty = false;
-            }
-        });
-    }
-
-    public async reload(): Promise<void> {
-        return this.runProtected(async () => {
-            switch (this.collectionConfig.memStorageStrategy) {
-                case EBMemStorageStrategy.MAP:
-                    this.mapData = await this.database.io.loadMapData<T>(this.collectionID);
-                    break;
-                default:
-                case EBMemStorageStrategy.ARRAY:
-                    this.arrayData = await this.database.io.loadArrayData<T>(this.collectionID);
-                    break;
-            }
-            this.loaded = true;
-            this.dirty = false;
-        });
-    }
-
-    public async unload(): Promise<void> {
-        return this.runProtected(async () => {
-            if (this.dirty) {
-                switch (this.collectionConfig.dirtyUnloadStrategy) {
-                    case EBDirtyUnloadStrategy.FLUSH:
-                        await this.flush();
-                        break;
-                    case EBDirtyUnloadStrategy.ERROR:
-                        throw new Error("cannot unload unwritten collection");
-                    default:
-                    case EBDirtyUnloadStrategy.IGNORE:
-                        // do nothing
-                        break;
-
-                }
-            }
-            switch (this.collectionConfig.memStorageStrategy) {
-                case EBMemStorageStrategy.MAP:
-                    this.mapData = new Map();
-                    break;
-                default:
-                case EBMemStorageStrategy.ARRAY:
-                    this.arrayData = [];
-                    break;
-            }
-            this.loaded = false;
-            this.dirty = false;
-        });
-    }
-
+import logger from "winston";
+
+import { uuidv7, } from "uuidv7";
+import { runMultiReentrantProtected, runReentrantProtected } from "@incoqnito.io/ts-iq-core";
+
+import type { IODatabase } from "./fileio.js";
+
+import {
+    EBArrayDeletionStrategy,
+    EBAutoFlushStrategy,
+    EBAutoIDStrategy,
+    EBDirtyUnloadStrategy,
+    EBDuplicateIDStrategy,
+    EBHandlerMode,
+    EBMemStorageStrategy,
+    EBModificationType,
+    IBCollection,
+    IBCollectionConfig,
+    IBDatabase,
+    IBDatabaseConfig,
+    IBFunctionRegistry,
+    IHandlerRegistration,
+    TAddHandler,
+    TDeleteHandler,
+    TGetIDFunction,
+    TIndexKeyFunction,
+    TSetIDFunction,
+    TUpdateHandler,
+} from "./base.js";
+import { canonicalName } from "./utils.js";
+
+/** Default {@link TGetIDFunction}: reads `v.id`, `undefined` if absent. */
+export const DEFAULT_GET_ID: TGetIDFunction = (v: any) => ("id" in v) ? v.id : undefined;
+/** Default {@link TSetIDFunction}: assigns to `v.id`. */
+export const DEFAULT_SET_ID: TSetIDFunction = async (v: any, id: string) => { v.id = id; };
+
+const enum MutexKeys {
+    // because of the spaces this will never be a collection name and as such is safe to use as a mutex key
+    DATABASE = " database ",
+}
+
+/** {@link IBCollectionConfig} used when a collection is created without an explicit one. */
+export const DEFAULT_COLLECTION_CONFIG: IBCollectionConfig = {
+    arrayDeletionStrategy: EBArrayDeletionStrategy.IN_PLACE,
+    autoIDStrategy: EBAutoIDStrategy.AUTO_ID_UUID,
+    duplicateIDStrategy: EBDuplicateIDStrategy.OVERWRITE,
+    memStorageStrategy: EBMemStorageStrategy.MAP,
+    dirtyUnloadStrategy: EBDirtyUnloadStrategy.FLUSH,
+    autoFlushStrategy: EBAutoFlushStrategy.ALWAYS_AUTO_FLUSH,
+};
+
+/** {@link IBDatabaseConfig} used when `getDatabase()` creates a fresh database. */
+export const DEFAULT_DATABASE_CONFIG: IBDatabaseConfig = {
+    defaultCollectionConfig: DEFAULT_COLLECTION_CONFIG,
+};
+
+export class BDatabase implements IBDatabase {
+    public readonly config: IBDatabaseConfig;
+    private collections: Map<string, BCollection> = new Map();
+    readonly io: IODatabase;
+    readonly functionRegistry?: IBFunctionRegistry;
+
+    public constructor(io: IODatabase, config?: IBDatabaseConfig, functionRegistry?: IBFunctionRegistry) {
+        this.config = config ?? DEFAULT_DATABASE_CONFIG;
+        this.io = io;
+        this.functionRegistry = functionRegistry;
+    }
+
+    public addCollection(collection: BCollection): void {
+        this.collections.set(collection.collectionName, collection);
+    }
+
+    public async getCollection<T>(collectionName: string, collectionConfig?: IBCollectionConfig): Promise<IBCollection<T>> {
+        return runReentrantProtected(MutexKeys.DATABASE, async () => {
+            const cname = canonicalName(collectionName);
+            let ret: BCollection<T> | undefined = this.collections.get(cname);
+            if (!ret) {
+                ret = await this.createCollection(collectionName, collectionConfig);
+            }
+            return ret;
+        });
+    }
+
+    public async createCollection<T = any>(collectionName: string, collectionConfig?: IBCollectionConfig): Promise<BCollection<T>> {
+        return runReentrantProtected(MutexKeys.DATABASE, async () => {
+            const cname = canonicalName(collectionName);
+            if (this.collections.has(cname)) {
+                throw new Error(`cannot create already existing collection ${collectionName}`);
+            }
+            const id = uuidv7();
+            const ret: BCollection<T> = new BCollection<T>(this, id, collectionName, collectionConfig ?? this.config.defaultCollectionConfig);
+            this.addCollection(ret);
+            return this.io.createCollection<T>(ret);
+        });
+    }
+
+    public async hasCollection(collectionName: string): Promise<boolean> {
+        return runReentrantProtected(MutexKeys.DATABASE, async () => {
+            return this.collections.has(canonicalName(collectionName));
+        });
+    }
+
+    public async dropCollection(collectionName: string): Promise<void> {
+        const cname = canonicalName(collectionName);
+        return runMultiReentrantProtected([MutexKeys.DATABASE, cname], async () => {
+            const coll: BCollection = this.collections.get(cname) as BCollection;
+            if (coll) {
+                coll.markDropped();
+                this.collections.delete(cname);
+                return this.io.deleteCollection(cname);
+            }
+            else {
+                throw new Error(`no such collection ${collectionName}`);
+            }
+        });
+    }
+
+}
+
+export class BCollection<T = any> implements IBCollection<T> {
+    readonly database: BDatabase;
+    readonly collectionConfig: IBCollectionConfig;
+    readonly collectionName: string;
+    readonly collectionID: string;
+
+    private mapData: Map<string, T> = new Map();
+    private arrayData: Array<T> = [];
+    private loaded: boolean = false;
+    private dirty: boolean = false;
+    private dropped: boolean = false;
+
+    private addHandlers: Array<{ handler: TAddHandler<T>; mode: EBHandlerMode }> = [];
+    private updateHandlers: Array<{ handler: TUpdateHandler<T>; mode: EBHandlerMode }> = [];
+    private deleteHandlers: Array<{ handler: TDeleteHandler<T>; mode: EBHandlerMode }> = [];
+
+    private indexDefinitions: Map<string, TIndexKeyFunction<T>> = new Map();
+    private indexRuntime: Map<string, Map<string, T[]>> = new Map();
+
+    public constructor(database: BDatabase, collectionID: string, collectionName: string, collectionConfig: IBCollectionConfig) {
+        this.database = database;
+        this.collectionID = collectionID;
+        this.collectionName = canonicalName(collectionName);
+        this.collectionConfig = collectionConfig;
+        switch (collectionConfig.memStorageStrategy) {
+            case EBMemStorageStrategy.MAP:
+                // if (!collectionConfig.getID || !collectionConfig.setID) {
+                //     throw new Error("need id accessors configured when using map mem storage");
+                // }
+                break;
+            default:
+            case EBMemStorageStrategy.ARRAY:
+                break;
+        }
+    }
+
+    private async checkLoaded(): Promise<void> {
+        if (!this.loaded) {
+            await this.reload();
+        }
+    }
+
+    private async runExclusive<R>(fn: () => Promise<R>): Promise<R> {
+        return runReentrantProtected(this.collectionName, fn);
+    }
+
+    private async runProtected<R>(fn: () => Promise<R>): Promise<R> {
+        return this.runExclusive(async () => {
+            if (this.dropped) {
+                throw new Error(`collection "${this.collectionName}" has been dropped`);
+            }
+            return fn();
+        });
+    }
+
+    /**
+     * call this only within the protecting mutex
+     */
+    public markDropped(): void {
+        this.dropped = true;
+    }
+
+    public async get(predicate?: (v: T) => boolean): Promise<T[]> {
+        return this.runProtected(async () => {
+            await this.checkLoaded();
+            switch (this.collectionConfig.memStorageStrategy) {
+                case EBMemStorageStrategy.MAP:
+                    return !predicate ?
+                        this.mapData.values().toArray() :
+                        this.mapData.values().filter(predicate).toArray();
+                default:
+                case EBMemStorageStrategy.ARRAY:
+                    return !predicate ?
+                        this.arrayData.slice() :
+                        this.arrayData.filter(predicate);
+            }
+        });
+    }
+
+    public async count(predicate?: (v: T) => boolean): Promise<number> {
+        return this.runProtected(async () => {
+            await this.checkLoaded();
+            switch (this.collectionConfig.memStorageStrategy) {
+                case EBMemStorageStrategy.MAP:
+                    return !predicate ?
+                        this.mapData.size :
+                        this.mapData.values().filter(predicate).reduce((n) => n + 1, 0);
+                default:
+                case EBMemStorageStrategy.ARRAY:
+                    return !predicate ?
+                        this.arrayData.length :
+                        this.arrayData.filter(predicate).length;
+            }
+        });
+    }
+
+    public async getByIndex(indexKey: string, key: string): Promise<T[]> {
+        return this.runProtected(async () => {
+            await this.checkLoaded();
+            const byKey = this.indexRuntime.get(indexKey);
+            if (!byKey) {
+                throw new Error(`index "${indexKey}" is not configured`);
+            }
+            return byKey.get(key)?.slice() ?? [];
+        });
+    }
+
+    private async invokeHandler(fn: () => void | Promise<void>, modificationType: EBModificationType): Promise<void> {
+        try {
+            await fn();
+        }
+        catch (error) {
+            logger.error(`bajadab: collection "${this.collectionName}" ${modificationType} handler failed`, error);
+        }
+    }
+
+    private async fireAdd(added: T[]): Promise<void> {
+        for (const v of added) {
+            for (const { handler, mode } of this.addHandlers) {
+                const call = this.invokeHandler(() => handler(v), EBModificationType.ADD);
+                if (mode === EBHandlerMode.SYNC) {
+                    await call;
+                }
+            }
+        }
+    }
+
+    private async fireUpdate(pairs: Array<[T, T]>): Promise<void> {
+        for (const [previous, updated] of pairs) {
+            for (const { handler, mode } of this.updateHandlers) {
+                const call = this.invokeHandler(() => handler(previous, updated), EBModificationType.UPDATE);
+                if (mode === EBHandlerMode.SYNC) {
+                    await call;
+                }
+            }
+        }
+    }
+
+    private async fireDelete(deleted: T[]): Promise<void> {
+        for (const v of deleted) {
+            for (const { handler, mode } of this.deleteHandlers) {
+                const call = this.invokeHandler(() => handler(v), EBModificationType.DELETE);
+                if (mode === EBHandlerMode.SYNC) {
+                    await call;
+                }
+            }
+        }
+    }
+
+    public async on(modificationType: EBModificationType.ADD, handler: TAddHandler<T>, mode?: EBHandlerMode): Promise<IHandlerRegistration>;
+    public async on(modificationType: EBModificationType.UPDATE, handler: TUpdateHandler<T>, mode?: EBHandlerMode): Promise<IHandlerRegistration>;
+    public async on(modificationType: EBModificationType.DELETE, handler: TDeleteHandler<T>, mode?: EBHandlerMode): Promise<IHandlerRegistration>;
+    public async on(modificationType: EBModificationType, handler: TAddHandler<T> | TUpdateHandler<T> | TDeleteHandler<T>, mode: EBHandlerMode = EBHandlerMode.SYNC): Promise<IHandlerRegistration> {
+        return this.runProtected(async () => {
+            switch (modificationType) {
+                case EBModificationType.ADD: {
+                    const h = handler as TAddHandler<T>;
+                    this.addHandlers.push({ handler: h, mode });
+                    return { cancel: async () => this.runExclusive(async () => { this.addHandlers = this.addHandlers.filter(x => x.handler !== h); }) };
+                }
+                case EBModificationType.UPDATE: {
+                    const h = handler as TUpdateHandler<T>;
+                    this.updateHandlers.push({ handler: h, mode });
+                    return { cancel: async () => this.runExclusive(async () => { this.updateHandlers = this.updateHandlers.filter(x => x.handler !== h); }) };
+                }
+                case EBModificationType.DELETE: {
+                    const h = handler as TDeleteHandler<T>;
+                    this.deleteHandlers.push({ handler: h, mode });
+                    return { cancel: async () => this.runExclusive(async () => { this.deleteHandlers = this.deleteHandlers.filter(x => x.handler !== h); }) };
+                }
+                default:
+                    throw new Error(`unknown modification type ${modificationType}`);
+            }
+        });
+    }
+
+    public async delete(predicate: (v: T) => boolean): Promise<number> {
+        return this.runProtected(async () => {
+            await this.checkLoaded();
+            let ret = 0;
+            const trackDeletes = this.deleteHandlers.length > 0;
+            const indexDeletes = this.indexDefinitions.size > 0;
+            const deleted: T[] = [];
+            switch (this.collectionConfig.memStorageStrategy) {
+                case EBMemStorageStrategy.MAP:
+                    for (const [key, value] of this.mapData) {
+                        if (predicate(value)) {
+                            ret++;
+                            if (indexDeletes) {
+                                await this.removeFromIndexes(value);
+                            }
+                            this.mapData.delete(key);
+                            if (trackDeletes) {
+                                deleted.push(value);
+                            }
+                        }
+                    }
+                    break;
+                default:
+                case EBMemStorageStrategy.ARRAY:
+                    switch (this.collectionConfig.arrayDeletionStrategy) {
+                        case EBArrayDeletionStrategy.IN_PLACE:
+                            let writeIndex = 0;
+                            for (let readIndex = 0; readIndex < this.arrayData.length; readIndex++) {
+                                const item = this.arrayData[readIndex];
+                                if (!predicate(item!)) {
+                                    this.arrayData[writeIndex] = item!;
+                                    writeIndex++;
+                                }
+                                else {
+                                    ret++;
+                                    if (indexDeletes) {
+                                        await this.removeFromIndexes(item!);
+                                    }
+                                    if (trackDeletes) {
+                                        deleted.push(item!);
+                                    }
+                                }
+                            }
+                            this.arrayData.length = writeIndex;
+                            break;
+                        default:
+                        case EBArrayDeletionStrategy.NEW_ARRAY:
+                            const ol = this.arrayData.length;
+                            if (trackDeletes || indexDeletes) {
+                                const removed = this.arrayData.filter(predicate);
+                                if (trackDeletes) {
+                                    deleted.push(...removed);
+                                }
+                                if (indexDeletes) {
+                                    for (const v of removed) {
+                                        await this.removeFromIndexes(v);
+                                    }
+                                }
+                            }
+                            this.arrayData = this.arrayData.filter(v => !predicate(v));
+                            ret += (ol - this.arrayData.length);
+                            break;
+                    }
+                    break;
+            }
+            if (ret > 0) {
+                await this.doOnDirty();
+            }
+            if (deleted.length > 0) {
+                await this.fireDelete(deleted);
+            }
+            return ret;
+        });
+    }
+
+    public async has(predicate: (v: T) => boolean): Promise<boolean> {
+        return this.runProtected(async () => {
+            await this.checkLoaded();
+            switch (this.collectionConfig.memStorageStrategy) {
+                case EBMemStorageStrategy.MAP:
+                    return this.mapData.values().find(predicate) !== undefined;
+                default:
+                case EBMemStorageStrategy.ARRAY:
+                    return this.arrayData.find(predicate) !== undefined;
+            }
+        });
+    }
+
+    private resolveIndexDefinitions(): void {
+        this.indexDefinitions = new Map();
+        if (!this.collectionConfig.indexes) {
+            return;
+        }
+        for (const [indexKey, registryKey] of Object.entries(this.collectionConfig.indexes)) {
+            const extractor = this.database.functionRegistry?.fetch(registryKey) as TIndexKeyFunction<T> | undefined;
+            if (!extractor) {
+                throw new Error(`index key function "${registryKey}" is not registered`);
+            }
+            this.indexDefinitions.set(indexKey, extractor);
+        }
+    }
+
+    private async indexKeysFor(v: T): Promise<Array<[string, string[]]>> {
+        const result: Array<[string, string[]]> = [];
+        for (const [indexKey, extractor] of this.indexDefinitions) {
+            const extracted = await extractor(v);
+            if (extracted === undefined) {
+                continue;
+            }
+            result.push([indexKey, Array.isArray(extracted) ? extracted : [extracted]]);
+        }
+        return result;
+    }
+
+    private async addToIndexes(v: T): Promise<void> {
+        for (const [indexKey, keys] of await this.indexKeysFor(v)) {
+            const byKey = this.indexRuntime.get(indexKey)!;
+            for (const key of keys) {
+                let bucket = byKey.get(key);
+                if (!bucket) {
+                    bucket = [];
+                    byKey.set(key, bucket);
+                }
+                bucket.push(v);
+            }
+        }
+    }
+
+    private async removeFromIndexes(v: T): Promise<void> {
+        for (const [indexKey, keys] of await this.indexKeysFor(v)) {
+            const byKey = this.indexRuntime.get(indexKey)!;
+            for (const key of keys) {
+                const bucket = byKey.get(key);
+                if (!bucket) {
+                    continue;
+                }
+                const at = bucket.indexOf(v);
+                if (at !== -1) {
+                    bucket.splice(at, 1);
+                }
+                if (bucket.length === 0) {
+                    byKey.delete(key);
+                }
+            }
+        }
+    }
+
+    /**
+     * full rebuild from currently loaded data; only needed right after (re)load, since every
+     * value then is a freshly deserialized object the incremental add/update/delete hooks never saw
+     */
+    private async rebuildIndexRuntime(): Promise<void> {
+        // every configured index gets an entry up front, even if empty, so getByIndex()/
+        // addToIndexes()/removeFromIndexes() can assume it's there instead of checking twice
+        this.indexRuntime = new Map(Array.from(this.indexDefinitions.keys(), (indexKey) => [indexKey, new Map()]));
+        if (this.indexDefinitions.size === 0) {
+            return;
+        }
+        const values = this.collectionConfig.memStorageStrategy === EBMemStorageStrategy.MAP ?
+            [...this.mapData.values()] : this.arrayData;
+        for (const v of values) {
+            await this.addToIndexes(v);
+        }
+    }
+
+    private async getID(v: T): Promise<string> {
+        let getter: TGetIDFunction;
+        if (this.collectionConfig.getID) {
+            const found = this.database.functionRegistry?.fetch(this.collectionConfig.getID) as TGetIDFunction | undefined;
+            if (!found) {
+                throw new Error(`getID function "${this.collectionConfig.getID}" is not registered`);
+            }
+            getter = found;
+        }
+        else {
+            getter = DEFAULT_GET_ID;
+        }
+        let id = await getter(v);
+        if (!id) {
+            switch (this.collectionConfig.autoIDStrategy) {
+                case EBAutoIDStrategy.NO_AUTO_ID:
+                    throw new Error("cannot omit auto-ids and than trying to add a value without an id");
+                default:
+                case EBAutoIDStrategy.AUTO_ID_UUID:
+                    id = uuidv7();
+                    let setter: TSetIDFunction;
+                    if (this.collectionConfig.setID) {
+                        const found = this.database.functionRegistry?.fetch(this.collectionConfig.setID) as TSetIDFunction | undefined;
+                        if (!found) {
+                            throw new Error(`setID function "${this.collectionConfig.setID}" is not registered`);
+                        }
+                        setter = found;
+                    }
+                    else {
+                        setter = DEFAULT_SET_ID;
+                    }
+                    await setter(v, id);
+                    break;
+            }
+        }
+        return id;
+    }
+
+    public async add(...vs: T[]): Promise<number> {
+        return this.runProtected(async () => {
+            await this.checkLoaded();
+            switch (this.collectionConfig.memStorageStrategy) {
+                case EBMemStorageStrategy.MAP:
+                    let ret = 0;
+                    const trackAddsMap = this.addHandlers.length > 0;
+                    const indexAddsMap = this.indexDefinitions.size > 0;
+                    const addedMap: T[] = [];
+                    for (const v of vs) {
+                        const id = await this.getID(v);
+                        if (this.mapData.has(id)) {
+                            switch (this.collectionConfig.duplicateIDStrategy) {
+                                case EBDuplicateIDStrategy.ERROR:
+                                    throw new Error(`duplicate id ${id}`);
+                                case EBDuplicateIDStrategy.IGNORE:
+                                    // ignore the new value
+                                    break;
+                                default:
+                                case EBDuplicateIDStrategy.OVERWRITE:
+                                    // replace the new value
+                                    ret++;
+                                    if (indexAddsMap) {
+                                        await this.removeFromIndexes(this.mapData.get(id)!);
+                                    }
+                                    this.mapData.set(id, v);
+                                    if (indexAddsMap) {
+                                        await this.addToIndexes(v);
+                                    }
+                                    if (trackAddsMap) {
+                                        addedMap.push(v);
+                                    }
+                                    break;
+                            }
+                        }
+                        else {
+                            ret++;
+                            this.mapData.set(id, v);
+                            if (indexAddsMap) {
+                                await this.addToIndexes(v);
+                            }
+                            if (trackAddsMap) {
+                                addedMap.push(v);
+                            }
+                        }
+                    }
+                    if (ret > 0) {
+                        await this.doOnDirty();
+                    }
+                    if (addedMap.length > 0) {
+                        await this.fireAdd(addedMap);
+                    }
+                    return ret;
+                default:
+                case EBMemStorageStrategy.ARRAY:
+                    const trackAddsArray = this.addHandlers.length > 0;
+                    const indexAddsArray = this.indexDefinitions.size > 0;
+                    const addedArray: T[] = [];
+                    switch (this.collectionConfig.duplicateIDStrategy) {
+                        case EBDuplicateIDStrategy.ERROR:
+                            const haveIDs = new Set<string>();
+                            for (const v of this.arrayData) {
+                                haveIDs.add(await this.getID(v));
+                            }
+                            for (const v of vs) {
+                                const id = await this.getID(v);
+                                if (haveIDs.has(id)) {
+                                    throw new Error(`duplicate id ${id}`);
+                                }
+                                haveIDs.add(id);
+                                this.arrayData.push(v);
+                                if (indexAddsArray) {
+                                    await this.addToIndexes(v);
+                                }
+                                if (trackAddsArray) {
+                                    addedArray.push(v);
+                                }
+                            }
+                            break;
+                        case EBDuplicateIDStrategy.IGNORE:
+                            const seenIDs = new Set<string>();
+                            for (const v of this.arrayData) {
+                                seenIDs.add(await this.getID(v));
+                            }
+                            for (const v of vs) {
+                                const id = await this.getID(v);
+                                if (!seenIDs.has(id)) {
+                                    seenIDs.add(id);
+                                    this.arrayData.push(v);
+                                    if (indexAddsArray) {
+                                        await this.addToIndexes(v);
+                                    }
+                                    if (trackAddsArray) {
+                                        addedArray.push(v);
+                                    }
+                                }
+                            }
+                            break;
+                        default:
+                        case EBDuplicateIDStrategy.OVERWRITE:
+                            const indexMap = new Map<string | number, number>();
+                            for (let i = 0; i < this.arrayData.length; i++) {
+                                indexMap.set(await this.getID(this.arrayData[i]!), i);
+                            }
+                            for (const v of vs) {
+                                const id = await this.getID(v);
+                                const existingIndex = indexMap.get(id);
+
+                                if (existingIndex !== undefined) {
+                                    if (indexAddsArray) {
+                                        await this.removeFromIndexes(this.arrayData[existingIndex]!);
+                                    }
+                                    this.arrayData[existingIndex] = v;
+                                }
+                                else {
+                                    // store new index in indexCache in the case this id appears again
+                                    indexMap.set(id, this.arrayData.length);
+                                    this.arrayData.push(v);
+                                }
+                                if (indexAddsArray) {
+                                    await this.addToIndexes(v);
+                                }
+                                if (trackAddsArray) {
+                                    addedArray.push(v);
+                                }
+                            }
+                            break;
+                    }
+                    if (vs.length > 0) {
+                        await this.doOnDirty();
+                    }
+                    if (addedArray.length > 0) {
+                        await this.fireAdd(addedArray);
+                    }
+                    return vs.length;
+            }
+        });
+    }
+
+    public async update(predicate: (v: T) => boolean, updater: (v: T) => Promise<T>): Promise<number> {
+        return this.runProtected(async () => {
+            await this.checkLoaded();
+            let ret = 0;
+            const trackUpdates = this.updateHandlers.length > 0;
+            const indexUpdates = this.indexDefinitions.size > 0;
+            const updated: Array<[T, T]> = [];
+            switch (this.collectionConfig.memStorageStrategy) {
+                case EBMemStorageStrategy.MAP:
+                    for (const [key, value] of this.mapData) {
+                        if (predicate(value)) {
+                            const replacement = await updater(value);
+                            if (replacement !== undefined) {
+                                ret++;
+                                if (indexUpdates) {
+                                    await this.removeFromIndexes(value);
+                                }
+                                this.mapData.set(key, replacement);
+                                if (indexUpdates) {
+                                    await this.addToIndexes(replacement);
+                                }
+                                if (trackUpdates) {
+                                    updated.push([value, replacement]);
+                                }
+                            }
+                            else {
+                                throw new Error("updater gave undefined");
+                            }
+                        }
+                    }
+                    break;
+                default:
+                case EBMemStorageStrategy.ARRAY:
+                    for (let i = 0; i < this.arrayData.length; i++) {
+                        if (predicate(this.arrayData[i]!)) {
+                            const previous = this.arrayData[i]!;
+                            const replacement = await updater(previous);
+                            if (replacement !== undefined) {
+                                ret++;
+                                if (indexUpdates) {
+                                    await this.removeFromIndexes(previous);
+                                }
+                                this.arrayData[i] = replacement;
+                                if (indexUpdates) {
+                                    await this.addToIndexes(replacement);
+                                }
+                                if (trackUpdates) {
+                                    updated.push([previous, replacement]);
+                                }
+                            }
+                            else {
+                                throw new Error("updater gave undefined");
+                            }
+                        }
+                    }
+                    break;
+            }
+            if (ret > 0) {
+                await this.doOnDirty();
+            }
+            if (updated.length > 0) {
+                await this.fireUpdate(updated);
+            }
+            return ret;
+        });
+    }
+
+    private async doOnDirty(): Promise<void> {
+        this.dirty = true;
+        if (this.collectionConfig.autoFlushStrategy == EBAutoFlushStrategy.ALWAYS_AUTO_FLUSH) {
+            await this.flush();
+        }
+    }
+
+    public async flush(): Promise<void> {
+        return this.runProtected(async () => {
+            if (this.loaded) {
+                switch (this.collectionConfig.memStorageStrategy) {
+                    case EBMemStorageStrategy.MAP:
+                        await this.database.io.writeMapData(this.collectionID, this.mapData);
+                        break;
+                    default:
+                    case EBMemStorageStrategy.ARRAY:
+                        await this.database.io.writeArrayData(this.collectionID, this.arrayData);
+                        break;
+                }
+                this.dirty = false;
+            }
+        });
+    }
+
+    public async reload(): Promise<void> {
+        return this.runProtected(async () => {
+            switch (this.collectionConfig.memStorageStrategy) {
+                case EBMemStorageStrategy.MAP:
+                    this.mapData = await this.database.io.loadMapData<T>(this.collectionID);
+                    break;
+                default:
+                case EBMemStorageStrategy.ARRAY:
+                    this.arrayData = await this.database.io.loadArrayData<T>(this.collectionID);
+                    break;
+            }
+            this.loaded = true;
+            this.dirty = false;
+            this.resolveIndexDefinitions();
+            await this.rebuildIndexRuntime();
+        });
+    }
+
+    public async unload(): Promise<void> {
+        return this.runProtected(async () => {
+            if (this.dirty) {
+                switch (this.collectionConfig.dirtyUnloadStrategy) {
+                    case EBDirtyUnloadStrategy.FLUSH:
+                        await this.flush();
+                        break;
+                    case EBDirtyUnloadStrategy.ERROR:
+                        throw new Error("cannot unload unwritten collection");
+                    default:
+                    case EBDirtyUnloadStrategy.IGNORE:
+                        // do nothing
+                        break;
+
+                }
+            }
+            switch (this.collectionConfig.memStorageStrategy) {
+                case EBMemStorageStrategy.MAP:
+                    this.mapData = new Map();
+                    break;
+                default:
+                case EBMemStorageStrategy.ARRAY:
+                    this.arrayData = [];
+                    break;
+            }
+            this.loaded = false;
+            this.dirty = false;
+        });
+    }
+
 }