@incoqnito.io/bajadab 1.1.0 → 1.2.0

package/README.md

@@ -8,6 +8,7 @@ A lightweight JSON-file-backed database for small, single-machine setups. No ser
 - `MAP` (keyed by id) or `ARRAY` (ordered list) in-memory storage per collection.
 - Configurable behavior for duplicate ids, missing ids, array deletion, auto-flush, and unloading dirty data.
 - Custom id extraction/assignment via a pluggable function registry.
+- Optional secondary indexes for fast lookup by a derived key.
 - Atomic writes (write-to-temp, then rename) and, on POSIX systems, owner-only file permissions (`0600` files, `0700` directories).
 - In-process concurrency safety per collection and per database.
 
@@ -54,6 +55,7 @@ Every collection has an `IBCollectionConfig`. `db.config.defaultCollectionConfig
 | `autoFlushStrategy` | `ALWAYS_AUTO_FLUSH` (default) / `NO_AUTO_FLUSH` | Whether every mutation is written to disk immediately, or only on an explicit `flush()`. |
 | `dirtyUnloadStrategy` | `FLUSH` (default) / `ERROR` / `IGNORE` | What `unload()` does with unsaved changes. |
 | `getID` / `setID` | registry key (optional) | Custom id accessors — see below. Omit to use the default, which reads/writes a plain `.id` property. |
+| `indexes` | `{ [indexKey: string]: registry key }` (optional) | Secondary indexes — see below. Omit for no indexes. |
 
 ## Custom ids
 
@@ -82,6 +84,34 @@ const users = await db.createCollection("users", {
 
 A `getID`/`setID` key that isn't found in the registry causes `add()` to reject, rather than silently falling back to the default.
 
+## Indexes
+
+A collection can declare secondary indexes for fast lookup by a derived key. Like `getID`/`setID`, the extractor function lives in the registry — only its key is part of the (persisted) collection config:
+
+```ts
+class Registry implements IBFunctionRegistry {
+    private fns = new Map<string, Function>([
+        ["byEmail", async (v: any) => v.email],
+    ]);
+    fetch(key?: string) {
+        return key ? this.fns.get(key) : undefined;
+    }
+}
+
+const db = await getDatabase("./data", new Registry());
+const users = await db.createCollection("users", {
+    ...db.config.defaultCollectionConfig,
+    indexes: { email: "byEmail" },
+});
+
+await users.add({ email: "a@example.com", name: "A" });
+const matches = await users.getByIndex("email", "a@example.com");
+```
+
+An extractor returns a `string`, a `string[]` for a multi-valued index, or `undefined` to leave a value out of the index. `getByIndex()` rejects if `indexKey` isn't configured. A configured index whose registry key isn't found rejects on the collection's first use after it's (re)loaded — `get()`, `add()`, whatever comes first — not just the mutations that actually need indexing; that's a wider blast radius than `getID`/`setID`, which only ever fail inside `add()`.
+
+The index is maintained incrementally: `add()`/`update()`/`delete()` update only the affected buckets, not the whole index. A full rebuild only happens once, right after `reload()` loads fresh data from disk.
+
 ## Persistence and file layout
 
 Each database directory contains:

package/dist/base.d.ts

@@ -48,6 +48,8 @@ export declare enum EBDirtyUnloadStrategy {
 export type TGetIDFunction = (v: any) => Promise<string | undefined>;
 /** Assigns an id onto a value. */
 export type TSetIDFunction = (v: any, id: string) => Promise<void>;
+/** Extracts an index key (or several, for a multi-valued index) from a value; `undefined` means "not indexed". */
+export type TIndexKeyFunction<T = any> = (v: T) => Promise<string | string[] | undefined>;
 /** Per-collection behavior. Passed to {@link IBDatabase.createCollection}/{@link IBDatabase.getCollection}. */
 export interface IBCollectionConfig {
     autoIDStrategy: EBAutoIDStrategy;
@@ -60,6 +62,10 @@ export interface IBCollectionConfig {
     getID?: string;
     /** Registry key for a custom {@link TSetIDFunction}. Omit to use the default (sets `.id`). */
     setID?: string;
+    /** Index name → registry key of a {@link TIndexKeyFunction} for that index. Omit for no indexes. */
+    indexes?: {
+        [indexKey: string]: string;
+    };
 }
 /** Database-wide configuration. */
 export interface IBDatabaseConfig {
@@ -78,6 +84,33 @@ export interface IBDatabase {
     /** Deletes a collection and invalidates any handle still held on it. Rejects if it doesn't exist. */
     dropCollection(collectionName: string): Promise<void>;
 }
+/** What kind of collection mutation a handler subscribes to via {@link IBCollection.on}. */
+export declare enum EBModificationType {
+    /** A value was stored by `add()` (including an id-based overwrite). */
+    ADD = "ADD",
+    /** A value was replaced by `update()`. */
+    UPDATE = "UPDATE",
+    /** A value was removed by `delete()`. */
+    DELETE = "DELETE"
+}
+/** Handler for {@link EBModificationType.ADD}: called with each value stored by `add()`. */
+export type TAddHandler<T> = (added: T) => void | Promise<void>;
+/** Handler for {@link EBModificationType.UPDATE}: called with the previous and the replacement value. */
+export type TUpdateHandler<T> = (previous: T, updated: T) => void | Promise<void>;
+/** Handler for {@link EBModificationType.DELETE}: called with each value removed by `delete()`. */
+export type TDeleteHandler<T> = (deleted: T) => void | Promise<void>;
+/** Handle returned by {@link IBCollection.on}. */
+export interface IHandlerRegistration {
+    /** Unsubscribes the handler. Idempotent. */
+    cancel(): Promise<void>;
+}
+/** Whether a handler registered via {@link IBCollection.on} blocks the call that triggered it. */
+export declare enum EBHandlerMode {
+    /** The triggering `add()`/`update()`/`delete()` call doesn't resolve until this handler has run. Default. */
+    SYNC = "SYNC",
+    /** The handler runs without the triggering call waiting for it. */
+    ASYNC = "ASYNC"
+}
 /** A single collection of values of type `T`. Obtain one via {@link IBDatabase}. */
 export interface IBCollection<T = any> {
     readonly database: IBDatabase;
@@ -96,6 +129,16 @@ export interface IBCollection<T = any> {
     add(...v: T[]): Promise<number>;
     /** Replaces each item matching `predicate` with the (defined) result of `updater`. Returns the number updated. */
     update(predicate: (v: T) => boolean, updater: (v: T) => Promise<T>): Promise<number>;
+    /** Number of items, or only those matching `predicate`. */
+    count(predicate?: (v: T) => boolean): Promise<number>;
+    /** Items whose {@link IBCollectionConfig.indexes}-configured `indexKey` extracts `key`. Always a defensive copy. Throws if `indexKey` isn't configured. */
+    getByIndex(indexKey: string, key: string): Promise<T[]>;
+    /** Subscribes to `add()` mutations. Fires after the value is stored, including any auto-flush to disk. `mode` defaults to {@link EBHandlerMode.SYNC}. A handler's own errors never fail the triggering call; they're logged only. */
+    on(modificationType: EBModificationType.ADD, handler: TAddHandler<T>, mode?: EBHandlerMode): Promise<IHandlerRegistration>;
+    /** Subscribes to `update()` mutations. Fires after the value is replaced, including any auto-flush to disk. `mode` defaults to {@link EBHandlerMode.SYNC}. A handler's own errors never fail the triggering call; they're logged only. */
+    on(modificationType: EBModificationType.UPDATE, handler: TUpdateHandler<T>, mode?: EBHandlerMode): Promise<IHandlerRegistration>;
+    /** Subscribes to `delete()` mutations. Fires after the value is removed, including any auto-flush to disk. `mode` defaults to {@link EBHandlerMode.SYNC}. A handler's own errors never fail the triggering call; they're logged only. */
+    on(modificationType: EBModificationType.DELETE, handler: TDeleteHandler<T>, mode?: EBHandlerMode): Promise<IHandlerRegistration>;
     /** Writes the current in-memory state to disk. */
     flush(): Promise<void>;
     /** Discards in-memory state and re-reads it from disk. */

package/dist/base.js

@@ -51,4 +51,22 @@ export var EBDirtyUnloadStrategy;
     /** Discard the unsaved changes silently. */
     EBDirtyUnloadStrategy["IGNORE"] = "IGNORE";
 })(EBDirtyUnloadStrategy || (EBDirtyUnloadStrategy = {}));
+/** What kind of collection mutation a handler subscribes to via {@link IBCollection.on}. */
+export var EBModificationType;
+(function (EBModificationType) {
+    /** A value was stored by `add()` (including an id-based overwrite). */
+    EBModificationType["ADD"] = "ADD";
+    /** A value was replaced by `update()`. */
+    EBModificationType["UPDATE"] = "UPDATE";
+    /** A value was removed by `delete()`. */
+    EBModificationType["DELETE"] = "DELETE";
+})(EBModificationType || (EBModificationType = {}));
+/** Whether a handler registered via {@link IBCollection.on} blocks the call that triggered it. */
+export var EBHandlerMode;
+(function (EBHandlerMode) {
+    /** The triggering `add()`/`update()`/`delete()` call doesn't resolve until this handler has run. Default. */
+    EBHandlerMode["SYNC"] = "SYNC";
+    /** The handler runs without the triggering call waiting for it. */
+    EBHandlerMode["ASYNC"] = "ASYNC";
+})(EBHandlerMode || (EBHandlerMode = {}));
 //# sourceMappingURL=base.js.map

package/dist/base.js.map

@@ -1 +1 @@
-{"version":3,"file":"base.js","sourceRoot":"","sources":["../src/main/base.ts"],"names":[],"mappings":"AAAA,+DAA+D;AAC/D,MAAM,CAAN,IAAY,gBAKX;AALD,WAAY,gBAAgB;IACxB,mCAAmC;IACnC,6CAAyB,CAAA;IACzB,mEAAmE;IACnE,iDAA6B,CAAA;AACjC,CAAC,EALW,gBAAgB,KAAhB,gBAAgB,QAK3B;AAED,uDAAuD;AACvD,MAAM,CAAN,IAAY,oBAMX;AAND,WAAY,oBAAoB;IAC5B,oEAAoE;IACpE,mCAAW,CAAA;IACX,+BAA+B;IAC/B,gDAAgD;IAChD,uCAAe,CAAA;AACnB,CAAC,EANW,oBAAoB,KAApB,oBAAoB,QAM/B;AAED,iFAAiF;AACjF,MAAM,CAAN,IAAY,qBAOX;AAPD,WAAY,qBAAqB;IAC7B,kCAAkC;IAClC,gDAAuB,CAAA;IACvB,6BAA6B;IAC7B,wCAAe,CAAA;IACf,iDAAiD;IACjD,0CAAiB,CAAA;AACrB,CAAC,EAPW,qBAAqB,KAArB,qBAAqB,QAOhC;AAED,wFAAwF;AACxF,MAAM,CAAN,IAAY,uBAKX;AALD,WAAY,uBAAuB;IAC/B,wCAAwC;IACxC,kDAAuB,CAAA;IACvB,2CAA2C;IAC3C,gDAAqB,CAAA;AACzB,CAAC,EALW,uBAAuB,KAAvB,uBAAuB,QAKlC;AAED,6DAA6D;AAC7D,MAAM,CAAN,IAAY,mBAKX;AALD,WAAY,mBAAmB;IAC3B,2DAA2D;IAC3D,sDAA+B,CAAA;IAC/B,2DAA2D;IAC3D,8DAAuC,CAAA;AAC3C,CAAC,EALW,mBAAmB,KAAnB,mBAAmB,QAK9B;AAED,kEAAkE;AAClE,MAAM,CAAN,IAAY,qBAOX;AAPD,WAAY,qBAAqB;IAC7B,yBAAyB;IACzB,wCAAe,CAAA;IACf,gCAAgC;IAChC,wCAAe,CAAA;IACf,4CAA4C;IAC5C,0CAAiB,CAAA;AACrB,CAAC,EAPW,qBAAqB,KAArB,qBAAqB,QAOhC"}
+{"version":3,"file":"base.js","sourceRoot":"","sources":["../src/main/base.ts"],"names":[],"mappings":"AAAA,+DAA+D;AAC/D,MAAM,CAAN,IAAY,gBAKX;AALD,WAAY,gBAAgB;IACxB,mCAAmC;IACnC,6CAAyB,CAAA;IACzB,mEAAmE;IACnE,iDAA6B,CAAA;AACjC,CAAC,EALW,gBAAgB,KAAhB,gBAAgB,QAK3B;AAED,uDAAuD;AACvD,MAAM,CAAN,IAAY,oBAMX;AAND,WAAY,oBAAoB;IAC5B,oEAAoE;IACpE,mCAAW,CAAA;IACX,+BAA+B;IAC/B,gDAAgD;IAChD,uCAAe,CAAA;AACnB,CAAC,EANW,oBAAoB,KAApB,oBAAoB,QAM/B;AAED,iFAAiF;AACjF,MAAM,CAAN,IAAY,qBAOX;AAPD,WAAY,qBAAqB;IAC7B,kCAAkC;IAClC,gDAAuB,CAAA;IACvB,6BAA6B;IAC7B,wCAAe,CAAA;IACf,iDAAiD;IACjD,0CAAiB,CAAA;AACrB,CAAC,EAPW,qBAAqB,KAArB,qBAAqB,QAOhC;AAED,wFAAwF;AACxF,MAAM,CAAN,IAAY,uBAKX;AALD,WAAY,uBAAuB;IAC/B,wCAAwC;IACxC,kDAAuB,CAAA;IACvB,2CAA2C;IAC3C,gDAAqB,CAAA;AACzB,CAAC,EALW,uBAAuB,KAAvB,uBAAuB,QAKlC;AAED,6DAA6D;AAC7D,MAAM,CAAN,IAAY,mBAKX;AALD,WAAY,mBAAmB;IAC3B,2DAA2D;IAC3D,sDAA+B,CAAA;IAC/B,2DAA2D;IAC3D,8DAAuC,CAAA;AAC3C,CAAC,EALW,mBAAmB,KAAnB,mBAAmB,QAK9B;AAED,kEAAkE;AAClE,MAAM,CAAN,IAAY,qBAOX;AAPD,WAAY,qBAAqB;IAC7B,yBAAyB;IACzB,wCAAe,CAAA;IACf,gCAAgC;IAChC,wCAAe,CAAA;IACf,4CAA4C;IAC5C,0CAAiB,CAAA;AACrB,CAAC,EAPW,qBAAqB,KAArB,qBAAqB,QAOhC;AA8CD,4FAA4F;AAC5F,MAAM,CAAN,IAAY,kBAOX;AAPD,WAAY,kBAAkB;IAC1B,uEAAuE;IACvE,iCAAW,CAAA;IACX,0CAA0C;IAC1C,uCAAiB,CAAA;IACjB,yCAAyC;IACzC,uCAAiB,CAAA;AACrB,CAAC,EAPW,kBAAkB,KAAlB,kBAAkB,QAO7B;AAeD,kGAAkG;AAClG,MAAM,CAAN,IAAY,aAKX;AALD,WAAY,aAAa;IACrB,6GAA6G;IAC7G,8BAAa,CAAA;IACb,mEAAmE;IACnE,gCAAe,CAAA;AACnB,CAAC,EALW,aAAa,KAAb,aAAa,QAKxB"}

package/dist/basics.d.ts

@@ -1,5 +1,5 @@
 import type { IODatabase } from "./fileio.js";
-import { IBCollection, IBCollectionConfig, IBDatabase, IBDatabaseConfig, IBFunctionRegistry, TGetIDFunction, TSetIDFunction } from "./base.js";
+import { EBHandlerMode, EBModificationType, IBCollection, IBCollectionConfig, IBDatabase, IBDatabaseConfig, IBFunctionRegistry, IHandlerRegistration, TAddHandler, TDeleteHandler, TGetIDFunction, TSetIDFunction, TUpdateHandler } from "./base.js";
 /** Default {@link TGetIDFunction}: reads `v.id`, `undefined` if absent. */
 export declare const DEFAULT_GET_ID: TGetIDFunction;
 /** Default {@link TSetIDFunction}: assigns to `v.id`. */
@@ -30,16 +30,40 @@ export declare class BCollection<T = any> implements IBCollection<T> {
     private loaded;
     private dirty;
     private dropped;
+    private addHandlers;
+    private updateHandlers;
+    private deleteHandlers;
+    private indexDefinitions;
+    private indexRuntime;
     constructor(database: BDatabase, collectionID: string, collectionName: string, collectionConfig: IBCollectionConfig);
     private checkLoaded;
+    private runExclusive;
     private runProtected;
     /**
-     * call this only within the proteing mutex
+     * call this only within the protecting mutex
      */
     markDropped(): void;
     get(predicate?: (v: T) => boolean): Promise<T[]>;
+    count(predicate?: (v: T) => boolean): Promise<number>;
+    getByIndex(indexKey: string, key: string): Promise<T[]>;
+    private invokeHandler;
+    private fireAdd;
+    private fireUpdate;
+    private fireDelete;
+    on(modificationType: EBModificationType.ADD, handler: TAddHandler<T>, mode?: EBHandlerMode): Promise<IHandlerRegistration>;
+    on(modificationType: EBModificationType.UPDATE, handler: TUpdateHandler<T>, mode?: EBHandlerMode): Promise<IHandlerRegistration>;
+    on(modificationType: EBModificationType.DELETE, handler: TDeleteHandler<T>, mode?: EBHandlerMode): Promise<IHandlerRegistration>;
     delete(predicate: (v: T) => boolean): Promise<number>;
     has(predicate: (v: T) => boolean): Promise<boolean>;
+    private resolveIndexDefinitions;
+    private indexKeysFor;
+    private addToIndexes;
+    private removeFromIndexes;
+    /**
+     * full rebuild from currently loaded data; only needed right after (re)load, since every
+     * value then is a freshly deserialized object the incremental add/update/delete hooks never saw
+     */
+    private rebuildIndexRuntime;
     private getID;
     add(...vs: T[]): Promise<number>;
     update(predicate: (v: T) => boolean, updater: (v: T) => Promise<T>): Promise<number>;

package/dist/basics.js

@@ -1,6 +1,7 @@
+import logger from "winston";
 import { uuidv7, } from "uuidv7";
 import { runMultiReentrantProtected, runReentrantProtected } from "@incoqnito.io/ts-iq-core";
-import { EBArrayDeletionStrategy, EBAutoFlushStrategy, EBAutoIDStrategy, EBDirtyUnloadStrategy, EBDuplicateIDStrategy, EBMemStorageStrategy, } from "./base.js";
+import { EBArrayDeletionStrategy, EBAutoFlushStrategy, EBAutoIDStrategy, EBDirtyUnloadStrategy, EBDuplicateIDStrategy, EBHandlerMode, EBMemStorageStrategy, EBModificationType, } from "./base.js";
 import { canonicalName } from "./utils.js";
 /** Default {@link TGetIDFunction}: reads `v.id`, `undefined` if absent. */
 export const DEFAULT_GET_ID = (v) => ("id" in v) ? v.id : undefined;
@@ -89,6 +90,11 @@ export class BCollection {
     loaded = false;
     dirty = false;
     dropped = false;
+    addHandlers = [];
+    updateHandlers = [];
+    deleteHandlers = [];
+    indexDefinitions = new Map();
+    indexRuntime = new Map();
     constructor(database, collectionID, collectionName, collectionConfig) {
         this.database = database;
         this.collectionID = collectionID;
@@ -110,8 +116,11 @@ export class BCollection {
             await this.reload();
         }
     }
+    async runExclusive(fn) {
+        return runReentrantProtected(this.collectionName, fn);
+    }
     async runProtected(fn) {
-        return runReentrantProtected(this.collectionName, async () => {
+        return this.runExclusive(async () => {
             if (this.dropped) {
                 throw new Error(`collection "${this.collectionName}" has been dropped`);
             }
@@ -119,7 +128,7 @@ export class BCollection {
         });
     }
     /**
-     * call this only within the proteing mutex
+     * call this only within the protecting mutex
      */
     markDropped() {
         this.dropped = true;
@@ -135,22 +144,117 @@ export class BCollection {
                 default:
                 case EBMemStorageStrategy.ARRAY:
                     return !predicate ?
-                        // MARK this may be a bit slow for bigger collections    
-                        [...this.arrayData] :
+                        this.arrayData.slice() :
                         this.arrayData.filter(predicate);
             }
         });
     }
+    async count(predicate) {
+        return this.runProtected(async () => {
+            await this.checkLoaded();
+            switch (this.collectionConfig.memStorageStrategy) {
+                case EBMemStorageStrategy.MAP:
+                    return !predicate ?
+                        this.mapData.size :
+                        this.mapData.values().filter(predicate).reduce((n) => n + 1, 0);
+                default:
+                case EBMemStorageStrategy.ARRAY:
+                    return !predicate ?
+                        this.arrayData.length :
+                        this.arrayData.filter(predicate).length;
+            }
+        });
+    }
+    async getByIndex(indexKey, key) {
+        return this.runProtected(async () => {
+            await this.checkLoaded();
+            const byKey = this.indexRuntime.get(indexKey);
+            if (!byKey) {
+                throw new Error(`index "${indexKey}" is not configured`);
+            }
+            return byKey.get(key)?.slice() ?? [];
+        });
+    }
+    async invokeHandler(fn, modificationType) {
+        try {
+            await fn();
+        }
+        catch (error) {
+            logger.error(`bajadab: collection "${this.collectionName}" ${modificationType} handler failed`, error);
+        }
+    }
+    async fireAdd(added) {
+        for (const v of added) {
+            for (const { handler, mode } of this.addHandlers) {
+                const call = this.invokeHandler(() => handler(v), EBModificationType.ADD);
+                if (mode === EBHandlerMode.SYNC) {
+                    await call;
+                }
+            }
+        }
+    }
+    async fireUpdate(pairs) {
+        for (const [previous, updated] of pairs) {
+            for (const { handler, mode } of this.updateHandlers) {
+                const call = this.invokeHandler(() => handler(previous, updated), EBModificationType.UPDATE);
+                if (mode === EBHandlerMode.SYNC) {
+                    await call;
+                }
+            }
+        }
+    }
+    async fireDelete(deleted) {
+        for (const v of deleted) {
+            for (const { handler, mode } of this.deleteHandlers) {
+                const call = this.invokeHandler(() => handler(v), EBModificationType.DELETE);
+                if (mode === EBHandlerMode.SYNC) {
+                    await call;
+                }
+            }
+        }
+    }
+    async on(modificationType, handler, mode = EBHandlerMode.SYNC) {
+        return this.runProtected(async () => {
+            switch (modificationType) {
+                case EBModificationType.ADD: {
+                    const h = handler;
+                    this.addHandlers.push({ handler: h, mode });
+                    return { cancel: async () => this.runExclusive(async () => { this.addHandlers = this.addHandlers.filter(x => x.handler !== h); }) };
+                }
+                case EBModificationType.UPDATE: {
+                    const h = handler;
+                    this.updateHandlers.push({ handler: h, mode });
+                    return { cancel: async () => this.runExclusive(async () => { this.updateHandlers = this.updateHandlers.filter(x => x.handler !== h); }) };
+                }
+                case EBModificationType.DELETE: {
+                    const h = handler;
+                    this.deleteHandlers.push({ handler: h, mode });
+                    return { cancel: async () => this.runExclusive(async () => { this.deleteHandlers = this.deleteHandlers.filter(x => x.handler !== h); }) };
+                }
+                default:
+                    throw new Error(`unknown modification type ${modificationType}`);
+            }
+        });
+    }
     async delete(predicate) {
         return this.runProtected(async () => {
             await this.checkLoaded();
             let ret = 0;
+            const trackDeletes = this.deleteHandlers.length > 0;
+            const indexDeletes = this.indexDefinitions.size > 0;
+            const deleted = [];
             switch (this.collectionConfig.memStorageStrategy) {
                 case EBMemStorageStrategy.MAP:
                     for (const [key, value] of this.mapData) {
                         if (predicate(value)) {
                             ret++;
+                            if (indexDeletes) {
+                                await this.removeFromIndexes(value);
+                            }
                             this.mapData.delete(key);
+                            if (trackDeletes) {
+                                deleted.push(value);
+                            }
                         }
                     }
                     break;
@@ -167,6 +271,12 @@ export class BCollection {
                                 }
                                 else {
                                     ret++;
+                                    if (indexDeletes) {
+                                        await this.removeFromIndexes(item);
+                                    }
+                                    if (trackDeletes) {
+                                        deleted.push(item);
+                                    }
                                 }
                             }
                             this.arrayData.length = writeIndex;
@@ -174,6 +284,17 @@ export class BCollection {
                         default:
                         case EBArrayDeletionStrategy.NEW_ARRAY:
                             const ol = this.arrayData.length;
+                            if (trackDeletes || indexDeletes) {
+                                const removed = this.arrayData.filter(predicate);
+                                if (trackDeletes) {
+                                    deleted.push(...removed);
+                                }
+                                if (indexDeletes) {
+                                    for (const v of removed) {
+                                        await this.removeFromIndexes(v);
+                                    }
+                                }
+                            }
                             this.arrayData = this.arrayData.filter(v => !predicate(v));
                             ret += (ol - this.arrayData.length);
                             break;
@@ -183,6 +304,9 @@ export class BCollection {
             if (ret > 0) {
                 await this.doOnDirty();
             }
+            if (deleted.length > 0) {
+                await this.fireDelete(deleted);
+            }
             return ret;
         });
     }
@@ -198,6 +322,78 @@ export class BCollection {
             }
         });
     }
+    resolveIndexDefinitions() {
+        this.indexDefinitions = new Map();
+        if (!this.collectionConfig.indexes) {
+            return;
+        }
+        for (const [indexKey, registryKey] of Object.entries(this.collectionConfig.indexes)) {
+            const extractor = this.database.functionRegistry?.fetch(registryKey);
+            if (!extractor) {
+                throw new Error(`index key function "${registryKey}" is not registered`);
+            }
+            this.indexDefinitions.set(indexKey, extractor);
+        }
+    }
+    async indexKeysFor(v) {
+        const result = [];
+        for (const [indexKey, extractor] of this.indexDefinitions) {
+            const extracted = await extractor(v);
+            if (extracted === undefined) {
+                continue;
+            }
+            result.push([indexKey, Array.isArray(extracted) ? extracted : [extracted]]);
+        }
+        return result;
+    }
+    async addToIndexes(v) {
+        for (const [indexKey, keys] of await this.indexKeysFor(v)) {
+            const byKey = this.indexRuntime.get(indexKey);
+            for (const key of keys) {
+                let bucket = byKey.get(key);
+                if (!bucket) {
+                    bucket = [];
+                    byKey.set(key, bucket);
+                }
+                bucket.push(v);
+            }
+        }
+    }
+    async removeFromIndexes(v) {
+        for (const [indexKey, keys] of await this.indexKeysFor(v)) {
+            const byKey = this.indexRuntime.get(indexKey);
+            for (const key of keys) {
+                const bucket = byKey.get(key);
+                if (!bucket) {
+                    continue;
+                }
+                const at = bucket.indexOf(v);
+                if (at !== -1) {
+                    bucket.splice(at, 1);
+                }
+                if (bucket.length === 0) {
+                    byKey.delete(key);
+                }
+            }
+        }
+    }
+    /**
+     * full rebuild from currently loaded data; only needed right after (re)load, since every
+     * value then is a freshly deserialized object the incremental add/update/delete hooks never saw
+     */
+    async rebuildIndexRuntime() {
+        // every configured index gets an entry up front, even if empty, so getByIndex()/
+        // addToIndexes()/removeFromIndexes() can assume it's there instead of checking twice
+        this.indexRuntime = new Map(Array.from(this.indexDefinitions.keys(), (indexKey) => [indexKey, new Map()]));
+        if (this.indexDefinitions.size === 0) {
+            return;
+        }
+        const values = this.collectionConfig.memStorageStrategy === EBMemStorageStrategy.MAP ?
+            [...this.mapData.values()] : this.arrayData;
+        for (const v of values) {
+            await this.addToIndexes(v);
+        }
+    }
     async getID(v) {
         let getter;
         if (this.collectionConfig.getID) {
@@ -241,6 +437,9 @@ export class BCollection {
             switch (this.collectionConfig.memStorageStrategy) {
                 case EBMemStorageStrategy.MAP:
                     let ret = 0;
+                    const trackAddsMap = this.addHandlers.length > 0;
+                    const indexAddsMap = this.indexDefinitions.size > 0;
+                    const addedMap = [];
                     for (const v of vs) {
                         const id = await this.getID(v);
                         if (this.mapData.has(id)) {
@@ -254,21 +453,42 @@ export class BCollection {
                                 case EBDuplicateIDStrategy.OVERWRITE:
                                     // replace the new value
                                     ret++;
+                                    if (indexAddsMap) {
+                                        await this.removeFromIndexes(this.mapData.get(id));
+                                    }
                                     this.mapData.set(id, v);
+                                    if (indexAddsMap) {
+                                        await this.addToIndexes(v);
+                                    }
+                                    if (trackAddsMap) {
+                                        addedMap.push(v);
+                                    }
                                     break;
                             }
                         }
                         else {
                             ret++;
                             this.mapData.set(id, v);
+                            if (indexAddsMap) {
+                                await this.addToIndexes(v);
+                            }
+                            if (trackAddsMap) {
+                                addedMap.push(v);
+                            }
                         }
                     }
                     if (ret > 0) {
                         await this.doOnDirty();
                     }
+                    if (addedMap.length > 0) {
+                        await this.fireAdd(addedMap);
+                    }
                     return ret;
                 default:
                 case EBMemStorageStrategy.ARRAY:
+                    const trackAddsArray = this.addHandlers.length > 0;
+                    const indexAddsArray = this.indexDefinitions.size > 0;
+                    const addedArray = [];
                     switch (this.collectionConfig.duplicateIDStrategy) {
                         case EBDuplicateIDStrategy.ERROR:
                             const haveIDs = new Set();
@@ -282,6 +502,12 @@ export class BCollection {
                                 }
                                 haveIDs.add(id);
                                 this.arrayData.push(v);
+                                if (indexAddsArray) {
+                                    await this.addToIndexes(v);
+                                }
+                                if (trackAddsArray) {
+                                    addedArray.push(v);
+                                }
                             }
                             break;
                         case EBDuplicateIDStrategy.IGNORE:
@@ -294,6 +520,12 @@ export class BCollection {
                                 if (!seenIDs.has(id)) {
                                     seenIDs.add(id);
                                     this.arrayData.push(v);
+                                    if (indexAddsArray) {
+                                        await this.addToIndexes(v);
+                                    }
+                                    if (trackAddsArray) {
+                                        addedArray.push(v);
+                                    }
                                 }
                             }
                             break;
@@ -307,6 +539,9 @@ export class BCollection {
                                 const id = await this.getID(v);
                                 const existingIndex = indexMap.get(id);
                                 if (existingIndex !== undefined) {
+                                    if (indexAddsArray) {
+                                        await this.removeFromIndexes(this.arrayData[existingIndex]);
+                                    }
                                     this.arrayData[existingIndex] = v;
                                 }
                                 else {
@@ -314,12 +549,21 @@ export class BCollection {
                                     indexMap.set(id, this.arrayData.length);
                                     this.arrayData.push(v);
                                 }
+                                if (indexAddsArray) {
+                                    await this.addToIndexes(v);
+                                }
+                                if (trackAddsArray) {
+                                    addedArray.push(v);
+                                }
                             }
                             break;
                     }
                     if (vs.length > 0) {
                         await this.doOnDirty();
                     }
+                    if (addedArray.length > 0) {
+                        await this.fireAdd(addedArray);
+                    }
                     return vs.length;
             }
         });
@@ -328,6 +572,9 @@ export class BCollection {
         return this.runProtected(async () => {
             await this.checkLoaded();
             let ret = 0;
+            const trackUpdates = this.updateHandlers.length > 0;
+            const indexUpdates = this.indexDefinitions.size > 0;
+            const updated = [];
             switch (this.collectionConfig.memStorageStrategy) {
                 case EBMemStorageStrategy.MAP:
                     for (const [key, value] of this.mapData) {
@@ -335,7 +582,16 @@ export class BCollection {
                             const replacement = await updater(value);
                             if (replacement !== undefined) {
                                 ret++;
+                                if (indexUpdates) {
+                                    await this.removeFromIndexes(value);
+                                }
                                 this.mapData.set(key, replacement);
+                                if (indexUpdates) {
+                                    await this.addToIndexes(replacement);
+                                }
+                                if (trackUpdates) {
+                                    updated.push([value, replacement]);
+                                }
                             }
                             else {
                                 throw new Error("updater gave undefined");
@@ -347,10 +603,20 @@ export class BCollection {
                 case EBMemStorageStrategy.ARRAY:
                     for (let i = 0; i < this.arrayData.length; i++) {
                         if (predicate(this.arrayData[i])) {
-                            const replacement = await updater(this.arrayData[i]);
+                            const previous = this.arrayData[i];
+                            const replacement = await updater(previous);
                             if (replacement !== undefined) {
                                 ret++;
+                                if (indexUpdates) {
+                                    await this.removeFromIndexes(previous);
+                                }
                                 this.arrayData[i] = replacement;
+                                if (indexUpdates) {
+                                    await this.addToIndexes(replacement);
+                                }
+                                if (trackUpdates) {
+                                    updated.push([previous, replacement]);
+                                }
                             }
                             else {
                                 throw new Error("updater gave undefined");
@@ -362,6 +628,9 @@ export class BCollection {
             if (ret > 0) {
                 await this.doOnDirty();
             }
+            if (updated.length > 0) {
+                await this.fireUpdate(updated);
+            }
             return ret;
         });
     }
@@ -400,6 +669,8 @@ export class BCollection {
             }
             this.loaded = true;
             this.dirty = false;
+            this.resolveIndexDefinitions();
+            await this.rebuildIndexRuntime();
         });
     }
     async unload() {