edinburgh 0.1.3 → 0.4.1

package/src/models.ts

@@ -1,8 +1,35 @@
-import { DatabaseError } from "olmdb";
-import * as olmdb from "olmdb";
+import { DatabaseError } from "olmdb/lowlevel";
+import { AsyncLocalStorage } from "node:async_hooks";
 import { TypeWrapper, identifier } from "./types.js";
-import { BaseIndex, TARGET_SYMBOL, PrimaryIndex } from "./indexes.js";
-import { assert, addErrorPath, logLevel } from "./utils.js";
+import { scheduleInit } from "./edinburgh.js";
+
+export const txnStorage = new AsyncLocalStorage<Transaction>();
+
+
+const PREVENT_PERSIST_DESCRIPTOR = {
+    get() {
+        throw new DatabaseError("Operation not allowed after preventPersist()", "NO_PERSIST");
+    },
+};
+
+/**
+ * Returns the current transaction from AsyncLocalStorage.
+ * Throws if called outside a transact() callback.
+ * @internal
+ */
+export function currentTxn(): Transaction {
+    const txn = txnStorage.getStore();
+    if (!txn) throw new DatabaseError("No active transaction. Operations must be performed within a transact() callback.", 'NO_TRANSACTION');
+    return txn;
+}
+
+export interface Transaction {
+    id: number;
+    instances: Set<Model<unknown>>;
+    instancesByPk: Map<number, Model<unknown>>;
+}
+import { BaseIndex as BaseIndex, PrimaryIndex, IndexRangeIterator } from "./indexes.js";
+import { addErrorPath, logLevel, assert, dbGet, hashBytes } from "./utils.js";
 
 /**
  * Configuration interface for model fields.
@@ -44,63 +71,20 @@ export function field<T>(type: TypeWrapper<T>, options: Partial<FieldConfig<T>>
 }
 
 // Model registration and initialization
-let uninitializedModels = new Set<typeof Model<unknown>>();
 export const modelRegistry: Record<string, typeof Model> = {};
 
-export function resetModelCaches() {
-    for(const model of Object.values(modelRegistry)) {
-        for(const index of model._indexes || []) {
-            index._cachedIndexId = undefined;
-        }
-    }
-}
-
 function isObjectEmpty(obj: object) {
-    for (let key in obj) {
-        if (obj.hasOwnProperty(key)) return false;
+    for (let _ of Object.keys(obj)) {
+        return false;
     }
     return true;
 }
 
-type OnSaveType = (model: InstanceType<typeof Model>, newKey: Uint8Array | undefined, oldKey: Uint8Array | undefined) => void;
-let onSave: OnSaveType | undefined;
-/**
- * Set a callback function to be called after a model is saved and committed.
- *
- * @param callback The callback function to set. As arguments, it receives the model instance, the new key (undefined in case of a delete), and the old key (undefined in case of a create).
- */
-export function setOnSaveCallback(callback: OnSaveType | undefined) {
-    onSave = callback;
-}
-const onSaveQueue: [InstanceType<typeof Model>, Uint8Array | undefined, Uint8Array | undefined][] = [];
-function onSaveRevert() {
-    onSaveQueue.length = 0;
-}
-function onSaveCommit() {
-    if (onSave) {
-        for(let arr of onSaveQueue) {
-            onSave(...arr);
-        }
-    }
-    onSaveQueue.length = 0;
-}
-function queueOnSave(arr: [InstanceType<typeof Model>, Uint8Array | undefined, Uint8Array | undefined]) {
-    if (onSave) {
-        if (!onSaveQueue.length) {
-            olmdb.onCommit(onSaveCommit);
-            olmdb.onRevert(onSaveRevert);
-        }
-        onSaveQueue.push(arr);
-    }
-}
+export type Change = Record<any, any> | "created" | "deleted";
 
 /**
  * Register a model class with the Edinburgh ORM system.
  * 
- * This decorator function transforms the model class to use a proxy-based constructor
- * that enables change tracking and automatic field initialization. It also extracts
- * field metadata and sets up default values on the prototype.
- * 
  * @template T - The model class type.
  * @param MyModel - The model class to register.
  * @returns The enhanced model class with ORM capabilities.
@@ -120,24 +104,22 @@ export function registerModel<T extends typeof Model<unknown>>(MyModel: T): T {
 
     // Copy own static methods/properties
     for(const name of Object.getOwnPropertyNames(MyModel)) {
-        if (name !== 'length' && name !== 'prototype' && name !== 'name' && name !== 'mock') {
+        if (name !== 'length' && name !== 'prototype' && name !== 'name' && name !== 'mock' && name !== 'override') {
             (MockModel as any)[name] = (MyModel as any)[name];
         }
     }
-
-    // Initialize an empty `fields` object, and set it on both constructors, as well as on the prototype.
-    MockModel.fields = MockModel.prototype._fields = {};
-    MockModel.tableName ||= MyModel.name; // Set the table name to the class name if not already set
+    MockModel.tableName ||= MyModel.name;
 
     // Register the constructor by name
-    if (MockModel.tableName in modelRegistry) throw new DatabaseError(`Model with table name '${MockModel.tableName}' already registered`, 'INIT_ERROR');
+    if (MockModel.tableName in modelRegistry) {
+        if (!(MyModel as any).override) {
+            throw new DatabaseError(`Model with table name '${MockModel.tableName}' already registered`, 'INIT_ERROR');
+        }
+        delete modelRegistry[MockModel.tableName];
+    }
     modelRegistry[MockModel.tableName] = MockModel;
 
-    // Attempt to instantiate the class and gather field metadata
-    uninitializedModels.add(MyModel);
-    initModels();
-
-    return MockModel;
+    return MockModel;   
 }
 
 export function getMockModel<T extends typeof Model<unknown>>(OrgModel: T): T {
@@ -145,17 +127,14 @@ export function getMockModel<T extends typeof Model<unknown>>(OrgModel: T): T {
     if (AnyOrgModel._isMock) return OrgModel;
     if (AnyOrgModel._mock) return AnyOrgModel._mock;
 
-    const MockModel = function (this: any, initial?: Record<string,any>) {
-        if (uninitializedModels.has(this.constructor)) {
-            throw new DatabaseError("Cannot instantiate while linked models haven't been registered yet", 'INIT_ERROR');
-        }
-        if (initial && !isObjectEmpty(initial)) {
+    const MockModel = function(this: any, initial?: Record<string,any> | undefined, txn: Transaction = currentTxn()) {
+        // This constructor should only be called when the user does 'new Model'. We'll bypass this when
+        // loading objects. Add to 'instances', so the object will be saved.
+        this._txn = txn;
+        txn.instances.add(this);
+        if (initial) {
             Object.assign(this, initial);
-            const modifiedInstances = olmdb.getTransactionData(MODIFIED_INSTANCES_SYMBOL) as Set<Model<any>>;
-            modifiedInstances.add(this);
         }
-
-        return new Proxy(this, modificationTracker);
     } as any as T;
 
     // We want .constructor to point at our fake constructor function.
@@ -165,85 +144,15 @@ export function getMockModel<T extends typeof Model<unknown>>(OrgModel: T): T {
     Object.setPrototypeOf(MockModel, Object.getPrototypeOf(OrgModel));
     MockModel.prototype = OrgModel.prototype;
     (MockModel as any)._isMock = true;
+    (MockModel as any)._original = OrgModel;
     AnyOrgModel._mock = MockModel;
+    scheduleInit();
     return MockModel;
 }
 
-function initModels() {
-    for(const OrgModel of uninitializedModels) {
-        const MockModel = getMockModel(OrgModel);
-        // Create an instance (the only one to ever exist) of the actual class,
-        // in order to gather field config data. 
-        let instance;
-        try {
-            instance = new (OrgModel as any)(INIT_INSTANCE_SYMBOL);
-        } catch(e) {
-            if (!(e instanceof ReferenceError)) throw e;
-            // ReferenceError: Cannot access 'SomeLinkedClass' before initialization.
-            // We'll try again after the next class has successfully initialized.
-            continue;
-        }
-
-        uninitializedModels.delete(OrgModel);
-
-        // If no primary key exists, create one using 'id' field
-        if (!MockModel._pk) {
-            // If no `id` field exists, add it automatically
-            if (!instance.id) {
-                instance.id = { type: identifier }; 
-            }
-            // @ts-ignore-next-line - `id` is not part of the type, but the user probably shouldn't touch it anyhow
-            new PrimaryIndex(MockModel, ['id']);
-        }
-
-        for (const key in instance) {
-            const value = instance[key] as FieldConfig<unknown>;
-            // Check if this property contains field metadata
-            if (value && value.type instanceof TypeWrapper) {
-                // Set the configuration on the constructor's `fields` property
-                MockModel.fields[key] = value;
-
-                // Set default value on the prototype
-                const defObj = value.default===undefined ? value.type : value;
-                const def = defObj.default;
-                if (typeof def === 'function') {
-                    // The default is a function. We'll define a getter on the property in the model prototype,
-                    // and once it is read, we'll run the function and set the value as a plain old property
-                    // on the instance object.
-                    Object.defineProperty(MockModel.prototype, key, {    
-                        get() {
-                            // This will call set(), which will define the property on the instance.
-                            return (this[key] = def.call(defObj, this));
-                        },
-                        set(val: any) {
-                            Object.defineProperty(this, key, {
-                                value: val,
-                                configurable: true,
-                                writable: true
-                            })
-                        },
-                        configurable: true,    
-                    });
-                } else if (def !== undefined) {
-                    (MockModel.prototype as any)[key] = def;
-                }
-            }
-        }
-
-        if (logLevel >= 1) {
-            console.log(`Registered model ${MockModel.tableName}[${MockModel._pk!._fieldNames.join(',')}] with fields: ${Object.keys(MockModel.fields).join(' ')}`);
-        }
-    }
-}
-
 // Model base class and related symbols/state
 const INIT_INSTANCE_SYMBOL = Symbol();
 
-/** @internal Symbol used to attach modified instances to running transaction */
-export const MODIFIED_INSTANCES_SYMBOL = Symbol('modifiedInstances');
-
-/** @internal Symbol used to access the underlying model from a proxy */
-
 /**
  * Model interface that ensures proper typing for the constructor property.
  * @template SUB - The concrete model subclass.
@@ -259,32 +168,99 @@ export interface Model<SUB> {
  * change tracking, and relationship management. All model classes should extend
  * this base class and be decorated with `@registerModel`.
  *
+ * ### Schema Evolution
+ *
+ * Edinburgh tracks the schema version of each model automatically. When you add, remove, or
+ * change the types of fields, or add/remove indexes, Edinburgh detects the new schema version.
+ *
+ * **Lazy migration:** Changes to non-key field values are migrated lazily, when a row with an
+ * old schema version is read from disk, it is deserialized using the old schema and optionally
+ * transformed by the static `migrate()` function. This happens transparently on every read
+ * and requires no downtime or batch processing.
+ *
+ * **Batch migration (via `npx migrate-edinburgh` or `runMigration()`):** Certain schema changes
+ * require an explicit migration run:
+ * - Adding or removing secondary/unique indexes
+ * - Changing the fields or types of an existing index
+ * - A `migrate()` function that changes values used in secondary index fields
+ *
+ * The batch migration tool populates new indexes, deletes orphaned ones, and updates index
+ * entries whose values were changed by `migrate()`. It does *not* rewrite primary data rows
+ * (lazy migration handles that).
+ *
+ * ### Lifecycle Hooks
+ *
+ * - **`static migrate(record)`**: Called when deserializing rows written with an older schema
+ *   version. Receives a plain record object; mutate it in-place to match the current schema.
+ *   See {@link Model.migrate}.
+ *
+ * - **`preCommit()`**: Called on each modified instance right before the transaction commits.
+ *   Useful for computing derived fields, enforcing cross-field invariants, or creating related
+ *   instances. See {@link Model.preCommit}.
+ *
  * @template SUB - The concrete model subclass (for proper typing).
  * 
  * @example
  * ```typescript
  * ⁣@E.registerModel
  * class User extends E.Model<User> {
- *   static pk = E.index(User, ["id"], "primary");
+ *   static pk = E.primary(User, "id");
  *   
  *   id = E.field(E.identifier);
  *   name = E.field(E.string);
  *   email = E.field(E.string);
  *   
- *   static byEmail = E.index(User, "email", "unique");
+ *   static byEmail = E.unique(User, "email");
  * }
  * ```
  */
+
+
 export abstract class Model<SUB> {
-    /** @internal Primary key index for this model. */
-    static _pk?: PrimaryIndex<any, any>;
-    /** @internal All indexes for this model, the primary key being first. */
-    static _indexes?: BaseIndex<any, any>[];
+    static _primary: PrimaryIndex<any, any>;
+
+    /** @internal All non-primary indexes for this model. */
+    static _secondaries?: BaseIndex<any, readonly (keyof any & string)[]>[];
 
     /** The database table name (defaults to class name). */
     static tableName: string;
+
+    /** When true, registerModel replaces an existing model with the same tableName. */
+    static override?: boolean;
+
     /** Field configuration metadata. */
-    static fields: Record<string, FieldConfig<unknown>>;
+    static fields: Record<string | symbol | number, FieldConfig<unknown>>;
+
+    /**
+     * Optional migration function called when deserializing rows written with an older schema version.
+     * Receives a plain record with all fields (primary key fields + value fields) and should mutate it
+     * in-place to match the current schema.
+     *
+     * This is called both during lazy loading (when a row is read from disk) and during batch
+     * migration (via `runMigration()` / `npx migrate-edinburgh`). The function's source code is hashed
+     * to detect changes. Modifying `migrate()` triggers a new schema version.
+     *
+     * If `migrate()` changes values of fields used in secondary or unique indexes, those indexes
+     * will only be updated when `runMigration()` is run (not during lazy loading).
+     *
+     * @param record - A plain object with all field values from the old schema version.
+     *
+     * @example
+     * ```typescript
+     * ⁣@E.registerModel
+     * class User extends E.Model<User> {
+     *   static pk = E.primary(User, "id");
+     *   id = E.field(E.identifier);
+     *   name = E.field(E.string);
+     *   role = E.field(E.string);  // new field
+     *
+     *   static migrate(record: Record<string, any>) {
+     *     record.role ??= "user";  // default for rows that predate the 'role' field
+     *   }
+     * }
+     * ```
+     */
+    static migrate?(record: Record<string, any>): void;
 
     /*
      * IMPORTANT: We cannot use instance property initializers here, because we will be
@@ -292,68 +268,248 @@ export abstract class Model<SUB> {
      * intentional, as we don't want to run the initializers for the fields.
      */
     
-    /** @internal Field configuration for this instance. */
-    _fields!: Record<string, FieldConfig<unknown>>;
-
     /** 
-     * @internal State tracking for this model instance:
-     * - undefined: new instance, unmodified
-     * - 1: new instance, modified (and in modifiedInstances)
-     * - 2: loaded from disk, unmodified
-     * - 3: persistence disabled
-     * - array: loaded from disk, modified (and in modifiedInstances), array values are original index buffers
+     * @internal
+     * - _oldValues===undefined: New instance, not yet saved.
+     * - _oldValues===null: Instance is to be deleted.
+     * - _oldValues is an object: Loaded (possibly only partial, still lazy) from disk, _oldValues contains (partial) old values
      */
-    _state: undefined | 1 | 2 | 3 | Array<Uint8Array>;
+    _oldValues: Record<string, any> | undefined | null;
+    _primaryKey: Uint8Array | undefined;
+    _primaryKeyHash: number | undefined;
+    _txn!: Transaction;
 
     constructor(initial: Partial<Omit<SUB, "constructor">> = {}) {
         // This constructor will only be called once, from `initModels`. All other instances will
         // be created by the 'fake' constructor. The typing for `initial` *is* important though.
-        if (initial as any !== INIT_INSTANCE_SYMBOL) {
-            throw new DatabaseError("The model needs a @registerModel decorator", 'INIT_ERROR');
-        }
+        if (initial as any === INIT_INSTANCE_SYMBOL) return;
+        throw new DatabaseError("The model needs a @registerModel decorator", 'INIT_ERROR');
     }
 
-    _save() {
-        // For performance, we'll work on the unproxied object, as we know we don't require change tracking for save.
-        const unproxiedModel = ((this as any)[TARGET_SYMBOL] || this) as Model<SUB>;
+    /**
+     * Optional hook called on each modified instance right before the transaction commits.
+     * Runs before data is written to disk, so changes made here are included in the commit.
+     *
+     * Common use cases:
+     * - Computing derived or denormalized fields
+     * - Enforcing cross-field validation rules
+     * - Creating or updating related model instances (newly created instances will also
+     *   have their `preCommit()` called)
+     *
+     * @example
+     * ```typescript
+     * ⁣@E.registerModel
+     * class Post extends E.Model<Post> {
+     *   static pk = E.primary(Post, "id");
+     *   id = E.field(E.identifier);
+     *   title = E.field(E.string);
+     *   slug = E.field(E.string);
+     *
+     *   preCommit() {
+     *     this.slug = this.title.toLowerCase().replace(/\s+/g, "-");
+     *   }
+     * }
+     * ```
+     */
+    preCommit?(): void;
 
-        unproxiedModel.validate(true);
+    static async _delayedInit(cleared?: boolean): Promise<void> {
+        const MockModel = getMockModel(this);
 
-        // Handle unique indexes
-        const indexes = this.constructor._indexes!;
-        const originalKeys = typeof unproxiedModel._state === 'object' ? unproxiedModel._state : undefined;
-        const newPk = indexes[0]._save(unproxiedModel, originalKeys?.[0]);
-        for (let i=1; i<indexes.length; i++) {
-            indexes[i]._save(unproxiedModel, originalKeys?.[i]);
+        if (cleared) {
+            MockModel._primary._indexId = undefined;
+            MockModel._primary._versions.clear();
+            for (const sec of MockModel._secondaries || []) sec._indexId = undefined;
         }
 
-        queueOnSave([this, newPk, originalKeys?.[0]]);
+        if (!MockModel.fields) {
+            // First-time init: gather field configs from a temporary instance of the original class.
+            const OrgModel = (MockModel as any)._original || this;
+            const instance = new (OrgModel as any)(INIT_INSTANCE_SYMBOL);
+
+            // If no primary key exists, create one using 'id' field
+            if (!MockModel._primary) {
+                if (!instance.id) {
+                    instance.id = { type: identifier }; 
+                }
+                // @ts-ignore-next-line - `id` is not part of the type, but the user probably shouldn't touch it anyhow
+                new PrimaryIndex(MockModel, ['id']);
+            }
+
+            MockModel.fields = {};
+            for (const key in instance) {
+                const value = instance[key] as FieldConfig<unknown>;
+                // Check if this property contains field metadata
+                if (value && value.type instanceof TypeWrapper) {
+                    // Set the configuration on the constructor's `fields` property
+                    MockModel.fields[key] = value;
+
+                    // Set default value on the prototype
+                    const defObj = value.default===undefined ? value.type : value;
+                    const def = defObj.default;
+                    if (typeof def === 'function') {
+                        // The default is a function. We'll define a getter on the property in the model prototype,
+                        // and once it is read, we'll run the function and set the value as a plain old property
+                        // on the instance object.
+                        Object.defineProperty(MockModel.prototype, key, {    
+                            get() {
+                                // This will call set(), which will define the property on the instance.
+                                return (this[key] = def.call(defObj, this));
+                            },
+                            set(val: any) {
+                                Object.defineProperty(this, key, {
+                                    value: val,
+                                    configurable: true,
+                                    writable: true,
+                                    enumerable: true,
+                                })
+                            },
+                            configurable: true,    
+                        });
+                    } else if (def !== undefined) {
+                        (MockModel.prototype as any)[key] = def;
+                    }
+                }
+            }
 
-        unproxiedModel._state = 2; // Loaded from disk, unmodified
+            if (logLevel >= 1) {
+                console.log(`[edinburgh] Registered model ${MockModel.tableName} with fields: ${Object.keys(MockModel.fields).join(' ')}`);
+            }
+        }
+
+        // Always run index inits (idempotent, skip if already initialized)
+        await MockModel._primary._delayedInit();
+        for (const sec of MockModel._secondaries || []) await sec._delayedInit();
+        await MockModel._primary._initVersioning();
     }
 
+    _setLoadedField(fieldName: string, value: any) {
+        const oldValues = this._oldValues!;
+        if (oldValues.hasOwnProperty(fieldName)) return; // Already loaded earlier (as part of index key?)
+
+        this[fieldName as keyof Model<SUB>] = value;
+        if (typeof value === 'object' && value !== null) {            
+            const fieldType = (this.constructor.fields[fieldName] as FieldConfig<unknown>).type;
+            oldValues[fieldName] = fieldType.clone(value);
+        } else {
+            // This path is just an optimization
+            oldValues[fieldName] = value;
+        }
+    }
 
     /**
-     * Load a model instance by primary key.
-     * @param args - Primary key field values.
-     * @returns The model instance if found, undefined otherwise.
-     * 
-     * @example
-     * ```typescript
-     * const user = User.load("user123");
-     * const post = Post.load("post456", "en");
-     * ```
+     * @returns The primary key for this instance.
+     */
+    getPrimaryKey(): Uint8Array {
+        let key = this._primaryKey;
+        if (key === undefined) {
+            key = this.constructor._primary!._serializeKeyFields(this).toUint8Array();
+            this._setPrimaryKey(key);
+        }
+        return key;
+    }
+
+    _setPrimaryKey(key: Uint8Array, hash?: number) {
+        this._primaryKey = key;
+        this._primaryKeyHash = hash ?? hashBytes(key);
+        Object.defineProperties(this, this.constructor._primary._freezePrimaryKeyDescriptors);
+    }
+
+    /**
+     * @returns A 53-bit positive integer non-cryptographic hash of the primary key, or undefined if not yet saved.
      */
-    static load<SUB>(this: typeof Model<SUB>, ...args: any[]): SUB | undefined {
-        return this._pk!.get(...args);
+    getPrimaryKeyHash(): number {
+        if (this._primaryKeyHash === undefined) this.getPrimaryKey();
+        return this._primaryKeyHash!;
+    }
+
+    isLazyField(field: keyof this) {
+        const descr = this.constructor._primary!._lazyDescriptors[field];
+        return !!(descr && 'get' in descr && descr.get === Reflect.getOwnPropertyDescriptor(this, field)?.get);
+    }
+
+    _write(txn: Transaction): undefined | Change {
+        const oldValues = this._oldValues;
+
+        if (oldValues === null) { // Delete instance
+            const pk = this._primaryKey;
+            this.constructor._primary._delete(txn, pk!, this);
+            for(const index of this.constructor._secondaries || []) {
+                index._delete(txn, pk!, this);
+            }
+            
+            return "deleted";
+        }
+        
+        if (oldValues === undefined) { // Create instance
+            this.validate(true);
+
+            // Make sure the primary key does not already exist
+            const pk = this.getPrimaryKey();
+            if (dbGet(txn.id, pk!)) {
+                throw new DatabaseError("Unique constraint violation", "UNIQUE_CONSTRAINT");
+            }
+
+            // Insert the primary index
+            this.constructor._primary!._write(txn, pk!, this);
+
+            // Insert all secondaries
+            for (const index of this.constructor._secondaries || []) {
+                index._write(txn, pk!, this);
+            }
+
+            return "created";
+        }
+
+        // oldValues is an object.
+        // We're doing an update. Note that we may still be in a lazy state, and we don't want to load
+        // the whole object just to see if something changed.
+
+        // Add old values of changed fields to 'changed'.
+        const fields = this.constructor.fields;
+        let changed : Record<any, any> = {};
+        for(const fieldName in oldValues) {
+            const oldValue = oldValues[fieldName];
+            const newValue = this[fieldName as keyof Model<SUB>];
+            if (newValue !== oldValue  && !(fields[fieldName] as FieldConfig<unknown>).type.equals(newValue, oldValue)) {
+                changed[fieldName] = oldValue;
+            }
+        }
+
+        if (isObjectEmpty(changed)) return; // No changes, nothing to do
+
+        // Make sure primary has not been changed
+        for (const field of this.constructor._primary!._fieldTypes.keys()) {
+            if (changed.hasOwnProperty(field)) {
+                throw new DatabaseError(`Cannot modify primary key field: ${field}`, "CHANGE_PRIMARY");
+            }
+        }
+
+        // We have changes. Now it's okay for any lazy fields to be loaded (which the validate will trigger).
+
+        // Raise any validation errors
+        this.validate(true);
+
+        // Update the primary index
+        const pk = this._primaryKey!;
+        this.constructor._primary!._write(txn, pk, this);
+
+        // Update any secondaries with changed fields
+        for (const index of this.constructor._secondaries || []) {
+            for (const field of index._fieldTypes.keys()) {
+                if (changed.hasOwnProperty(field)) {
+                    index._delete(txn, pk, oldValues);
+                    index._write(txn, pk, this);
+                    break;
+                }
+            }
+        }
+        return changed;
     }
 
     /**
      * Prevent this instance from being persisted to the database.
      * 
-     * Removes the instance from the modified instances set and disables
-     * automatic persistence at transaction commit.
-     * 
      * @returns This model instance for chaining.
      * 
      * @example
@@ -364,14 +520,54 @@ export abstract class Model<SUB> {
      * ```
      */
     preventPersist() {
-        const modifiedInstances = olmdb.getTransactionData(MODIFIED_INSTANCES_SYMBOL) as Set<Model<any>>;
-        const unproxiedModel = (this as any)[TARGET_SYMBOL] || this;
-        modifiedInstances.delete(unproxiedModel);
-
-        unproxiedModel._state = 3; // no persist
+        this._txn.instances.delete(this);
+        // Have access to '_txn' throw a descriptive error:
+        Object.defineProperty(this, "_txn", PREVENT_PERSIST_DESCRIPTOR);
         return this;
     }
 
+    /**
+     * Find all instances of this model in the database, ordered by primary key.
+     * @param opts - Optional parameters.
+     * @param opts.reverse - If true, iterate in reverse order.
+     * @returns An iterator.
+     */
+    static findAll<T extends typeof Model<unknown>>(this: T, opts?: {reverse?: boolean}): IndexRangeIterator<T> {
+        return this._primary!.find(opts);
+    }
+
+    /**
+     * Load an existing instance by primary key and update it, or create a new one.
+     *
+     * The provided object must contain all primary key fields. If a matching row exists,
+     * the remaining properties from `obj` are set on the loaded instance. Otherwise a
+     * new instance is created with `obj` as its initial properties.
+     *
+     * @param obj - Partial model data that **must** include every primary key field.
+     * @returns The loaded-and-updated or newly created instance.
+     */
+    static replaceInto<T extends typeof Model<any>>(this: T, obj: Partial<Omit<InstanceType<T>, "constructor">>): InstanceType<T> {
+        const pk = this._primary!;
+        const keyArgs = [];
+        for (const fieldName of pk._fieldTypes.keys()) {
+            if (!(fieldName in (obj as any))) {
+                throw new DatabaseError(`replaceInto: missing primary key field '${fieldName}'`, "MISSING_PRIMARY_KEY");
+            }
+            keyArgs.push((obj as any)[fieldName]);
+        }
+
+        const existing = pk.get(...keyArgs as any) as InstanceType<T> | undefined;
+        if (existing) {
+            for (const key in obj as any) {
+                if (!pk._fieldTypes.has(key as any)) {
+                    (existing as any)[key] = (obj as any)[key];
+                }
+            }
+            return existing;
+        }
+        return new (this as any)(obj) as InstanceType<T>;
+    }
+
     /**
      * Delete this model instance from the database.
      * 
@@ -384,17 +580,8 @@ export abstract class Model<SUB> {
      * ```
      */
     delete() {
-        const unproxiedModel = ((this as any)[TARGET_SYMBOL] || this) as Model<SUB>;
-        
-        if (this._state === 2 || typeof this._state === 'object') {
-            for(const index of unproxiedModel.constructor._indexes!) {
-                const key = index._getKeyFromModel(unproxiedModel, true);
-                olmdb.del(key);
-                if (index instanceof PrimaryIndex) queueOnSave([this, undefined, key]);
-            }
-        }
-
-        this.preventPersist();
+        if (this._oldValues === undefined) throw new DatabaseError("Cannot delete unsaved instance", "NOT_SAVED");
+        this._oldValues = null;
     }
 
     /**
@@ -411,14 +598,15 @@ export abstract class Model<SUB> {
      * }
      * ```
      */
-    validate(raise: boolean = false): DatabaseError[] {
-        const errors: DatabaseError[] = [];
+    validate(raise: boolean = false): Error[] {
+        const errors: Error[] = [];
         
-        for (const [key, fieldConfig] of Object.entries(this._fields)) {
-            for (const error of fieldConfig.type.getErrors(this, key)) {
-                addErrorPath(error, key);
-                if (raise) throw error;
-                errors.push(error);
+        for (const [key, fieldConfig] of Object.entries(this.constructor.fields)) {
+            let e = fieldConfig.type.getError((this as any)[key]);
+            if (e) {
+                e = addErrorPath(e, this.constructor.tableName+"."+key);
+                if (raise) throw e;
+                errors.push(e as Error);
             }
         }
         return errors;
@@ -437,83 +625,25 @@ export abstract class Model<SUB> {
     isValid(): boolean {
         return this.validate().length === 0;
     }
-}
 
-// We use recursive proxies to track modifications made to, say, arrays within models. In
-// order to know which model a nested object belongs to, we maintain a WeakMap that maps
-// objects to their owner (unproxied) model.
-const modificationOwnerMap = new WeakMap<object, Model<any>>();
-
-// A cache for the proxies around nested objects, so that we don't need to recreate them
-// every time we access a property on a nested object (and so that their identity remains
-// the same).
-const modificationProxyCache = new WeakMap<object, any>();
-
-// Single proxy handler for both models and nested objects
-export const modificationTracker: ProxyHandler<any> = {
-    get(target, prop) {
-        if (prop === TARGET_SYMBOL) return target;
-        const value = target[prop];
-        if (!value || typeof value !== 'object' || (value instanceof Model)) return value;
-
-        // Check cache first
-        let proxy = modificationProxyCache.get(value);
-        if (proxy) return proxy;
-
-        let model;
-        if (target instanceof Model) {
-            if (!target._fields[prop as string]) {
-                // No need to track properties that are not model fields.
-                return value;
+    getState(): "deleted" | "created" | "loaded" | "lazy" {
+        if (this._oldValues === null) return "deleted";
+        if (this._oldValues === undefined) return "created";
+        for(const [key,descr] of Object.entries(this.constructor._primary!._lazyDescriptors)) {
+            if (descr && 'get' in descr && descr.get === Reflect.getOwnPropertyDescriptor(this, key)?.get) {
+                return "lazy";
             }
-            model = target;
-        } else {
-            model = modificationOwnerMap.get(target);
-            assert(model);
         }
+        return "loaded";
+    }
 
-        let state = model._state;
-        if (state !== undefined && state !== 2) {
-            // We don't need to track changes for this model (anymore). So we can just return the unproxied object.
-            // As we doing the modificationProxyCache lookup first, the identity of returned objects will not change:
-            // once a proxied object is returned, the same property will always return a proxied object.
-            return value;
-        }
-        
-        if (modificationOwnerMap.get(value)) {
-            throw new DatabaseError("Object cannot be embedded in multiple model instances", 'VALUE_ERROR');
-        }
-        modificationOwnerMap.set(value, model);
-        proxy = new Proxy(value, modificationTracker);
-        modificationProxyCache.set(value, proxy);
-        return proxy;
-    },
-    set(target, prop, value) {
-        let model;
-        if (target instanceof Model) {
-            model = target;
-            if (!model._fields[prop as string]) {
-                // No need to track properties that are not model fields.
-                (target as any)[prop] = value;
-                return true;
-            }
-        } else {
-            model = modificationOwnerMap.get(target);
-            assert(model);
-        }
-        
-        let state = model._state;
-        if (state === undefined || state === 2) {
-            const modifiedInstances = olmdb.getTransactionData(MODIFIED_INSTANCES_SYMBOL) as Set<Model<any>>;
-            modifiedInstances.add(model);
-            if (state === 2) {
-                model._state = model.constructor._indexes!.map(idx => idx._getKeyFromModel(model, true));
-            } else {
-                model._state = 1;
-            }
-        }
+    toString(): string {
+        const primary = this.constructor._primary;
+        const pk = primary._keyToArray(this._primaryKey || primary._serializeKeyFields(this).toUint8Array(false));
+        return `{Model:${this.constructor.tableName} ${this.getState()} ${pk}}`;
+    }
 
-        target[prop] = value;
-        return true;
+    [Symbol.for('nodejs.util.inspect.custom')]() {
+        return this.toString();
     }
-};
+}