edinburgh 0.3.0 → 0.4.2

package/src/indexes.ts

@@ -1,8 +1,9 @@
-import * as olmdb from "olmdb";
-import { DatabaseError } from "olmdb";
-import { DataPack } from "./datapack.js";
-import { FieldConfig, getMockModel, Model, modelRegistry } from "./models.js";
-import { assert, logLevel, delayedInits, tryDelayedInits } from "./utils.js";
+import * as lowlevel from "olmdb/lowlevel";
+import { DatabaseError } from "olmdb/lowlevel";
+import DataPack from "./datapack.js";
+import { FieldConfig, getMockModel, Model, Transaction, currentTxn } from "./models.js";
+import { scheduleInit, transact } from "./edinburgh.js";
+import { assert, logLevel, dbGet, dbPut, dbDel, hashBytes, toBuffer } from "./utils.js";
 import { deserializeType, serializeType, TypeWrapper } from "./types.js";
 
 // Index system types and utilities
@@ -12,6 +13,26 @@ type IndexArgTypes<M extends typeof Model<any>, F extends readonly (keyof Instan
 
 const MAX_INDEX_ID_PREFIX = -1;
 const INDEX_ID_PREFIX = -2;
+const VERSION_INFO_PREFIX = -3;
+
+const MAX_INDEX_ID_BUFFER = new DataPack().write(MAX_INDEX_ID_PREFIX).toUint8Array();
+
+/** Cached information about a specific version of a primary index's value format. */
+interface VersionInfo {
+    migrateHash: number;
+    /** Non-key field names → TypeWrappers for deserialization of this version's data. */
+    nonKeyFields: Map<string, TypeWrapper<any>>;
+    /** Set of serialized secondary index signatures that existed in this version. */
+    secondaryKeys: Set<string>;
+}
+
+function bytesEqual(a: Uint8Array, b: Uint8Array): boolean {
+    if (a.length !== b.length) return false;
+    for (let i = 0; i < a.length; i++) {
+        if (a[i] !== b[i]) return false;
+    }
+    return true;
+}
 
 /**
  * Iterator for range queries on indexes.
@@ -20,35 +41,28 @@ const INDEX_ID_PREFIX = -2;
  */
 export class IndexRangeIterator<M extends typeof Model> implements Iterator<InstanceType<M>>, Iterable<InstanceType<M>> {
     constructor(
-        private iterator: olmdb.DbIterator<any,any> | undefined,
+        private txn: Transaction,
+        private iteratorId: number,
         private indexId: number,
         private parentIndex: BaseIndex<M, any>
-    ) {}
+    ) {
+    }
 
     [Symbol.iterator](): Iterator<InstanceType<M>> {
         return this;
     }
 
     next(): IteratorResult<InstanceType<M>> {
-        if (!this.iterator) return { done: true, value: undefined };
-        const entry = this.iterator.next();
-        if (entry.done) {
-            this.iterator.close();
+        if (this.iteratorId < 0) return { done: true, value: undefined };
+        const raw = lowlevel.readIterator(this.iteratorId);
+        if (!raw) {
+            lowlevel.closeIterator(this.iteratorId);
+            this.iteratorId = -1;
             return { done: true, value: undefined };
         }
         
-        // Extract the key without the index ID
-        const keyBytes = new DataPack(entry.value.key);
-        const entryIndexId = keyBytes.readNumber();
-        assert(entryIndexId === this.indexId);
-
-        // Use polymorphism to get the model from the entry
-        const model = this.parentIndex._pairToInstance(keyBytes, entry.value.value);
-
-        if (!model) {
-            // This shouldn't happen, but skip if it does
-            return this.next();
-        }
+        // Dispatches to the _pairToInstance specific to the index type
+        const model = this.parentIndex._pairToInstance(this.txn, raw.key, raw.value);
 
         return { done: false, value: model };
     }
@@ -95,45 +109,6 @@ type FindOptions<ARG_TYPES extends readonly any[]> = (
     }
 );
 
-const canonicalUint8Arrays = new Map<number, WeakRef<Uint8Array>>();
-
-export function testArraysEqual(array1: Uint8Array, array2: Uint8Array): boolean {
-    if (array1.length !== array2.length) return false;
-    for (let i = 0; i < array1.length; i++) {
-        if (array1[i] !== array2[i]) return false;
-    }
-    return true;
-}
-
-/**
- * Get a singleton instance of a Uint8Array containing the given data.
- * @param data - The Uint8Array to canonicalize.
- * @returns A unique Uint8Array, backed by a right-sized copy of the ArrayBuffer.
- */
-function getSingletonUint8Array<T>(data: Uint8Array): Uint8Array {
-    let hash : number = 5381, reclaimHash: number | undefined;
-    for (const byte of data) {
-        hash = ((hash << 5) + hash + byte) >>> 0;
-    }
-    while(true) {
-        let weakRef = canonicalUint8Arrays.get(hash);
-        if (!weakRef) break;
-        if (weakRef) {
-            const orgData = weakRef.deref();
-            if (!orgData) { // weakRef expired
-                if (reclaimHash === undefined) reclaimHash = hash;
-            } else if (data===orgData || testArraysEqual(data, orgData)) {
-                return orgData;
-            }
-            // else: hash collision, use open addressing
-        }
-        hash = (hash+1) >>> 0;
-    }
-    let copy = data.slice(); // Make a copy, backed by a new, correctly sized ArrayBuffer
-    canonicalUint8Arrays.set(reclaimHash === undefined ? hash : reclaimHash, new WeakRef(copy));
-    return copy;
-}
-
 
 /**
  * Base class for database indexes for efficient lookups on model fields.
@@ -147,7 +122,8 @@ export abstract class BaseIndex<M extends typeof Model, const F extends readonly
     public _MyModel: M;
     public _fieldTypes: Map<keyof InstanceType<M> & string, TypeWrapper<any>> = new Map();
     public _fieldCount!: number;
-
+    _resetIndexFieldDescriptors: Record<string | symbol | number, PropertyDescriptor> = {};
+    
     /**
      * Create a new index.
      * @param MyModel - The model class this index belongs to.
@@ -155,21 +131,34 @@ export abstract class BaseIndex<M extends typeof Model, const F extends readonly
      */
     constructor(MyModel: M, public _fieldNames: F) {
         this._MyModel = getMockModel(MyModel);
-        delayedInits.add(this);
-        tryDelayedInits();
     }
 
-    _delayedInit(): boolean {
-        if (!this._MyModel.fields) return false; // Awaiting model init
+    async _delayedInit() {
+        if (this._indexId != null) return; // Already initialized
         for(const fieldName of this._fieldNames) {
             assert(typeof fieldName === 'string', 'Field names must be strings');
             this._fieldTypes.set(fieldName, this._MyModel.fields[fieldName].type);
         }
         this._fieldCount = this._fieldNames.length;
-        return true;
+        await this._retrieveIndexId();
+
+        // Human-readable signature for version tracking, e.g. "secondary category:string"
+        this._signature = this._getTypeName() + ' ' +
+            Array.from(this._fieldTypes.entries()).map(([n, t]) => n + ':' + t).join(' ');
+
+        for(const fieldName of this._fieldTypes.keys()) {
+            this._resetIndexFieldDescriptors[fieldName] = {
+                writable: true,
+                configurable: true,
+                enumerable: true
+            };
+        }
     }
 
-    _cachedIndexId?: number;
+    _indexId?: number;
+
+    /** Human-readable signature for version tracking, e.g. "secondary category:string" */
+    _signature?: string;
 
     /**
      * Serialize array of key values to a (index-id prefixed) Bytes instance that can be used as a key.
@@ -183,7 +172,7 @@ export abstract class BaseIndex<M extends typeof Model, const F extends readonly
     _argsToKeyBytes(args: any, allowPartial: boolean) {
         assert(allowPartial ? args.length <= this._fieldCount : args.length === this._fieldCount);
         const bytes = new DataPack();
-        bytes.write(this._getIndexId());
+        bytes.write(this._indexId!);
         let index = 0;
         for(const fieldType of this._fieldTypes.values()) {
             // For partial keys, undefined values are acceptable and represent open range suffixes
@@ -193,77 +182,88 @@ export abstract class BaseIndex<M extends typeof Model, const F extends readonly
         return bytes;
     }
 
-    _argsToKeySingleton(args: IndexArgTypes<M, F>): Uint8Array {
-        const bytes = this._argsToKeyBytes(args, false);
-        return getSingletonUint8Array(bytes.toUint8Array());
-    }
-
     /**
      * Extract model from iterator entry - implemented differently by each index type.
-     * @param keyBytes - Key bytes with index ID already read.
-     * @param valueBuffer - Value Uint8Array from the entry.
+     * @param keyBuffer - Key bytes (including index id).
+     * @param valueBuffer - Value bytes from the entry.
      * @returns Model instance or undefined.
      * @internal
      */
-    abstract _pairToInstance(keyBytes: DataPack, valueBuffer: Uint8Array): InstanceType<M> | undefined;
+    abstract _pairToInstance(txn: Transaction, keyBuffer: ArrayBuffer, valueBuffer: ArrayBuffer): InstanceType<M>;
 
-    _hasNullIndexValues(model: InstanceType<M>) {
+    _hasNullIndexValues(data: Record<string, any>) {
         for(const fieldName of this._fieldTypes.keys()) {
-            if (model[fieldName] == null) return true;
+            if (data[fieldName] == null) return true;
         }
         return false;
     }
 
-    _instanceToKeyBytes(model: InstanceType<M>): DataPack {
+    // Must return the exact key that will be used to write to the K/V store
+    abstract _serializeKey(primaryKey: Uint8Array, data: Record<string, any>): Uint8Array;
+
+    // Returns the indexId + serialized key fields. Used in some _serializeKey implementations
+    // and for calculating _primaryKey.
+    _serializeKeyFields(data: Record<string, any>): DataPack {
         const bytes = new DataPack();
-        bytes.write(this._getIndexId());
+        bytes.write(this._indexId!);
         for(const [fieldName, fieldType] of this._fieldTypes.entries()) {
-            fieldType.serialize(model[fieldName], bytes);
+            fieldType.serialize(data[fieldName], bytes);
         }
         return bytes;
     }
 
     /**
-     * Get or create unique index ID for this index.
-     * @returns Numeric index ID.
+     * Retrieve (or create) a stable index ID from the DB, with retry on transaction races.
+     * Sets `this._indexId` on success.
      */
-    _getIndexId(): number {
-        // Resolve an index to a number
-        let indexId = this._cachedIndexId;
-        if (indexId == null) {
-            const indexNameBytes = new DataPack().write(INDEX_ID_PREFIX).write(this._MyModel.tableName).write(this._getTypeName());
-            for(let name of this._fieldNames) {
+    async _retrieveIndexId(): Promise<void> {
+        const indexNameBytes = new DataPack().write(INDEX_ID_PREFIX).write(this._MyModel.tableName).write(this._getTypeName());
+        for(let name of this._fieldNames) {
+            indexNameBytes.write(name);
+            serializeType(this._MyModel.fields[name].type, indexNameBytes);
+        }
+        // For non-primary indexes, include primary key field info to avoid misinterpreting
+        // values when the primary key schema changes.
+        if (this._MyModel._primary !== (this as any)) {
+            indexNameBytes.write(undefined); // separator
+            for (const name of this._MyModel._primary._fieldNames) {
                 indexNameBytes.write(name);
                 serializeType(this._MyModel.fields[name].type, indexNameBytes);
             }
-            const indexNameBuf = indexNameBytes.toUint8Array();
-
-            let result = olmdb.get(indexNameBuf);
-            if (result) {
-                indexId = this._cachedIndexId = new DataPack(result).readNumber();
-            } else {
-                const maxIndexIdBuf = new DataPack().write(MAX_INDEX_ID_PREFIX).toUint8Array();
-                result = olmdb.get(maxIndexIdBuf);
-                indexId = result ? new DataPack(result).readNumber() + 1 : 1;
-                olmdb.onCommit(() => {
-                    // Only if the transaction succeeds can we cache this id
-                    this._cachedIndexId = indexId;
-                });
-
-                const idBuf = new DataPack().write(indexId).toUint8Array();
-                olmdb.put(indexNameBuf, idBuf);
-                olmdb.put(maxIndexIdBuf, idBuf); // This will also cause the transaction to rerun if we were raced
-                if (logLevel >= 1) {
-                    console.log(`Create ${this} with id ${indexId}`);
+        }
+        const indexNameBuf = indexNameBytes.toUint8Array();
+
+        while (true) {
+            const txnId = lowlevel.startTransaction();
+            try {
+                let result = dbGet(txnId, indexNameBuf);
+                let id: number;
+                if (result) {
+                    id = new DataPack(result).readNumber();
+                } else {
+                    result = dbGet(txnId, MAX_INDEX_ID_BUFFER);
+                    id = result ? new DataPack(result).readNumber() + 1 : 1;
+                    const idBuf = new DataPack().write(id).toUint8Array();
+                    dbPut(txnId, indexNameBuf, idBuf);
+                    dbPut(txnId, MAX_INDEX_ID_BUFFER, idBuf);
+                    if (logLevel >= 1) console.log(`[edinburgh] Create index ${this}`);
+                }
+                const commitResult = lowlevel.commitTransaction(txnId);
+                const commitSeq = typeof commitResult === 'number' ? commitResult : await commitResult;
+                if (commitSeq > 0) {
+                    this._indexId = id;
+                    return;
                 }
+            } catch (e) {
+                try { lowlevel.abortTransaction(txnId); } catch {}
+                throw e;
             }
         }
-        return indexId;
     }
 
 
-    abstract _delete(model: InstanceType<M>): void;
-    abstract _write(model: InstanceType<M>): void;
+    abstract _delete(txn: Transaction, primaryKey: Uint8Array, model: InstanceType<M>): void;
+    abstract _write(txn: Transaction, primaryKey: Uint8Array, model: InstanceType<M>): void;
 
     /**
      * Find model instances using flexible range query options.
@@ -323,61 +323,129 @@ export abstract class BaseIndex<M extends typeof Model, const F extends readonly
      * }
      * ```
      */
-    public find(opts: FindOptions<IndexArgTypes<M, F>> = {}): IndexRangeIterator<M> {
-        const indexId = this._getIndexId();
-        
+    _computeKeyBounds(opts: FindOptions<IndexArgTypes<M, F>>): [DataPack | undefined, DataPack | undefined] | null {
         let startKey: DataPack | undefined;
         let endKey: DataPack | undefined;
-
         if ('is' in opts) {
-            // Exact match - set both 'from' and 'to' to the same value
             startKey = this._argsToKeyBytes(toArray(opts.is), true);
             endKey = startKey.clone(true).increment();
         } else {
-            // Range query
             if ('from' in opts) {
                 startKey = this._argsToKeyBytes(toArray(opts.from), true);
             } else if ('after' in opts) {
                 startKey = this._argsToKeyBytes(toArray(opts.after), true);
-                if (!startKey.increment()) {
-                    // There can be nothing 'after' - return an empty iterator
-                    return new IndexRangeIterator(undefined, indexId, this);
-                }
+                if (!startKey.increment()) return null;
             } else {
-                // Open start: begin at first key for this index id
                 startKey = this._argsToKeyBytes([], true);
             }
-            
             if ('to' in opts) {
                 endKey = this._argsToKeyBytes(toArray(opts.to), true).increment();
             } else if ('before' in opts) {
                 endKey = this._argsToKeyBytes(toArray(opts.before), true);
             } else {
-                // Open end: end at first key of the next index id
-                endKey = this._argsToKeyBytes([], true).increment(); // Next indexId
+                endKey = this._argsToKeyBytes([], true).increment();
             }
         }
+        return [startKey, endKey];
+    }
+
+    public find(opts: FindOptions<IndexArgTypes<M, F>> = {}): IndexRangeIterator<M> {
+        const txn = currentTxn();
+        const indexId = this._indexId!;
+
+        const bounds = this._computeKeyBounds(opts);
+        if (!bounds) return new IndexRangeIterator(txn, -1, indexId, this);
+        const [startKey, endKey] = bounds;
 
         // For reverse scans, swap start/end keys since OLMDB expects it
         const scanStart = opts.reverse ? endKey : startKey;
         const scanEnd = opts.reverse ? startKey : endKey;
 
         if (logLevel >= 3) {
-            console.log(`Scan ${this} start=${scanStart} end=${scanEnd} reverse=${opts.reverse||false}`);
+            console.log(`[edinburgh] Scan ${this} start=${scanStart} end=${scanEnd} reverse=${opts.reverse||false}`);
         }
-        const iterator = olmdb.scan({
-            start: scanStart?.toUint8Array(),
-            end: scanEnd?.toUint8Array(),
-            reverse: opts.reverse || false,
-        });
+        const startBuf = scanStart?.toUint8Array();
+        const endBuf = scanEnd?.toUint8Array();
+        const iteratorId = lowlevel.createIterator(
+            txn.id,
+            startBuf ? toBuffer(startBuf) : undefined,
+            endBuf ? toBuffer(endBuf) : undefined,
+            opts.reverse || false,
+        );
         
-        return new IndexRangeIterator(iterator, indexId, this);
+        return new IndexRangeIterator(txn, iteratorId, indexId, this);
+    }
+
+    /**
+     * Process all matching rows in batched transactions.
+     *
+     * Uses the same query options as {@link find}. The batch is committed and a new
+     * transaction started once either `limitSeconds` or `limitRows` is exceeded.
+     *
+     * @param opts - Query options (same as `find()`), plus:
+     * @param opts.limitSeconds - Max seconds per transaction batch (default: 1)
+     * @param opts.limitRows - Max rows per transaction batch (default: 4096)
+     * @param callback - Called for each matching row within a transaction
+     */
+    public async batchProcess(
+        opts: FindOptions<IndexArgTypes<M, F>> & { limitSeconds?: number; limitRows?: number } = {} as any,
+        callback: (row: InstanceType<M>) => void | Promise<void>
+    ): Promise<void> {
+        const limitMs = (opts.limitSeconds ?? 1) * 1000;
+        const limitRows = opts.limitRows ?? 4096;
+        const reverse = opts.reverse ?? false;
+
+        const bounds = this._computeKeyBounds(opts);
+        if (!bounds) return;
+        const startKey = bounds[0]?.toUint8Array();
+        const endKey = bounds[1]?.toUint8Array();
+        let cursor: Uint8Array | undefined;
+
+        while (true) {
+            const next = await transact(async (): Promise<Uint8Array | null> => {
+                const txn = currentTxn();
+                const batchStart = cursor && !reverse ? cursor : startKey;
+                const batchEnd = cursor && reverse ? cursor : endKey;
+                const scanStart = reverse ? batchEnd : batchStart;
+                const scanEnd = reverse ? batchStart : batchEnd;
+
+                const iteratorId = lowlevel.createIterator(
+                    txn.id,
+                    scanStart ? toBuffer(scanStart) : undefined,
+                    scanEnd ? toBuffer(scanEnd) : undefined,
+                    reverse,
+                );
+
+                const t0 = Date.now();
+                let count = 0;
+                let lastRawKey: Uint8Array | undefined;
+                try {
+                    while (true) {
+                        const raw = lowlevel.readIterator(iteratorId);
+                        if (!raw) return null;
+                        lastRawKey = new Uint8Array(raw.key);
+                        await callback(this._pairToInstance(txn, raw.key, raw.value));
+                        if (++count >= limitRows || Date.now() - t0 >= limitMs) break;
+                    }
+                } finally {
+                    lowlevel.closeIterator(iteratorId);
+                }
+
+                lastRawKey = lastRawKey.slice(); // Copy, as lastRawKey points at OLMDB's internal read-only mmap
+                if (reverse) return lastRawKey!;
+                const nk = new DataPack(lastRawKey!);
+                return nk.increment() ? nk.toUint8Array() : null;
+            });
+
+            if (next === null) break;
+            cursor = next;
+        }
     }
 
     abstract _getTypeName(): string;
 
     toString() {
-        return `${this._getIndexId()}:${this._MyModel.tableName}:${this._getTypeName()}[${Array.from(this._fieldTypes.keys()).join(',')}]`;
+        return `${this._indexId}:${this._MyModel.tableName}:${this._getTypeName()}[${Array.from(this._fieldTypes.keys()).join(',')}]`;
     }
 }
 
@@ -386,9 +454,6 @@ function toArray<ARG_TYPES extends readonly any[]>(args: ArrayOrOnlyItem<ARG_TYP
     return (Array.isArray(args) ? args : [args]) as Partial<ARG_TYPES>;
 }
 
-/** @internal Symbol used to attach modified instances, keyed by singleton primary key, to a transaction */
-export const INSTANCES_BY_PK_SYMBOL = Symbol('instances');
-
 /**
  * Primary index that stores the actual model data.
  * 
@@ -400,17 +465,26 @@ export class PrimaryIndex<M extends typeof Model, const F extends readonly (keyo
     _nonKeyFields!: (keyof InstanceType<M> & string)[];
     _lazyDescriptors: Record<string | symbol | number, PropertyDescriptor> = {};
     _resetDescriptors: Record<string | symbol | number, PropertyDescriptor> = {};
+    _freezePrimaryKeyDescriptors: Record<string | symbol | number, PropertyDescriptor> = {};
+
+    /** Current version number for this primary index's value format. */
+    _currentVersion!: number;
+    /** Hash of the current migrate() function source, or 0 if none. */
+    _currentMigrateHash!: number;
+    /** Cached version info for old versions (loaded on demand). */
+    _versions: Map<number, VersionInfo> = new Map();
 
     constructor(MyModel: M, fieldNames: F) {
         super(MyModel, fieldNames);
         if (MyModel._primary) {
-            throw new DatabaseError(`Model ${MyModel.tableName} already has a primary key defined`, 'INIT_ERROR');
+            throw new DatabaseError(`There's already a primary index defined: ${MyModel._primary}. This error may also indicate that your tsconfig.json needs to have "target": "ES2022" set.`, 'INIT_ERROR');
         }
         MyModel._primary = this;
     }
 
-    _delayedInit(): boolean {
-        if (!super._delayedInit()) return false;
+    async _delayedInit() {
+        if (this._indexId != null) return; // Already initialized
+        await super._delayedInit();
         const MyModel = this._MyModel;
         this._nonKeyFields = Object.keys(MyModel.fields).filter(fieldName => !this._fieldNames.includes(fieldName as any)) as any;
 
@@ -432,7 +506,104 @@ export class PrimaryIndex<M extends typeof Model, const F extends readonly (keyo
                 enumerable: true
             };
         }
-        return true;
+
+        for(const fieldName of this._fieldNames) {
+            this._freezePrimaryKeyDescriptors[fieldName] = {
+                writable: false,
+                enumerable: true
+            };
+        }
+
+    }
+
+    /** Serialize the current version fingerprint as a DataPack object. */
+    _serializeVersionValue(): Uint8Array {
+        const fields: [string, Uint8Array][] = [];
+        for (const fieldName of this._nonKeyFields) {
+            const tp = new DataPack();
+            serializeType(this._MyModel.fields[fieldName].type, tp);
+            fields.push([fieldName, tp.toUint8Array()]);
+        }
+        return new DataPack().write({
+            migrateHash: this._currentMigrateHash,
+            fields,
+            secondaryKeys: new Set((this._MyModel._secondaries || []).map(sec => sec._signature!)),
+        }).toUint8Array();
+    }
+
+    /** Look up or create the current version number for this primary index. */
+    async _initVersioning(): Promise<void> {
+        // Compute migrate hash from function source
+        const migrateFn = (this._MyModel as any)._original?.migrate ?? (this._MyModel as any).migrate;
+        this._currentMigrateHash = migrateFn ? hashBytes(new TextEncoder().encode(migrateFn.toString().replace(/\s\s+/g, ' ').trim())) : 0;
+
+        const currentValueBytes = this._serializeVersionValue();
+
+        // Scan last 20 version info rows for this primary index
+        const scanStart = new DataPack().write(VERSION_INFO_PREFIX).write(this._indexId!);
+        const scanEnd = scanStart.clone(true).increment();
+
+        while (true) {
+            const txnId = lowlevel.startTransaction();
+            try {
+                const iteratorId = lowlevel.createIterator(
+                    txnId,
+                    scanEnd ? toBuffer(scanEnd.toUint8Array()) : undefined,
+                    toBuffer(scanStart.toUint8Array()),
+                    true // reverse - scan newest versions first
+                );
+
+                let count = 0;
+                let maxVersion = 0;
+                let found = false;
+
+                try {
+                    while (count < 20) {
+                        const raw = lowlevel.readIterator(iteratorId);
+                        if (!raw) break;
+                        count++;
+
+                        const keyPack = new DataPack(new Uint8Array(raw.key));
+                        keyPack.readNumber(); // skip VERSION_INFO_PREFIX
+                        keyPack.readNumber(); // skip indexId
+                        const versionNum = keyPack.readNumber();
+                        maxVersion = Math.max(maxVersion, versionNum);
+
+                        const valueBytes = new Uint8Array(raw.value);
+                        if (bytesEqual(valueBytes, currentValueBytes)) {
+                            this._currentVersion = versionNum;
+                            found = true;
+                            break;
+                        }
+                    }
+                } finally {
+                    lowlevel.closeIterator(iteratorId);
+                }
+
+                if (found) {
+                    lowlevel.abortTransaction(txnId);
+                    return;
+                }
+
+                // No match found - create new version
+                this._currentVersion = maxVersion + 1;
+                const versionKey = new DataPack()
+                    .write(VERSION_INFO_PREFIX)
+                    .write(this._indexId!)
+                    .write(this._currentVersion)
+                    .toUint8Array();
+                dbPut(txnId, versionKey, currentValueBytes);
+                if (logLevel >= 1) console.log(`[edinburgh] Create version ${this._currentVersion} for ${this}`);
+
+                const commitResult = lowlevel.commitTransaction(txnId);
+                const commitSeq = typeof commitResult === 'number' ? commitResult : await commitResult;
+                if (commitSeq > 0) return;
+                // Race - retry
+            } catch (e) {
+                try { lowlevel.abortTransaction(txnId); } catch {}
+                throw e;
+            }
+        }
     }
 
     /**
@@ -445,8 +616,8 @@ export class PrimaryIndex<M extends typeof Model, const F extends readonly (keyo
      * const user = User.pk.get("john_doe");
      * ```
      */
-    get(...args: IndexArgTypes<M, F> | [Uint8Array]): InstanceType<M> | undefined {
-        return this._get(args, false);
+    get(...args: IndexArgTypes<M, F>): InstanceType<M> | undefined {
+        return this._get(currentTxn(), args, true);
     }
 
     /**
@@ -456,39 +627,50 @@ export class PrimaryIndex<M extends typeof Model, const F extends readonly (keyo
      * @param args Primary key field values. (Or a single Uint8Array containing the key.)
      * @returns The (lazily loaded) model instance.
      */
-    getLazy(...args: IndexArgTypes<M, F> | [Uint8Array]): InstanceType<M> {
-        return this._get(args, true);
+    getLazy(...args: IndexArgTypes<M, F>): InstanceType<M> {
+        return this._get(currentTxn(), args, false);
     }
 
-    _get(args: IndexArgTypes<M, F> | [Uint8Array], lazy: true): InstanceType<M>;
-    _get(args: IndexArgTypes<M, F> | [Uint8Array], lazy: false): InstanceType<M> | undefined;
-    _get(args: IndexArgTypes<M, F> | [Uint8Array], lazy: boolean) {
-        let key, keyParts;
-        if (args.length === 1 && args[0] instanceof Uint8Array) {
-            key = getSingletonUint8Array(args[0]);
+    _get(txn: Transaction, args: IndexArgTypes<M, F> | Uint8Array, loadNow: false | Uint8Array): InstanceType<M>;
+    _get(txn: Transaction, args: IndexArgTypes<M, F> | Uint8Array, loadNow: true): InstanceType<M> | undefined;
+    _get(txn: Transaction, args: IndexArgTypes<M, F> | Uint8Array, loadNow: boolean | Uint8Array) {
+        let key: Uint8Array, keyParts;
+        if (args instanceof Uint8Array) {
+            key = args;
         } else {
-            key = this._argsToKeySingleton(args as IndexArgTypes<M, F>);
+            key = this._argsToKeyBytes(args as IndexArgTypes<M, F>, false).toUint8Array();
             keyParts = args;
         }
+
+        const keyHash = hashBytes(key);
+        const cached = txn.instancesByPk.get(keyHash) as InstanceType<M>;
+        if (cached) {
+            if (loadNow && loadNow !== true) {
+                // The object already exists, but it may still be lazy-loaded
+                Object.defineProperties(cached, this._resetDescriptors);
+                this._setNonKeyValues(cached, loadNow);
+            }
+            return cached;
+        }
         
-        const cachedInstances = olmdb.getTransactionData(INSTANCES_BY_PK_SYMBOL) as Map<Uint8Array, InstanceType<M>>;
-        const cached = cachedInstances.get(key);
-        if (cached) return cached;
-        
-        let valueBuffer;
-        if (!lazy) {
-            valueBuffer = olmdb.get(key);
-            if (logLevel >= 3) {
-                console.log(`Get ${this} key=${new DataPack(key)} result=${valueBuffer && new DataPack(valueBuffer)}`);
+        let valueBuffer: Uint8Array | undefined;
+        if (loadNow) {
+            if (loadNow === true) {
+                valueBuffer = dbGet(txn.id, key);
+                if (logLevel >= 3) {
+                    console.log(`[edinburgh] Get ${this} key=${new DataPack(key)} result=${valueBuffer && new DataPack(valueBuffer)}`);
+                }
+                if (!valueBuffer) return;
+            } else {
+                valueBuffer = loadNow; // Uint8Array
             }
-            if (!valueBuffer) return;
         }
         
         // This is a primary index. So we can now deserialize all primary and non-primary fields into instance values.
-        const model = new (this._MyModel as any)() as InstanceType<M>;
-        
-        // Store the canonical primary key on the model
-        model._primaryKey = key;
+        const model = new (this._MyModel as any)(undefined, txn) as InstanceType<M>;
+
+        // Set to the original value for all fields that are loaded by _setLoadedField
+        model._oldValues = {};
 
         // Set the primary key fields on the model
         if (keyParts) {
@@ -498,97 +680,151 @@ export class PrimaryIndex<M extends typeof Model, const F extends readonly (keyo
             }
         } else {
             const bytes = new DataPack(key);
-            assert(bytes.readNumber() === this._MyModel._primary._getIndexId()); // Skip index id
+            assert(bytes.readNumber() === this._MyModel._primary._indexId); // Skip index id
             for(const [fieldName, fieldType] of this._fieldTypes.entries()) {
                 model._setLoadedField(fieldName, fieldType.deserialize(bytes));
             }
         }
 
+        // Store the canonical primary key on the model, set the hash, and freeze the primary key fields.
+        model._setPrimaryKey(key, keyHash);
+
         if (valueBuffer) {
-            // Set other fields
-            this._setNonKeyValues(model, new DataPack(valueBuffer));
+            // Non-lazy load. Set other fields
+            this._setNonKeyValues(model, valueBuffer);
         } else {
             // Lazy - set getters for other fields
             Object.defineProperties(model, this._lazyDescriptors);
+            // When creating a lazy instance, we don't need to add it to txn.instances yet, as only the
+            // primary key fields are loaded, and they cannot be modified (so we don't need to check).
+            // When any other field is set, that will trigger a lazy-load, adding the instance to
+            // txn.instances.
         }
-    
-        cachedInstances.set(key, model);
+
+        txn.instancesByPk.set(keyHash, model);
         return model;
     }
 
-    /**
-     * Create a canonical primary key buffer for the given model instance.
-     * Returns a singleton Uint8Array for stable Map/Set identity usage.
-     */
-    _instanceToKeySingleton(model: InstanceType<M>): Uint8Array {
-        const bytes = this._instanceToKeyBytes(model);
-        return getSingletonUint8Array(bytes.toUint8Array());
+    _serializeKey(primaryKey: Uint8Array, _data: Record<string, any>): Uint8Array {
+        return primaryKey;
     }
 
     _lazyNow(model: InstanceType<M>) {
-        let valueBuffer = olmdb.get(model._primaryKey!);
+        let valueBuffer = dbGet(model._txn.id, model._primaryKey!);
         if (logLevel >= 3) {
-            console.log(`Lazy retrieve ${this} key=${new DataPack(model._primaryKey)} result=${valueBuffer && new DataPack(valueBuffer)}`);
+            console.log(`[edinburgh] Lazy retrieve ${this} key=${new DataPack(model._primaryKey)} result=${valueBuffer && new DataPack(valueBuffer)}`);
         }
         if (!valueBuffer) throw new DatabaseError(`Lazy-loaded ${model.constructor.name}#${model._primaryKey} does not exist`, 'LAZY_FAIL');
         Object.defineProperties(model, this._resetDescriptors);
-        this._setNonKeyValues(model, new DataPack(valueBuffer));
+        this._setNonKeyValues(model, valueBuffer);
     }
 
-    _setNonKeyValues(model: InstanceType<M>, valueBytes: DataPack) {
+    _setNonKeyValues(model: InstanceType<M>, valueArray: Uint8Array) {
         const fieldConfigs = this._MyModel.fields;
+        const valuePack = new DataPack(valueArray);
+        const version = valuePack.readNumber();
 
-        for (const fieldName of this._nonKeyFields) {
-            const value = fieldConfigs[fieldName].type.deserialize(valueBytes);
-            model._setLoadedField(fieldName, value);
+        if (version === this._currentVersion) {
+            for (const fieldName of this._nonKeyFields) {
+                model._setLoadedField(fieldName, fieldConfigs[fieldName].type.deserialize(valuePack));
+            }
+        } else {
+            this._migrateFromVersion(model, version, valuePack);
         }
     }
 
-    _keyToArray(key: Uint8Array): IndexArgTypes<M, F> {
-        const bytes = new DataPack(key);
-        return this._fieldTypes.values().map((fieldType) => {
-            return fieldType.deserialize(bytes);
-        }) as any;
+    /** Load a version's info from DB, caching the result. */
+    _loadVersionInfo(txnId: number, version: number): VersionInfo {
+        let info = this._versions.get(version);
+        if (info) return info;
+
+        const key = new DataPack()
+            .write(VERSION_INFO_PREFIX)
+            .write(this._indexId!)
+            .write(version)
+            .toUint8Array();
+        const raw = dbGet(txnId, key);
+        if (!raw) throw new DatabaseError(`Version ${version} info not found for index ${this}`, 'CONSISTENCY_ERROR');
+
+        const obj = new DataPack(raw).read() as any;
+        if (!obj || typeof obj.migrateHash !== 'number' || !Array.isArray(obj.fields) || !(obj.secondaryKeys instanceof Set))
+            throw new DatabaseError(`Version ${version} info is corrupted for index ${this}`, 'CONSISTENCY_ERROR');
+
+        const nonKeyFields = new Map<string, TypeWrapper<any>>();
+        for (const [name, typeBytes] of obj.fields) {
+            nonKeyFields.set(name, deserializeType(new DataPack(typeBytes), 0));
+        }
+
+        info = { migrateHash: obj.migrateHash, nonKeyFields, secondaryKeys: obj.secondaryKeys as Set<string> };
+        this._versions.set(version, info);
+        return info;
     }
 
-    _pairToInstance(keyBytes: DataPack, valueBuffer: Uint8Array): InstanceType<M> | undefined {
-        const valueBytes = new DataPack(valueBuffer);
-        const model = new (this._MyModel as any)() as InstanceType<M>;
+    /** Deserialize and migrate a row from an old version. */
+    _migrateFromVersion(model: InstanceType<M>, version: number, valuePack: DataPack) {
+        const versionInfo = this._loadVersionInfo(model._txn.id, version);
 
-        for(const [fieldName, fieldType] of this._fieldTypes.entries()) {
-            model._setLoadedField(fieldName, fieldType.deserialize(keyBytes));
+        // Deserialize using old field types into a plain record
+        const record: Record<string, any> = {};
+        for (const [name] of this._fieldTypes.entries()) record[name] = (model as any)[name]; // pk fields
+        for (const [name, type] of versionInfo.nonKeyFields.entries()) {
+            record[name] = type.deserialize(valuePack);
         }
-        model._primaryKey = getSingletonUint8Array(keyBytes.toUint8Array());
-        
-        this._setNonKeyValues(model, valueBytes);
 
-        return model;
+        // Run migrate() if it exists
+        const migrateFn = (this._MyModel as any).migrate;
+        if (migrateFn) migrateFn(record);
+
+        // Set non-key fields on model from the (possibly migrated) record
+        for (const fieldName of this._nonKeyFields) {
+            if (fieldName in record) {
+                model._setLoadedField(fieldName, record[fieldName]);
+            } else if (fieldName in model) {
+                // Instantiate the default value
+                model._setLoadedField(fieldName, (model as any)[fieldName]);
+            } else {
+                throw new DatabaseError(`Field ${fieldName} is missing in migrated data for ${model}`, 'MIGRATION_ERROR');
+            }
+        }
+    }
+
+    _keyToArray(key: Uint8Array): IndexArgTypes<M, F> {
+        const bytes = new DataPack(key);
+        assert(bytes.readNumber() === this._indexId);
+        const result = [] as any[];
+        for (const fieldType of this._fieldTypes.values()) {
+            result.push(fieldType.deserialize(bytes));
+        }
+        return result as any;
+    }
+
+    _pairToInstance(txn: Transaction, keyBuffer: ArrayBuffer, valueBuffer: ArrayBuffer): InstanceType<M> {
+        return this._get(txn, new Uint8Array(keyBuffer), new Uint8Array(valueBuffer));
     }
 
     _getTypeName(): string {
         return 'primary';
     }
 
-    _write(model: InstanceType<M>) {
+    _write(txn: Transaction, primaryKey: Uint8Array, data: Record<string, any>) {
         let valueBytes = new DataPack();
+        valueBytes.write(this._currentVersion);
         const fieldConfigs = this._MyModel.fields as any;
         for (const fieldName of this._nonKeyFields) {
             const fieldConfig = fieldConfigs[fieldName] as FieldConfig<unknown>;
-            fieldConfig.type.serialize(model[fieldName], valueBytes);
+            fieldConfig.type.serialize(data[fieldName], valueBytes);
         }
         if (logLevel >= 2) {
-            console.log(`Write ${this} key=${new DataPack(model._getCreatePrimaryKey())} value=${valueBytes}`);
+            console.log(`[edinburgh] Write ${this} key=${new DataPack(primaryKey)} value=${valueBytes}`);
         }
-        olmdb.put(model._getCreatePrimaryKey(), valueBytes.toUint8Array());
+        dbPut(txn.id, primaryKey, valueBytes.toUint8Array());
     }
 
-    _delete(model: InstanceType<M>) {
-        if (model._primaryKey) {
-            if (logLevel >= 2) {
-                console.log(`Delete ${this} key=${new DataPack(model._primaryKey)}`);
-            }
-            olmdb.del(model._primaryKey);
+    _delete(txn: Transaction, primaryKey: Uint8Array, _data: Record<string, any>) {
+        if (logLevel >= 2) {
+            console.log(`[edinburgh] Delete ${this} key=${new DataPack(primaryKey)}`);
         }
+        dbDel(txn.id, primaryKey);
     }
 }
 
@@ -603,6 +839,7 @@ export class UniqueIndex<M extends typeof Model, const F extends readonly (keyof
     constructor(MyModel: M, fieldNames: F) {
         super(MyModel, fieldNames);
         (this._MyModel._secondaries ||= []).push(this);
+        scheduleInit();
     }
 
     /**
@@ -616,72 +853,69 @@ export class UniqueIndex<M extends typeof Model, const F extends readonly (keyof
      * ```
      */
     get(...args: IndexArgTypes<M, F>): InstanceType<M> | undefined {
-        let keyBuffer = this._argsToKeySingleton(args);
+        const txn = currentTxn();
+        let keyBuffer = this._argsToKeyBytes(args, false).toUint8Array();
 
-        let valueBuffer = olmdb.get(keyBuffer);
+        let valueBuffer = dbGet(txn.id, keyBuffer);
         if (logLevel >= 3) {
-            console.log(`Get ${this} key=${new DataPack(keyBuffer)} result=${valueBuffer}`);
+            console.log(`[edinburgh] Get ${this} key=${new DataPack(keyBuffer)} result=${valueBuffer}`);
         }
         if (!valueBuffer) return;
 
         const pk = this._MyModel._primary!;
-        const result = pk.get(valueBuffer);
+        const result = pk._get(txn, valueBuffer, true);
         if (!result) throw new DatabaseError(`Unique index ${this} points at non-existing primary for key: ${args.join(', ')}`, 'CONSISTENCY_ERROR');
         return result;
     }
 
-    _delete(model: InstanceType<M>) {
-        if (!this._hasNullIndexValues(model)) {
-            const keyBytes = this._instanceToKeyBytes(model);
+    _serializeKey(primaryKey: Uint8Array, data: Record<string, any>): Uint8Array {
+        return this._serializeKeyFields(data).toUint8Array();
+    }
+
+    _delete(txn: Transaction, primaryKey: Uint8Array, data: Record<string, any>) {
+        if (!this._hasNullIndexValues(data)) {
+            const key = this._serializeKey(primaryKey, data);
             if (logLevel >= 2) {
-                console.log(`Delete ${this} key=${keyBytes}`);
+                console.log(`[edinburgh] Delete ${this} key=${key}`);
             }
-            olmdb.del(keyBytes.toUint8Array());
+            dbDel(txn.id, key);
         }
     }
 
-    _write(model: InstanceType<M>) {
-        if (!this._hasNullIndexValues(model)) {
-            const key = this._instanceToKeyBytes(model);
+    _write(txn: Transaction, primaryKey: Uint8Array, data: Record<string, any>) {
+        if (!this._hasNullIndexValues(data)) {
+            const key = this._serializeKey(primaryKey, data);
             if (logLevel >= 2) {
-                console.log(`Write ${this} key=${key} value=${new DataPack(model._primaryKey)}`);
+                console.log(`[edinburgh] Write ${this} key=${key} value=${new DataPack(primaryKey)}`);
             }
-            const keyBuffer = key.toUint8Array();
-            if (olmdb.get(keyBuffer)) {
+            if (dbGet(txn.id, key)) {
                 throw new DatabaseError(`Unique constraint violation for ${this} key ${key}`, 'UNIQUE_CONSTRAINT');
             }
-            olmdb.put(keyBuffer, model._primaryKey!);
+            dbPut(txn.id, key, primaryKey);
         }
     }
 
-    /**
-     * Extract model from iterator entry for unique index.
-     * @param keyBytes - Key bytes with index ID already read.
-     * @param valueBytes - Value bytes from the entry.
-     * @returns Model instance or undefined.
-     * @internal
-     */
-    _pairToInstance(keyBytes: DataPack, valueBuffer: Uint8Array): InstanceType<M> | undefined {
+    _pairToInstance(txn: Transaction, keyBuffer: ArrayBuffer, valueBuffer: ArrayBuffer): InstanceType<M> {
         // For unique indexes, the value contains the primary key
 
+        const keyPack = new DataPack(new Uint8Array(keyBuffer));
+        keyPack.readNumber(); // discard index id
+
         const pk = this._MyModel._primary!;
-        const model = pk.getLazy(valueBuffer);
+        const model = pk._get(txn, new Uint8Array(valueBuffer), false);
 
-        // Read the index fields from the key, overriding lazy loading for these fields
+        // _get will have created lazy-load getters for our indexed fields. Let's turn them back into
+        // regular properties:
+        Object.defineProperties(model, this._resetIndexFieldDescriptors);
+
+        // Set the values for our indexed fields
         for(const [name, fieldType] of this._fieldTypes.entries()) {
-            // getLazy will have created a getter for this field - make it a normal property instead
-            Object.defineProperty(model, name, {
-                writable: true,
-                configurable: true,
-                enumerable: true
-            });
-            model._setLoadedField(name, fieldType.deserialize(keyBytes));
+            model._setLoadedField(name, fieldType.deserialize(keyPack));
         }
 
         return model;
     }
 
-
     _getTypeName(): string {
         return 'unique';
     }
@@ -701,64 +935,60 @@ export class SecondaryIndex<M extends typeof Model, const F extends readonly (ke
     constructor(MyModel: M, fieldNames: F) {
         super(MyModel, fieldNames);
         (this._MyModel._secondaries ||= []).push(this);
+        scheduleInit();
     }
 
-    /**
-     * Extract model from iterator entry for secondary index.
-     * @param keyBytes - Key bytes with index ID already read.
-     * @param valueBuffer - Value Uint8Array from the entry.
-     * @returns Model instance or undefined.
-     * @internal
-     */
-    _pairToInstance(keyBytes: DataPack, valueBuffer: Uint8Array): InstanceType<M> | undefined {
+    _pairToInstance(txn: Transaction, keyBuffer: ArrayBuffer, _valueBuffer: ArrayBuffer): InstanceType<M> {
         // For secondary indexes, the primary key is stored after the index fields in the key
+
+        const keyPack = new DataPack(new Uint8Array(keyBuffer));
+        keyPack.readNumber(); // discard index id
         
         // Read the index fields, saving them for later
         const indexFields = new Map();
         for(const [name, type] of this._fieldTypes.entries()) {
-            indexFields.set(name, type.deserialize(keyBytes));
+            indexFields.set(name, type.deserialize(keyPack));
         }
 
-        const primaryKey = keyBytes.readUint8Array();
-        const model = this._MyModel._primary!.getLazy(primaryKey);
+        const primaryKey = keyPack.readUint8Array();
+        const model = this._MyModel._primary!._get(txn, primaryKey, false);
+
+
+        // _get will have created lazy-load getters for our indexed fields. Let's turn them back into
+        // regular properties:
+        Object.defineProperties(model, this._resetIndexFieldDescriptors);
 
-        // Add the index fields to the model, overriding lazy loading for these fields
+        // Set the values for our indexed fields
         for(const [name, value] of indexFields) {
-            // getLazy will have created a getter for this field - make it a normal property instead
-            Object.defineProperty(model, name, {
-                writable: true,
-                configurable: true,
-                enumerable: true
-            });
             model._setLoadedField(name, value);
         }
 
         return model;
     }
 
-    _instanceToKeyBytes(model: InstanceType<M>): DataPack {
+    _serializeKey(primaryKey: Uint8Array, model: InstanceType<M>): Uint8Array {
         // index id + index fields + primary key
-        const bytes = super._instanceToKeyBytes(model);
-        bytes.write(model._getCreatePrimaryKey());
-        return bytes;
+        const bytes = super._serializeKeyFields(model);
+        bytes.write(primaryKey);
+        return bytes.toUint8Array();
     }
 
-    _write(model: InstanceType<M>) {
+    _write(txn: Transaction, primaryKey: Uint8Array, model: InstanceType<M>) {
         if (this._hasNullIndexValues(model)) return;
-        const keyBytes = this._instanceToKeyBytes(model);
+        const key = this._serializeKey(primaryKey, model);
         if (logLevel >= 2) {
-            console.log(`Write ${this} key=${keyBytes}`);
+            console.log(`[edinburgh] Write ${this} key=${key}`);
         }
-        olmdb.put(keyBytes.toUint8Array(), SECONDARY_VALUE);
+        dbPut(txn.id, key, SECONDARY_VALUE);
     }
 
-    _delete(model: InstanceType<M>): void {
+    _delete(txn: Transaction, primaryKey: Uint8Array, model: InstanceType<M>): void {
         if (this._hasNullIndexValues(model)) return;
-        const keyBytes = this._instanceToKeyBytes(model);
+        const key = this._serializeKey(primaryKey, model);
         if (logLevel >= 2) {
-            console.log(`Delete ${this} key=${keyBytes}`);
+            console.log(`[edinburgh] Delete ${this} key=${key}`);
         }
-        olmdb.del(keyBytes.toUint8Array());
+        dbDel(txn.id, key);
     }
 
     _getTypeName(): string {
@@ -852,38 +1082,71 @@ export function index(MyModel: typeof Model, fields: any): SecondaryIndex<any, a
  * This is primarily useful for development and debugging purposes.
  */
 export function dump() {
+    const txn = currentTxn();
     let indexesById = new Map<number, {name: string, type: string, fields: Record<string, TypeWrapper<any>>}>();
-    console.log("--- Database dump ---")
-    for(const {key,value} of olmdb.scan()) {
-        const kb = new DataPack(key);
-        const vb = new DataPack(value);
+    let versions = new Map<number, Map<number, Map<string, TypeWrapper<any>>>>();
+    console.log("--- edinburgh database dump ---")
+    const iteratorId = lowlevel.createIterator(txn.id, undefined, undefined, false);
+    try {
+    while (true) {
+        const raw = lowlevel.readIterator(iteratorId);
+        if (!raw) break;
+        const kb = new DataPack(new Uint8Array(raw.key));
+        const vb = new DataPack(new Uint8Array(raw.value));
         const indexId = kb.readNumber();
         if (indexId === MAX_INDEX_ID_PREFIX) {
             console.log("* Max index id", vb.readNumber());
+        } else if (indexId === VERSION_INFO_PREFIX) {
+            const idxId = kb.readNumber();
+            const version = kb.readNumber();
+            const obj = vb.read() as any;
+            const nonKeyFields = new Map<string, TypeWrapper<any>>();
+            for (const [name, typeBytes] of obj.fields) {
+                nonKeyFields.set(name, deserializeType(new DataPack(typeBytes), 0));
+            }
+            if (!versions.has(idxId)) versions.set(idxId, new Map());
+            versions.get(idxId)!.set(version, nonKeyFields);
+            console.log(`* Version ${version} for index ${idxId}: fields=[${[...nonKeyFields.keys()].join(',')}]`);
         } else if (indexId === INDEX_ID_PREFIX) {
             const name = kb.readString();
             const type = kb.readString();
             const fields: Record<string, TypeWrapper<any>> = {};
             while(kb.readAvailable()) {
-                const name = kb.readString();
+                const name = kb.read();
+                if (name === undefined) break; // what follows are primary key fields (when this is a secondary index)
                 fields[name] = deserializeType(kb, 0);
             }
+
             const indexId = vb.readNumber();
             console.log(`* Index definition ${indexId}:${name}:${type}[${Object.keys(fields).join(',')}]`);
             indexesById.set(indexId, {name, type, fields});
         } else if (indexId > 0 && indexesById.has(indexId)) {
             const index = indexesById.get(indexId)!;
-            const {name, type, fields} = index;
-            const rowKey: any = {};
-            for(const [fieldName, fieldType] of Object.entries(fields)) {
-                rowKey[fieldName] = fieldType.deserialize(kb);
+            let name, type, rowKey: any, rowValue: any;
+            if (index) {
+                name = index.name;
+                type = index.type;
+                const fields = index.fields;
+                rowKey = {};
+                for(const [fieldName, fieldType] of Object.entries(fields)) {
+                    rowKey[fieldName] = fieldType.deserialize(kb);
+                }
+                if (type === 'primary') {
+                    const version = vb.readNumber();
+                    const vFields = versions.get(indexId)?.get(version);
+                    if (vFields) {
+                        rowValue = {};
+                        for (const [fieldName, fieldType] of vFields) {
+                            rowValue[fieldName] = fieldType.deserialize(vb);
+                        }
+                    }
+                }
             }
-            // const Model = modelRegistry[name]!;
-            // TODO: once we're storing schemas (serializeType) in the db, we can deserialize here
-            console.log(`* Row for ${indexId}:${name}:${type}[${Object.keys(fields).join(',')}] key=${kb} value=${vb}`);
+            console.log(`* Row for ${indexId}:${name}:${type}`, rowKey ?? kb, rowValue ?? vb);
         } else {
             console.log(`* Unhandled '${indexId}' key=${kb} value=${vb}`);
         }
     }
-    console.log("--- End of database dump ---")
+    } finally { lowlevel.closeIterator(iteratorId); }
+    console.log("--- end ---")
 }