edinburgh 0.3.0 → 0.4.1

package/src/datapack.ts

@@ -8,12 +8,15 @@ const BASE64_CHARS = '0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuv
 const BASE64_LOOKUP = new Uint8Array(128).fill(255); // Use 255 as invalid marker;
 for (let i = 0; i < BASE64_CHARS.length; ++i) BASE64_LOOKUP[BASE64_CHARS.charCodeAt(i)] = i;
 
-const COLORS = ['\x1b[32m', '\x1b[33m', '\x1b[34m', '\x1b[35m']; // green, yellow, blue, magenta
-const RESET_COLOR = '\x1b[0m';
-const ERROR_COLOR = '\x1b[31m'; // red
+const USE_COLORS = typeof process !== 'undefined' && process?.stdout?.isTTY;
+
+const COLORS = USE_COLORS ? ['\x1b[32m', '\x1b[33m', '\x1b[34m', '\x1b[35m'] : ['']; // green, yellow, blue, magenta
+const RESET_COLOR = USE_COLORS ? '\x1b[0m' : '';
+const ERROR_COLOR = USE_COLORS ? '\x1b[31m' : ''; // red
 
 let toStringTermCount = 0;
-let useExtendedLogging = !!process.env.DATAPACK_EXTENDED_LOGGING;
+let useExtendedLogging = typeof process !== 'undefined' ? !!process.env?.DATAPACK_EXTENDED_LOGGING : false;
+
 
 /**
  * A byte buffer for efficient reading and writing of primitive values and bit sequences.
@@ -23,7 +26,7 @@ let useExtendedLogging = !!process.env.DATAPACK_EXTENDED_LOGGING;
  * It supports both reading and writing operations with automatic buffer management.
  */
 
-export class DataPack {
+export default class DataPack {
     private buffer: Uint8Array;
     private dataView?: DataView;
 
@@ -41,7 +44,7 @@ export class DataPack {
      * Create a new DataPack instance.
      * @param data - Optional initial data as Uint8Array or buffer size as number.
      */
-    constructor(data: Uint8Array | number = 3900) {
+    constructor(data: Uint8Array | number = 1900) {
         if (data instanceof Uint8Array) {
             this.buffer = data;
             this.writePos = data.length;
@@ -123,6 +126,8 @@ export class DataPack {
      *   9: EOD
      *   10: identifier (6 byte positive int follows, represented as a base64 string of length 8)
      *   11: null-terminated string
+     *   12: Date/Time (varint with whole seconds since epoch follows)
+     *   13: custom type, followed by name string and data value
      * 5: short string (length in lower bits, 0-31)
      * 6: string (byte count of length in lower bits)
      * 7: blob (byte count of length in lower bits)
@@ -205,21 +210,21 @@ export class DataPack {
             this.buffer[this.writePos++] = (4 << 5) | 12;
             // Write a varint -- whole seconds should be plenty of resolution
             this.write(Math.floor(data.getTime()/1000));
-        } else if (typeof data === 'object' && data.constructor === Object) {
-            // Type 4, subtype 6: object start
-            this.buffer[this.writePos++] = (4 << 5) | 6;
-            for (const [key, value] of Object.entries(data)) {
-                this.write(key);
-                this.write(value);
+        } else if (typeof data === 'object') {
+            if (data instanceof CustomData) {
+                this.writeCustom(data.name, data.data);
+            } else if (typeof data.toDataPack === 'function') {
+                data.toDataPack(this);
+            } else {
+                this.writeAsObject(data);
             }
-            this.buffer[this.writePos++] = (4 << 5) | 9; // EOD
         } else {
             throw new Error(`Unsupported data type: ${typeof data}`);
         }
         return this;
     }
 
-    read(): any {
+    read(customConverters?: {[name: string]: ((data: any) => any)} | undefined): any {
         if (this.readPos > this.writePos) {
             throw new Error('Not enough data');
         }
@@ -251,7 +256,7 @@ export class DataPack {
             }
 
             case 4: {
-                // Special values and container starts
+                // Floats, special values and collections
                 switch (subtype) {
                     case 0: {
                         // float64
@@ -269,19 +274,19 @@ export class DataPack {
                         // Array start
                         const result = [];
                         while (true) {
-                            const nextValue = this.read();
+                            const nextValue = this.read(customConverters);
                             if (nextValue === EOD) break;
                             result.push(nextValue);
                         }
                         return result;
                     }
                     case 6: {
-                        // Object start
-                        const result: any = {};
+                        // Plain object or custom class instance
+                        let result: any = {};
                         while (true) {
-                            const key = this.read();
+                            const key = this.read(customConverters);
                             if (key === EOD) break;
-                            const value = this.read();
+                            const value = this.read(customConverters);
                             result[key] = value;
                         }
                         return result;
@@ -290,9 +295,9 @@ export class DataPack {
                         // Map start
                         const result = new Map();
                         while (true) {
-                            const key = this.read();
+                            const key = this.read(customConverters);
                             if (key === EOD) break;
-                            const value = this.read();
+                            const value = this.read(customConverters);
                             result.set(key, value);
                         }
                         return result;
@@ -301,7 +306,7 @@ export class DataPack {
                         // Set start
                         const result = new Set();
                         while (true) {
-                            const value = this.read();
+                            const value = this.read(customConverters);
                             if (value === EOD) break;
                             result.add(value);
                         }
@@ -310,7 +315,7 @@ export class DataPack {
                     case 9: return EOD;
                     case 10: {
                         // Identifier (6 byte positive int follows, represented as a base64 string of length 8)
-                        --this.readPos;
+                        --this.readPos; // readIdentifier() will read the header byte again
                         return this.readIdentifier();
                     }
                     case 11: {
@@ -330,6 +335,12 @@ export class DataPack {
                         const seconds = this.readPositiveInt();
                         return new Date(seconds * 1000);
                     }
+                    case 13: {
+                        // Custom type, followed by name string and data value
+                        const name = this.readString();
+                        const data = this.read();
+                        return (customConverters && name in customConverters) ? customConverters[name](data) : new CustomData(name, data);
+                    }
                     default: throw new Error(`Unknown type 4 subtype: ${subtype}`);
                 }
             }
@@ -430,19 +441,24 @@ export class DataPack {
         return result;
     }
 
-    writeIdentifier(id: string): DataPack {
-        if (id.length !== 8) {
-            throw new Error(`Identifier must be exactly 8 characters, got ${id.length}`);
-        }
-        
-        // Convert base64 string to 48-bit number
-        let value, num = 0;
-        for (let i = 0; i < 8; i++) {
-            const char = id.charCodeAt(i);
-            if (char > 127 || (value = BASE64_LOOKUP[char]) === 255) {
-                throw new Error(`Invalid base64 character: ${id[i]}`);
+    writeIdentifier(id: string | number): DataPack {
+        let num: number = 0;
+        if (typeof id === 'number') {
+            num = id;
+        } else {
+            if (id.length !== 8) {
+                throw new Error(`Identifier must be exactly 8 characters, got ${id.length}`);
+            }
+            
+            // Convert base64 string to 48-bit number
+            let value;
+            for (let i = 0; i < 8; i++) {
+                const char = id.charCodeAt(i);
+                if (char > 127 || (value = BASE64_LOOKUP[char]) === 255) {
+                    throw new Error(`Invalid base64 character: ${id[i]}`);
+                }
+                num = num * 64 + value;
             }
-            num = num * 64 + value;
         }
         
         // Write type 4, subtype 10 header
@@ -451,13 +467,28 @@ export class DataPack {
         
         // Write the 6-byte number in big-endian format
         for (let i = 5; i >= 0; i--) {
-            this.buffer[this.writePos++] = Math.floor(num / Math.pow(256, i)) & 0xFF;
+            this.buffer[this.writePos + i] = num % 256;
+            num = Math.floor(num / 256);
         }
+        this.writePos += 6;
         
         return this;
     }
 
     readIdentifier(): string {
+        let num = this.readIdentifierNumber();
+        
+        // Convert 48-bit number back to 8-character base64 string
+        let id = '';
+        for (let i = 0; i < 8; i++) {
+            id = BASE64_CHARS[num % 64] + id;
+            num = Math.floor(num / 64);
+        }
+        
+        return id;
+    }
+
+    readIdentifierNumber(): number {
         // Read the 6-byte number in big-endian format
         if (this.readPos + 7 > this.writePos) this.notEnoughData('identifier');
 
@@ -470,15 +501,7 @@ export class DataPack {
         for (let i = 0; i < 6; i++) {
             num = (num * 256) + this.buffer[this.readPos++];
         }
-        
-        // Convert 48-bit number back to 8-character base64 string
-        let id = '';
-        for (let i = 0; i < 8; i++) {
-            id = BASE64_CHARS[num % 64] + id;
-            num = Math.floor(num / 64);
-        }
-        
-        return id;
+        return num;
     }
 
     /**
@@ -503,6 +526,10 @@ export class DataPack {
         return copyBuffer ? this.buffer.slice(startPos, endPos) : this.buffer.subarray(startPos, endPos);
     }
 
+    toBuffer(): ArrayBuffer {
+        return this.buffer.buffer.slice(0, this.writePos) as ArrayBuffer;
+    }
+
     clone(copyBuffer: boolean, readPos: number = 0, writePos: number = this.writePos): DataPack {
         if (copyBuffer) {
             return new DataPack(this.buffer.slice(readPos, writePos));
@@ -514,6 +541,23 @@ export class DataPack {
         }
     }
 
+    writeCustom(name: string, data: any) {
+        this.buffer[this.writePos++] = (4 << 5) | 13;
+        this.write(name);
+        this.write(data);
+    }
+
+    writeAsObject(obj: Record<string, any>) {
+        // Type 4, subtype 6: object start
+        this.buffer[this.writePos++] = (4 << 5) | 6;
+        for (const key of Object.keys(obj)) {
+            this.write(key);
+            this.write(obj[key]);
+        }
+        this.buffer[this.writePos++] = (4 << 5) | 9; // EOD
+        return this;
+    }
+
     /**
      * Write a collection (array, set, object, or map) using a callback to add items/fields.
      * @param type 'array' | 'set' | 'object' | 'map'
@@ -640,6 +684,22 @@ export class DataPack {
         }
         return id;
     }
+
+    static createBuffer(...args: any) {
+        const pack = new DataPack();
+        for (const arg of args) {
+            pack.write(arg);
+        }
+        return pack.toBuffer();
+    }
+
+    static createUint8Array(...args: any) {
+        const pack = new DataPack();
+        for (const arg of args) {
+            pack.write(arg);
+        }
+        return pack.toUint8Array(false);
+    }
 }
 
 function toText(v: any): string {
@@ -653,3 +713,14 @@ function toText(v: any): string {
     }
     return JSON.stringify(v);
 }
+
+
+class CustomData {
+    constructor(public name: string, public data: any) {}
+}
+
+export default interface DataPack {
+    CustomData: typeof CustomData;
+}
+
+DataPack.prototype.CustomData = CustomData;

package/src/edinburgh.ts

@@ -1,6 +1,10 @@
-import * as olmdb from "olmdb";
-import { Model, INSTANCES_SYMBOL, resetModelCaches } from "./models.js";
-import { INSTANCES_BY_PK_SYMBOL } from "./indexes.js";
+import * as lowlevel from "olmdb/lowlevel";
+import { init as olmdbInit, DatabaseError } from "olmdb/lowlevel";
+import { modelRegistry, txnStorage, currentTxn } from "./models.js";
+
+let initNeeded = false;
+export function scheduleInit() { initNeeded = true; }
+
 
 // Re-export public API from models
 export {
@@ -9,7 +13,7 @@ export {
     field,
 } from "./models.js";
 
-import type { ChangedModel } from "./models.js";
+import type { Transaction, Change, Model } from "./models.js";
 
 // Re-export public API from types (only factory functions and instances)
 export {
@@ -37,14 +41,40 @@ export {
     dump,
 } from "./indexes.js";
 
-export {
-    setLogLevel
-} from "./utils.js";
+export { BaseIndex, UniqueIndex, PrimaryIndex, SecondaryIndex } from './indexes.js';
+
+export { type Change } from './models.js';
+export type { Transaction } from './models.js';
+export { DatabaseError } from "olmdb/lowlevel";
+export { runMigration } from './migrate.js';
+export type { MigrationOptions, MigrationResult } from './migrate.js';
+
+let olmdbReady = false;
+let maxRetryCount = 6;
+
+/**
+ * Initialize the database with the specified directory path.
+ * This function may be called multiple times with the same parameters. If it is not called before the first transact(),
+ * the database will be automatically initialized with the default directory.
+ * 
+ * @example
+ * ```typescript
+ * init("./my-database");
+ * ```
+ */
+export function init(dbDir: string): void {
+    olmdbReady = true;
+    olmdbInit(dbDir);
+}
+
+let pendingInit: Promise<void> | undefined;
 
-export { BaseIndex, UniqueIndex, PrimaryIndex } from './indexes.js';
 
-// Re-export from OLMDB
-export { init, onCommit, onRevert, getTransactionData, setTransactionData, DatabaseError } from "olmdb";
+const STALE_INSTANCE_DESCRIPTOR = {
+    get() {
+        throw new DatabaseError("The transaction for this model instance has ended", "STALE_INSTANCE");
+    },
+};
 
 /**
 * Executes a function within a database transaction context.
@@ -54,15 +84,13 @@ export { init, onCommit, onRevert, getTransactionData, setTransactionData, Datab
 * 
 * Transactions have a consistent view of the database, and changes made within a transaction are
 * isolated from other transactions until they are committed. In case a commit clashes with changes
-* made by another transaction, the transaction function will automatically be re-executed up to 10
+* made by another transaction, the transaction function will automatically be re-executed up to 6
 * times.
 * 
 * @template T - The return type of the transaction function.
-* @param fn - The function to execute within the transaction context.
+* @param fn - The function to execute within the transaction context. Receives a Transaction instance.
 * @returns A promise that resolves with the function's return value.
-* @throws {TypeError} If nested transactions are attempted.
 * @throws {DatabaseError} With code "RACING_TRANSACTION" if the transaction fails after retries due to conflicts.
-* @throws {DatabaseError} With code "TRANSACTION_FAILED" if the transaction fails for other reasons.
 * @throws {DatabaseError} With code "TXN_LIMIT" if maximum number of transactions is reached.
 * @throws {DatabaseError} With code "LMDB-{code}" for LMDB-specific errors.
 * 
@@ -70,7 +98,6 @@ export { init, onCommit, onRevert, getTransactionData, setTransactionData, Datab
 * ```typescript
 * const paid = await E.transact(() => {
 *   const user = User.pk.get("john_doe");
-*   // This is concurrency-safe - the function will rerun if it is raced by another transaction
 *   if (user.credits > 0) {
 *     user.credits--;
 *     return true;
@@ -81,78 +108,143 @@ export { init, onCommit, onRevert, getTransactionData, setTransactionData, Datab
 * ```typescript
 * // Transaction with automatic retry on conflicts
 * await E.transact(() => {
-*   const counter = Counter.load("global") || new Counter({id: "global", value: 0});
+*   const counter = Counter.pk.get("global") || new Counter({id: "global", value: 0});
 *   counter.value++;
 * });
 * ```
 */
 export async function transact<T>(fn: () => T): Promise<T> {
-    try {
-        const onSaveQueue: ChangedModel[] | undefined = onSaveCallback ? [] : undefined;
-        return await olmdb.transact(async (): Promise<T> => {
-            const instances = new Set<Model<any>>();
-            olmdb.setTransactionData(INSTANCES_SYMBOL, instances);
-            olmdb.setTransactionData(INSTANCES_BY_PK_SYMBOL, new Map());
-            
-            const savedInstances: Set<Model<any>> = new Set();
+    while (initNeeded || pendingInit) {
+        // Make sure only one async task is doing the inits, the rest should wait for it
+        if (pendingInit) {
+            await pendingInit;
+        } else {
+            pendingInit = (async () => {
+                if (!olmdbReady) olmdbInit('.edinburgh');
+                olmdbReady = true;
+
+                initNeeded = false;
+                for (const model of Object.values(modelRegistry)) {
+                    await model._delayedInit();
+                }
+                pendingInit = undefined;
+            })();
+        }
+    }
+
+    // try {
+        for (let retryCount = 0; retryCount < maxRetryCount; retryCount++) {
+            const txnId = lowlevel.startTransaction();
+            const txn: Transaction = { id: txnId, instances: new Set(), instancesByPk: new Map() };
+            const onSaveItems: Map<Model<unknown>, Change> | undefined = onSaveCallback ? new Map() : undefined;
+
+            let result: T | undefined;
             try {
-                const result = await fn();
-                // Save all modified instances before committing.
-                while(instances.size > 0) {
-                    // Back referencing can cause models to be scheduled for save() a second time,
-                    // which is why we require the outer loop.
-                    for (const instance of instances) {
-                        instance._onCommit(onSaveQueue);
-                        savedInstances.add(instance);
-                        instances.delete(instance);
+                await txnStorage.run(txn, async function() {
+                    result = await fn();
+
+                    // Call preCommit() on all instances before writing.
+                    // Note: Set iteration visits newly added items, so preCommit() creating
+                    // new instances is handled correctly.
+                    for (const instance of txn.instances) {
+                        instance.preCommit?.();
+                    }
+
+                    // Save all modified instances before committing
+                    // This needs to happen inside txnStorage.run, because resolving default values
+                    // for identifiers requires database access.
+                    for (const instance of txn.instances) {
+                        const change = instance._write(txn);
+                        if (onSaveItems && change) {
+                            onSaveItems.set(instance, change);
+                        }
                     }
+                });
+            } catch (e: any) {
+                try { lowlevel.abortTransaction(txnId); } catch {}
+                throw e;
+            } finally {
+                // Make the instances read-only to make it clear that their transaction has ended.
+                for (const instance of txn.instances) {
+                    delete instance._oldValues;
+                    Object.defineProperty(instance, "_txn", STALE_INSTANCE_DESCRIPTOR);
+                    Object.freeze(instance);
                 }
-                if (onSaveQueue?.length) {
-                    olmdb.onCommit((commitId: number) => {
-                        if (onSaveCallback) onSaveCallback(commitId, onSaveQueue);
-                    });
+                // Destroy the transaction object, to make sure things crash if they are used after
+                // this point, and to help the GC reclaim memory.
+                txn.id = txn.instances = txn.instancesByPk = undefined as any;
+            }
+
+            const commitResult = lowlevel.commitTransaction(txnId);
+            const commitSeq = typeof commitResult === 'number' ? commitResult : await commitResult;
+
+            if (commitSeq > 0) {
+                // Success
+                if (onSaveItems?.size) {
+                    onSaveCallback!(commitSeq, onSaveItems);
                 }
-                
-                return result;
-            } catch (error) {
-                // Discard changes on all saved and still unsaved instances
-                for (const instance of savedInstances) instance.preventPersist();
-                for (const instance of instances) instance.preventPersist();
-                throw error;
+                return result as T;
             }
-        });
-    } catch (e: Error | any) {
-        // This hackery is required to provide useful stack traces. Without this,
-        // both Bun and Node (even with --async-stack-traces) don't show which
-        // line called the transact(), which is pretty important info when validation
-        // fails, for instance. Though the line numbers in Bun still don't really
-        // make sense. Probably this bug: https://github.com/oven-sh/bun/issues/15859
-        e.stack += "\nat async:\n" + new Error().stack?.replace(/^.*?\n/, '');
-        throw e;
-    }
+
+            // Race condition - retry
+        }
+        throw new DatabaseError("Transaction keeps getting raced", "RACING_TRANSACTION");
+    // } catch (e: Error | any) {
+    //     // This hackery is required to provide useful stack traces.
+    //     const callerStack = new Error().stack?.replace(/^.*?\n/, '');
+    //     e.stack += "\nat async:\n" + callerStack;
+    //     throw e;
+    // }
 }
 
-let onSaveCallback: ((commitId: number, items: ChangedModel[]) => void) | undefined;
+/**
+ * Set the maximum number of retries for a transaction in case of conflicts.
+ * The default value is 6. Setting it to 0 will disable retries and cause transactions to fail immediately on conflict.
+ *
+ * @param count The maximum number of retries for a transaction.
+ */
+export function setMaxRetryCount(count: number) {
+    maxRetryCount = count;
+}
 
+let onSaveCallback: ((commitId: number, items: Map<Model<any>, Change>) => void) | undefined;
+ 
 /**
  * Set a callback function to be called after a model is saved and committed.
  *
  * @param callback The callback function to set. It gets called after each successful
  * `transact()` commit that has changes, with the following arguments:
  *   - A sequential number. Higher numbers have been committed after lower numbers.
- *   - An array of model instances that have been modified, created, or deleted.
- *     You can used its {@link Model.changed} property to figure out what changed.
+ *   - A map of model instances to their changes. The change can be "created", "deleted", or an object containing the old values.
  */
-export function setOnSaveCallback(callback: ((commitId: number, items: ChangedModel[]) => void) | undefined) {
+export function setOnSaveCallback(callback: ((commitId: number, items: Map<Model<any>, Change>) => void) | undefined) {
     onSaveCallback = callback;
 }
 
 
 export async function deleteEverything(): Promise<void> {
-    await olmdb.transact(() => {
-        for (const {key} of olmdb.scan()) {
-            olmdb.del(key);
-        }
-    });
-    await resetModelCaches();
+    let done = false;
+    while (!done) {
+        await transact(() => {
+            const txn = currentTxn();
+            const iteratorId = lowlevel.createIterator(txn.id, undefined, undefined, false);
+            const deadline = Date.now() + 150;
+            let count = 0;
+            try {
+                while (true) {
+                    const raw = lowlevel.readIterator(iteratorId);
+                    if (!raw) { done = true; break; }
+                    lowlevel.del(txn.id, raw.key);
+                    if (++count >= 4096 || Date.now() >= deadline) break;
+                }
+            } finally {
+                lowlevel.closeIterator(iteratorId);
+            }
+        });
+    }
+    // Re-init indexes since metadata was deleted
+    for (const model of Object.values(modelRegistry)) {
+        if (!model.fields) continue;
+        await model._delayedInit(true);
+    }
 }