inibase 2.0.1 → 3.1.0

package/dist/file.js

@@ -1,4 +1,5 @@
-import { access, appendFile, copyFile, constants as fsConstants, open, readFile, stat, unlink, writeFile, } from "node:fs/promises";
+import { access, appendFile, copyFile, constants as fsConstants, open, readFile, stat, unlink, } from "node:fs/promises";
+import { hostname } from "node:os";
 import { join, resolve } from "node:path";
 import { createInterface } from "node:readline";
 import { Transform } from "node:stream";
@@ -6,37 +7,231 @@ import { pipeline } from "node:stream/promises";
 import { createGunzip, createGzip } from "node:zlib";
 import Inison from "inison";
 import { globalConfig, } from "./index.js";
+import { recover } from "./journal.js";
 import { detectFieldType, isArrayOfObjects, isNumber, isObject, isStringified, isValidID, } from "./utils.js";
 import { compare, decodeID, encodeID, exec, gunzip, gzip, } from "./utils.server.js";
-// Locks older than this are assumed abandoned by a crashed/killed process, not a slow operation.
-const STALE_LOCK_MS = 30_000;
-export const lock = async (folderPath, prefix) => {
-    let lockFile = null;
-    const lockFilePath = join(folderPath, `${prefix ?? ""}.locked`);
+// Locks older than this are candidates for removal. Same-host locks are only
+// stolen when the recorded owner PID is provably dead; foreign-host locks fall
+// back to this TTL (NFS has no reliable cross-host liveness check).
+const DEFAULT_LOCK_TTL_MS = Number(process.env.INIBASE_LOCK_TTL_MS ?? 60_000);
+// Durability knob: `full` (default) fsyncs every temp file, journal entry and
+// directory touch before acknowledging a mutation. `none` skips all fsync
+// calls but keeps the write-ahead journal protocol unchanged, so a *process*
+// crash (page cache survives) still recovers atomically — only power-loss
+// durability is lost. The ACID claim documented in the README holds at `full`.
+const DURABILITY = process.env.INIBASE_DURABILITY ?? "full";
+export const DURABLE = DURABILITY !== "none";
+/** fsync a handle, or no-op when `INIBASE_DURABILITY=none` is set. */
+const maybeSync = async (handle) => {
+    if (DURABLE)
+        await handle.sync();
+};
+const LOCK_RETRY_MS = 13;
+// Per-process reentrancy: the same process may acquire the same lock file
+// multiple times (e.g. post() -> get() -> sort-cache). Depth 1 is a real
+// filesystem acquisition; deeper acquisitions just bump the counter.
+const lockDepths = new Map();
+const lockFilePathFor = (folderPath, prefix) => join(folderPath, `${prefix ?? ""}.locked`);
+const lockOwnerState = async (lockFilePath) => {
     try {
-        lockFile = await open(lockFilePath, "wx");
+        const metadata = JSON.parse(await readFile(lockFilePath, "utf8"));
+        if (typeof metadata.pid === "number" && metadata.host === hostname()) {
+            try {
+                process.kill(metadata.pid, 0);
+            }
+            catch (error) {
+                return error?.code === "ESRCH" ? "dead" : "unknown";
+            }
+            return "alive";
+        }
+    }
+    catch {
+        // No/invalid metadata (e.g. crash between create and metadata write).
+    }
+    return "unknown"; // foreign host or unparsable metadata
+};
+/**
+ * A lock may be stolen when its owner is provably dead on this host
+ * (immediately — a same-host crash must not wedge writers or block crash
+ * recovery) or, for foreign/unknown owners where no liveness check exists
+ * (e.g. NFS), once the recorded lock is older than the TTL.
+ */
+const stealableLock = async (lockFilePath, mtimeMs, ttl) => {
+    const state = await lockOwnerState(lockFilePath);
+    if (state === "alive")
+        return false;
+    if (state === "dead")
+        return true;
+    return Date.now() - mtimeMs > ttl;
+};
+export const lock = async (folderPath, prefix, ttl = DEFAULT_LOCK_TTL_MS) => {
+    const lockFilePath = lockFilePathFor(folderPath, prefix);
+    const resolvedPath = resolve(lockFilePath);
+    const depth = lockDepths.get(resolvedPath);
+    if (depth) {
+        lockDepths.set(resolvedPath, depth + 1);
         return;
     }
-    catch ({ message }) {
-        if (message.split(":")[0] === "EEXIST") {
+    for (;;) {
+        try {
+            const lockFile = await open(lockFilePath, "wx");
+            try {
+                await lockFile.writeFile(JSON.stringify({
+                    pid: process.pid,
+                    host: hostname(),
+                    startedAt: Date.now(),
+                }));
+                await maybeSync(lockFile);
+            }
+            finally {
+                await lockFile.close();
+            }
+            // The global (prefix-less) lock is the writer lock: run crash
+            // recovery for this table while we exclusively hold it. Locked
+            // calls with a prefix are read-side helpers (e.g. sort cache) and
+            // must never roll back an in-flight transaction.
+            if (!prefix) {
+                try {
+                    await recover(folderPath);
+                }
+                catch {
+                    // Recovery is best-effort at lock time; reads retry on
+                    // torn state, writers re-check before mutating.
+                }
+            }
+            lockDepths.set(resolvedPath, 1);
+            return;
+        }
+        catch (error) {
+            const message = String(error?.message ?? error);
+            if (message.split(":")[0] !== "EEXIST")
+                throw error;
             const lockStat = await stat(lockFilePath).catch(() => null);
-            if (lockStat && Date.now() - lockStat.mtimeMs > STALE_LOCK_MS)
+            // Someone else released the lock between our failed open and the
+            // stat: retry immediately instead of counting down the TTL.
+            if (!lockStat)
+                continue;
+            if (await stealableLock(lockFilePath, lockStat.mtimeMs, ttl)) {
                 await unlink(lockFilePath).catch(() => { });
-            return await new Promise((resolve) => setTimeout(() => resolve(lock(folderPath, prefix)), 13));
+            }
+            await new Promise((resolvePromise) => setTimeout(() => resolvePromise(), LOCK_RETRY_MS));
         }
     }
-    finally {
-        await lockFile?.close();
+};
+/**
+ * Non-blocking lock acquisition (used by the read path and the open-time
+ * recovery sweep). Returns true when the lock was acquired (running crash
+ * recovery for prefix-less locks, exactly like `lock`), false when another
+ * process holds it. A single stale-lock steal (dead same-host owner, or aged
+ * foreign/unknown owner) is attempted so a crashed owner can't wedge readers
+ * behind it forever.
+ */
+export const tryLock = async (folderPath, prefix, ttl = DEFAULT_LOCK_TTL_MS) => {
+    const lockFilePath = lockFilePathFor(folderPath, prefix);
+    const resolvedPath = resolve(lockFilePath);
+    const depth = lockDepths.get(resolvedPath);
+    if (depth) {
+        lockDepths.set(resolvedPath, depth + 1);
+        return true;
+    }
+    for (let attempt = 0; attempt < 2; attempt++) {
+        try {
+            const lockFile = await open(lockFilePath, "wx");
+            try {
+                await lockFile.writeFile(JSON.stringify({
+                    pid: process.pid,
+                    host: hostname(),
+                    startedAt: Date.now(),
+                }));
+                await maybeSync(lockFile);
+            }
+            finally {
+                await lockFile.close();
+            }
+            if (!prefix) {
+                try {
+                    await recover(folderPath);
+                }
+                catch {
+                    // Best-effort at lock time, mirroring `lock`.
+                }
+            }
+            lockDepths.set(resolvedPath, 1);
+            return true;
+        }
+        catch (error) {
+            if (String(error?.message ?? error).split(":")[0] !== "EEXIST")
+                throw error;
+            const lockStat = await stat(lockFilePath).catch(() => null);
+            if (!lockStat)
+                continue; // released between open and stat
+            if (attempt === 0 &&
+                (await stealableLock(lockFilePath, lockStat.mtimeMs, ttl)))
+                await unlink(lockFilePath).catch(() => { });
+            else
+                return false; // genuinely held -> don't wait
+        }
     }
+    return false;
 };
 export const unlock = async (folderPath, prefix) => {
+    const lockFilePath = lockFilePathFor(folderPath, prefix);
+    const resolvedPath = resolve(lockFilePath);
+    const depth = lockDepths.get(resolvedPath);
+    if (depth && depth > 1) {
+        lockDepths.set(resolvedPath, depth - 1);
+        return;
+    }
+    lockDepths.delete(resolvedPath);
     try {
-        await unlink(join(folderPath, `${prefix ?? ""}.locked`));
+        await unlink(lockFilePath);
+    }
+    catch {
+        // Already released (stolen by a foreign-host stealer or cleaned up).
     }
-    catch { }
 };
 export const write = async (filePath, data) => {
-    await writeFile(filePath, filePath.endsWith(".gz") ? await gzip(data) : data);
+    const handle = await open(filePath, "w");
+    try {
+        await handle.writeFile(filePath.endsWith(".gz") ? await gzip(data) : data);
+        await maybeSync(handle);
+    }
+    finally {
+        await handle.close();
+    }
+};
+/**
+ * fsync an existing file. Used to flush temp files (written via streams or
+ * shell pipelines) before they are renamed into place.
+ */
+export const syncFile = async (filePath) => {
+    let handle = null;
+    try {
+        handle = await open(filePath, "r+");
+        await maybeSync(handle);
+    }
+    catch {
+        // Unsupported filesystem/platform: best effort.
+    }
+    finally {
+        await handle?.close();
+    }
+};
+/**
+ * fsync a directory so that renames performed inside it are durable.
+ */
+export const syncDir = async (dirPath) => {
+    let handle = null;
+    try {
+        handle = await open(dirPath, "r");
+        await maybeSync(handle);
+    }
+    catch {
+        // Directory fsync is not supported on every platform/filesystem.
+    }
+    finally {
+        await handle?.close();
+    }
 };
 export const read = async (filePath) => filePath.endsWith(".gz")
     ? (await gunzip(await readFile(filePath, "utf8"))).toString()

package/dist/index.d.ts

@@ -1,4 +1,6 @@
 import "dotenv/config";
+import { type ComputedFieldSpec } from "./expression.js";
+import { type JournalFileOp } from "./journal.js";
 export interface Data {
     id?: string | number;
     [key: string]: any;
@@ -15,6 +17,10 @@ export type Field = {
     unique?: boolean | number | string;
     children?: FieldType | FieldType[] | Schema;
     regex?: string;
+    /** Computed-fields expression (see the README). Either a raw expression
+     *  string, or the persisted `{ expr, ast }` spec written by Inibase once
+     *  the expression has been compiled. */
+    computed?: string | ComputedFieldSpec;
 };
 export type Schema = Field[];
 export interface Options {
@@ -33,6 +39,27 @@ export interface TableObject {
     schema?: Schema;
     config: TableConfig;
 }
+/**
+ * Per-table state maintained while a database transaction is open: the writer
+ * lock is held for the whole transaction, pagination state is staged in
+ * memory (the live files only change at commit()), and `staged` collects the
+ * journaled ops that commit() publishes (one per table per transaction).
+ */
+export interface TxnTableEntry {
+    locked: boolean;
+    /** Live pagination path the first staged op builds on. */
+    paginationFrom: string;
+    /** Staged last-id / row count (chained across ops in the txn). */
+    lastId: number;
+    total: number;
+    staged: {
+        ops: JournalFileOp[];
+        pagination: {
+            from: string;
+            to: string;
+        } | null;
+    }[];
+}
 export type ComparisonOperator = "=" | "!=" | ">" | "<" | ">=" | "<=" | "*" | "!*" | "[]" | "![]";
 export type pageInfo = {
     total?: number;
@@ -51,7 +78,7 @@ declare global {
         entries<T extends object>(o: T): Entries<T>;
     }
 }
-export declare const ERROR_CODES: readonly ["GROUP_UNIQUE", "FIELD_UNIQUE", "FIELD_REQUIRED", "NO_SCHEMA", "TABLE_EMPTY", "INVALID_ID", "INVALID_TYPE", "INVALID_PARAMETERS", "NO_ENV", "TABLE_EXISTS", "TABLE_NOT_EXISTS", "INVALID_REGEX_MATCH", "INVALID_NAME"];
+export declare const ERROR_CODES: readonly ["GROUP_UNIQUE", "FIELD_UNIQUE", "FIELD_REQUIRED", "NO_SCHEMA", "TABLE_EMPTY", "INVALID_ID", "INVALID_TYPE", "INVALID_PARAMETERS", "NO_ENV", "TABLE_EXISTS", "TABLE_NOT_EXISTS", "INVALID_REGEX_MATCH", "INVALID_NAME", "COMPUTED_FIELD_SYNTAX", "COMPUTED_FIELD_UNKNOWN_FIELD", "COMPUTED_FIELD_INVALID_LINK", "COMPUTED_FIELD_INVALID_TARGET", "COMPUTED_FIELD_CONFLICT", "COMPUTED_FIELD_CYCLE", "COMPUTED_FIELD_SETTABLE", "COMPUTED_FIELD_DANGLING_LINK", "COMPUTED_FIELD_ARITHMETIC"];
 export type ErrorCode = (typeof ERROR_CODES)[number];
 export type ErrorLang = "en" | "ar" | "fr" | "es";
 export declare const globalConfig: {
@@ -78,9 +105,22 @@ export default class Inibase {
      *  resolve numeric ids to line numbers arithmetically instead of scanning
      *  the id file. Set false by any partial row deletion. */
     private idDensity;
+    /** Per-table computed-plan cache (null = table has no computed fields).
+     *  A plan depends only on the table's persisted schema (compiled ASTs are
+     *  id-based and rename-proof), so it is rebuilt only when the schema
+     *  changes: see the invalidation in `getTable`'s reload branch,
+     *  `createTable` and `updateTableLocked`. Bounded by the number of tables. */
+    private readonly computedPlanCache;
     private databasePath;
     private uniqueMap;
     private schemaFileExtension;
+    /**
+     * Open database transaction (see begin/commit/rollback). Holds the
+     * database lock (`<db>/.tmp/.locked`) for its whole lifetime and the
+     * per-table writer lock of every table it mutates, so mutations stage into
+     * the database journal and publish only on commit().
+     */
+    private transaction;
     constructor(database: string, mainFolder?: string, language?: ErrorLang);
     createError(name: ErrorCode, variable?: string | number | (string | number)[]): Error;
     private validateName;
@@ -116,6 +156,7 @@ export default class Inibase {
     updateTable(tableName: string, schema?: Schema, config?: TableConfig & {
         name?: string;
     }): Promise<void>;
+    private updateTableLocked;
     /**
      * Get table schema and config
      *
@@ -137,6 +178,57 @@ export default class Inibase {
     private joinPathesContents;
     private _processSchemaDataHelper;
     private processSchemaData;
+    /**
+     * Extract the raw expression from a field's `computed` property (string or
+     * persisted `{ expr, ast }` spec).
+     */
+    private computedExprOf;
+    /**
+     * Compile every `computed` expression of a schema (which must already have
+     * ids assigned) into its persisted `{ expr, ast }` form, in dependency
+     * order. Throws `COMPUTED_FIELD_CYCLE` on cyclic fields. Used by DDL so the
+     * on-disk schema always carries compiled ASTs.
+     */
+    private compileComputedFields;
+    /**
+     * Build the evaluation plan (topological order + id index) for a table's
+     * computed fields. Throws on cycles or unresolvable expressions.
+     */
+    private buildComputedPlan;
+    /** Id index of a table's schema (link targets are re-resolved at write
+     *  time, so renames never retarget a compiled expression). */
+    private tableFieldIndex;
+    /**
+     * Evaluate a batch of (merged) rows against the table's computed fields.
+     * Returns `lineNo -> { computedKey -> value }` in dependency order.
+     *
+     * Batched link-hop reads: when the plan contains any link hop, a dry
+     * collection pass records every (table, column, id) triple the rows'
+     * expressions need (no file I/O), each distinct triple is then resolved
+     * exactly once — deduplicated across rows and fields — and a final pass
+     * evaluates against the warm cache. Plans without hops run the single
+     * evaluation pass unchanged.
+     */
+    private evaluateComputedRows;
+    private createLinkReader;
+    /** Evaluate every computed field of one row (topological order) and merge
+     *  the results back into the row so dependent fields see them. */
+    private evaluateRowComputed;
+    private evaluateNode;
+    /** Evaluate a path (`ids`, dot-separated link hops) against the current
+     *  frame. Returns `null` when an intermediate link value is missing. */
+    private evaluatePath;
+    /** Id index lookup with a shared per-evaluation cache. */
+    private indexFor;
+    /** Read a single column of one linked row via `get`, or null when the
+     *  row does not exist (dangling link). */
+    private readLinkedRow;
+    private coerceNumber;
+    private applyBinaryOp;
+    private aggregate;
+    /** Merge evaluated per-line computed values into a `pathesContents` map
+     *  as line-numbered replace records (encoded cells). */
+    private mergeComputedLineRecords;
     private isSimpleField;
     private processSimpleField;
     /**
@@ -162,6 +254,89 @@ export default class Inibase {
      * @param {string} tableName
      */
     clearCache(tableName: string): Promise<void>;
+    /**
+     * Commit a multi-file mutation crash-atomically:
+     * 1. fsync every freshly-written temp file;
+     * 2. write the journal `begin` entry and fsync it;
+     * 3. rename the pagination metadata file first (atomic publication point:
+     *    the row count flips in a single rename, which is what lock-free
+     *    readers observe) and then swap each live file aside (backup) and
+     *    rename the temp into place;
+     * 4. write the journal `commit` marker and fsync it;
+     * 5. discard backups/temps + the journal, fsync the directories.
+     *
+     * On any failure before `commit`, the journal is rolled back so the table
+     * is left exactly as it was. `renameList` entries are [tempPath, livePath]
+     * pairs; a null tempPath means a pure removal (live file taken out).
+     */
+    private commitFiles;
+    /**
+     * Runs crash recovery for a table (and any crashed database transaction)
+     * before a read. Mutation paths get the same guarantee implicitly (the
+     * writer lock runs recovery on acquire); reads call this explicitly
+     * because they never take the table lock.
+     */
+    private ensureTableRecovered;
+    /**
+     * Blocking database-journal recovery, used by mutation paths (writers are
+     * serialized with live transactions on the database lock anyway).
+     */
+    private ensureDatabaseRecovered;
+    private ensureDatabaseTmpDir;
+    /** Staged per-table entry of the open transaction, or null when none. */
+    private txnTableEntry;
+    /** Lock a table for the open transaction (idempotent per transaction). */
+    private ensureTxnLock;
+    /**
+     * Resolve the pagination state a DML op should build on. Outside a
+     * transaction this reads the live pagination file (as before); inside a
+     * transaction the first touch reads it once and the entry keeps the staged
+     * id/count so chained ops (guarded to one per table) and commit() stay
+     * consistent without publishing anything early.
+     */
+    private resolvePagination;
+    /**
+     * Stage one table mutation into the open transaction: fsync its temps and
+     * append an `op` entry to the database journal (no live file is touched;
+     * commit() performs the actual renames). One staged mutation per table per
+     * transaction (multi-table atomicity; a second touch of the same table
+     * would need read-your-writes composition).
+     */
+    private stageTxnOp;
+    /**
+     * Begin a database transaction. Mutations issued while the transaction is
+     * open (post/put/delete, including cascade deletes) are staged into the
+     * database journal and published atomically at commit(); rollback()
+     * discards them without touching any live file.
+     *
+     * @param tables Optional table names to pre-lock at begin() in sorted
+     * order (the deadlock-free way to span tables). Tables not listed are
+     * locked on first touch, in first-touch order.
+     */
+    begin(tables?: string[]): Promise<void>;
+    /**
+     * Publish every staged mutation atomically: per table (sorted), the
+     * pagination rename comes first (the atomic publication point readers
+     * observe) and then live->backup + tmp->live swaps, before a single fsynced
+     * `commit` marker makes the whole transaction durable. A crash at any
+     * point is recovered by the journal rule (no marker -> roll back all
+     * tables, marker -> roll forward all tables).
+     */
+    commit(): Promise<void>;
+    /**
+     * Discard the open transaction: temps and the journal are removed and no
+     * live file is touched (nothing is published before commit()).
+     */
+    rollback(): Promise<void>;
+    /**
+     * Snapshot the identity (dev:inode:mtime:size) of every column file and
+     * the pagination file. Reading data and then re-verifying this snapshot
+     * lets lock-free readers detect an in-flight writer commit and retry
+     * instead of returning a torn row set.
+     */
+    private snapshotTableFiles;
+    /** True when every snapshotted file is still present and unchanged. */
+    private verifyTableFiles;
     /**
      * Retrieve item(s) from a table
      *
@@ -177,6 +352,7 @@ export default class Inibase {
     get<TData extends Record<string, any> & Partial<Data>>(tableName: string, where?: string | number | (string | number)[] | Criteria, options?: Options, onlyOne?: boolean, onlyLinesNumbers?: false, _whereIsLinesNumbers?: boolean): Promise<(Data & TData)[] | null>;
     get<_TData extends Record<string, any> & Partial<Data>>(tableName: string, where: string | number | (string | number)[] | Criteria | undefined, options: Options | undefined, onlyOne: false | undefined, onlyLinesNumbers: true, _whereIsLinesNumbers?: boolean): Promise<number[] | null>;
     get<_TData extends Record<string, any> & Partial<Data>>(tableName: string, where: string | number | (string | number)[] | Criteria | undefined, options: Options | undefined, onlyOne: true, onlyLinesNumbers: true, _whereIsLinesNumbers?: boolean): Promise<number | null>;
+    private getOnce;
     /**
      * Create new item(s) in a table
      *