inibase 3.6.0 → 3.7.1

package/dist/index.js

@@ -7,9 +7,21 @@ import { inspect } from "node:util";
 import Inison from "inison";
 import { buildFieldIndex, collectFieldDeps, parseExpression, resolveExpression, resolveFramePath, topoSortComputedFields, } from "./expression.js";
 import * as File from "./file.js";
+import { CURRENT_FORMAT_VERSION, FORMAT_FILE, readFormat, serializeFormat, unsupportedFormatReason, writeFormat, } from "./format.js";
 import { DatabaseJournal, Journal } from "./journal.js";
+import { TableOverlay } from "./overlay.js";
+import { buildFrames, DEFAULT_MAX_FRAME_BYTES, UPDATES_LOG_NAME, } from "./updatelog.js";
 import * as Utils from "./utils.js";
 import * as UtilsServer from "./utils.server.js";
+/**
+ * Default accumulation bounds for a format-2 table. Before each append the
+ * writer compacts when the log would exceed `maxLogBytes()` bytes or the
+ * overlay would exceed `maxOverlayRecords()` distinct rows; the log resets to
+ * empty on compaction, so these bound *between* compactions, not total writes.
+ * Both are overridable per deployment via environment for large tables.
+ */
+const maxLogBytes = () => Number(process.env.INIBASE_MAX_LOG_BYTES) || 64 * 1024 * 1024;
+const maxOverlayRecords = () => Number(process.env.INIBASE_MAX_OVERLAY_RECORDS) || 100_000;
 export const ERROR_CODES = [
     "GROUP_UNIQUE",
     "FIELD_UNIQUE",
@@ -33,6 +45,10 @@ export const ERROR_CODES = [
     "COMPUTED_FIELD_SETTABLE",
     "COMPUTED_FIELD_DANGLING_LINK",
     "COMPUTED_FIELD_ARITHMETIC",
+    "UPDATE_LOG_CORRUPT",
+    "UPDATE_LOG_FRAME_TOO_LARGE",
+    "MAINTENANCE_REQUIRED",
+    "UNSUPPORTED_FORMAT",
 ];
 // hide ExperimentalWarning glob()
 // Guard against non-Node environments (e.g. accidental import in a browser bundle)
@@ -110,6 +126,18 @@ function astHasHops(node) {
             return astHasHops(node.arg);
     }
 }
+/**
+ * Await a batch of column-write promises and surface the first rejection.
+ * `Promise.allSettled` never short-circuits, so a swallowed failure would let
+ * the caller publish a partial column set — the pagination row count and the
+ * column line counts would diverge. Every batch that fills a `renameList` must
+ * be inspected through here before the journal/pagination is touched.
+ */
+async function settleWrites(results) {
+    const failed = results.find((result) => result.status === "rejected");
+    if (failed)
+        throw failed.reason;
+}
 /**
  * @param {string} database - Database name
  * @param {string} [mainFolder="."] - Main folder path
@@ -131,6 +159,15 @@ export default class Inibase {
      *  changes: see the invalidation in `getTable`'s reload branch,
      *  `createTable` and `updateTableLocked`. Bounded by the number of tables. */
     computedPlanCache = new Map();
+    /** Per-table `.format` marker + live update-log overlays (format 2 tables).
+     *  Keyed by `databasePath\0tableName` so two databases in one process never
+     *  share state. Populated lazily by `getOverlay`. */
+    tableFormats = new Map();
+    overlays = new Map();
+    /** Last observed identity of each table's `.format` file (`mtime:size:ino`),
+     *  used to detect a compaction by another process without re-reading the
+     *  marker on every access. */
+    formatStamps = new Map();
     databasePath;
     uniqueMap;
     schemaFileExtension = process.env.INIBASE_SCHEMA_EXTENSION ?? "json";
@@ -253,6 +290,23 @@ export default class Inibase {
                 await File.write(join(tablePath, ".prepend.config"), "");
             if (config.decodeID)
                 await File.write(join(tablePath, ".decodeID.config"), "");
+            if (config.log) {
+                if (config.prepend)
+                    throw this.createError("UNSUPPORTED_FORMAT", "append layout (no prepend)");
+                if (schema)
+                    this.assertLogCompatibleSchema(schema);
+                await File.write(join(tablePath, UPDATES_LOG_NAME), "");
+                await writeFormat(tablePath, {
+                    format: CURRENT_FORMAT_VERSION,
+                    log: true,
+                    baseGen: 0,
+                });
+                this.tableFormats.set(tableName, {
+                    format: CURRENT_FORMAT_VERSION,
+                    log: true,
+                    baseGen: 0,
+                });
+            }
         }
         if (schema) {
             const lastSchemaID = { value: 0 };
@@ -327,6 +381,34 @@ export default class Inibase {
         }
     }
     async updateTableLocked(tableName, table, tablePath, schema, config) {
+        // A log table cannot be reshaped in place yet: its log frames reference
+        // column names and its overlay would have to be materialised into the
+        // base columns first (compaction, 3.9). Rename/other config are fine.
+        if (this.isLogTable(tableName)) {
+            if (schema)
+                throw this.createError("MAINTENANCE_REQUIRED", `${tableName} (schema change on update-log table)`);
+            if (config?.prepend)
+                throw this.createError("MAINTENANCE_REQUIRED", `${tableName} (prepend layout on update-log table)`);
+            if (config?.log === false)
+                throw this.createError("MAINTENANCE_REQUIRED", `${tableName} (disabling the update log needs compaction)`);
+        }
+        else if (config?.log) {
+            if (config.prepend || table.config.prepend)
+                throw this.createError("UNSUPPORTED_FORMAT", "append layout (no prepend)");
+            if (table.schema?.length)
+                this.assertLogCompatibleSchema(table.schema);
+            await File.write(join(tablePath, UPDATES_LOG_NAME), "");
+            await writeFormat(tablePath, {
+                format: CURRENT_FORMAT_VERSION,
+                log: true,
+                baseGen: 0,
+            });
+            this.tableFormats.set(tableName, {
+                format: CURRENT_FORMAT_VERSION,
+                log: true,
+                baseGen: 0,
+            });
+        }
         if (schema) {
             this.validateSchema(schema);
             // remove id from schema
@@ -531,6 +613,13 @@ export default class Inibase {
         this.computedPlanCache.delete(tableName);
         if (config?.name && config.name !== tableName)
             this.computedPlanCache.delete(config.name);
+        // Drop log/format caches for the touched table(s) so next access re-reads
+        // the (possibly moved) marker and overlay from disk.
+        for (const name of config?.name ? [tableName, config.name] : [tableName]) {
+            this.tableFormats.delete(name);
+            this.formatStamps.delete(name);
+            this.overlays.delete(this.overlayKey(name));
+        }
     }
     /**
      * Get table schema and config
@@ -545,22 +634,313 @@ export default class Inibase {
             throw this.createError("TABLE_NOT_EXISTS", tableName);
         if (!globalConfig[this.databasePath].tables?.has(tableName) ||
             globalConfig[this.databasePath].tables?.get(tableName)?.timestamp !==
-                (await File.getFileDate(join(tablePath, `schema.${this.schemaFileExtension}`))))
+                (await File.getFileDate(join(tablePath, `schema.${this.schemaFileExtension}`)))) {
+            const format = await this.readValidatedFormat(tableName, tablePath);
+            this.tableFormats.set(tableName, format);
+            await this.stampFormat(tableName);
+            const config = {
+                compression: await File.isExists(join(tablePath, ".compression.config")),
+                cache: await File.isExists(join(tablePath, ".cache.config")),
+                prepend: await File.isExists(join(tablePath, ".prepend.config")),
+                decodeID: await File.isExists(join(tablePath, ".decodeID.config")),
+            };
+            // Only surface `log` when enabled, so the public config shape of a
+            // legacy table is byte-for-byte what it was before 3.8.
+            if (format.log)
+                config.log = true;
             globalConfig[this.databasePath].tables?.set(tableName, {
                 schema: await this.getTableSchema(tableName),
-                config: {
-                    compression: await File.isExists(join(tablePath, ".compression.config")),
-                    cache: await File.isExists(join(tablePath, ".cache.config")),
-                    prepend: await File.isExists(join(tablePath, ".prepend.config")),
-                    decodeID: await File.isExists(join(tablePath, ".decodeID.config")),
-                },
+                config,
                 timestamp: await File.getFileDate(join(tablePath, `schema.${this.schemaFileExtension}`)),
             });
+        }
         // A reload means the on-disk schema changed (create/update/external
         // edit): the cached computed plan is derived from it.
         this.computedPlanCache.delete(tableName);
         return globalConfig[this.databasePath].tables?.get(tableName);
     }
+    /* -----------------------------------------------------------------------
+     * Update-log overlay (format 2 tables)
+     * --------------------------------------------------------------------- */
+    overlayKey(tableName) {
+        return `${this.databasePath}\u0000${tableName}`;
+    }
+    /** True when the table opted into the incremental update log. */
+    isLogTable(tableName) {
+        const cached = this.tableFormats.get(tableName);
+        if (cached)
+            return cached.log;
+        return (globalConfig[this.databasePath].tables?.get(tableName)?.config.log ===
+            true);
+    }
+    /**
+     * Resolve a table's format marker, revalidating a cached *log* table when
+     * its `.format` file changed underneath us. A compaction by another process
+     * bumps `baseGen` and empties the log; without re-reading the marker this
+     * process would scan the new (empty) log with the old `baseGen`, and once a
+     * post-compaction frame appears the generation gap would look like
+     * corruption. Legacy tables never change format implicitly, so they skip
+     * the stat entirely and keep the pre-3.8 read path cost.
+     */
+    /**
+     * Read a table's `.format` marker and refuse a table written by a newer
+     * engine (§4.1) *before* any column is read: a higher `format`, or a
+     * `minReader` above this build, throws UNSUPPORTED_FORMAT rather than being
+     * silently mis-read as format 2/legacy.
+     */
+    async readValidatedFormat(tableName, tablePath) {
+        const format = await readFormat(tablePath);
+        const reason = unsupportedFormatReason(format);
+        if (reason)
+            throw this.createError("UNSUPPORTED_FORMAT", `${tableName} (${reason})`);
+        return format;
+    }
+    async currentFormat(tableName) {
+        const cached = this.tableFormats.get(tableName);
+        if (cached && !cached.log)
+            return cached;
+        const tablePath = join(this.databasePath, tableName);
+        let stamp = "none";
+        try {
+            const stat_ = await stat(join(tablePath, FORMAT_FILE));
+            stamp = `${stat_.mtimeMs}:${stat_.size}:${stat_.ino}`;
+        }
+        catch {
+            stamp = "none";
+        }
+        if (cached && this.formatStamps.get(tableName) === stamp)
+            return cached;
+        const format = await this.readValidatedFormat(tableName, tablePath);
+        this.tableFormats.set(tableName, format);
+        this.formatStamps.set(tableName, stamp);
+        // A changed baseGen invalidates the cached overlay: rebuild from the
+        // (possibly now empty) log at the new generation.
+        const key = this.overlayKey(tableName);
+        const overlay = this.overlays.get(key);
+        if (overlay && overlay.baseGen !== format.baseGen)
+            this.overlays.delete(key);
+        return format;
+    }
+    /**
+     * Load (or refresh) a table's live overlay. Returns null for legacy tables
+     * (no `.format` marker) so every existing code path stays untouched. With
+     * `forWrite`, repairs a torn tail and rebuilds to the committed boundary;
+     * the caller must already hold the table writer lock.
+     */
+    async getOverlay(tableName, forWrite = false) {
+        const format = await this.currentFormat(tableName);
+        if (!format?.log)
+            return null;
+        const key = this.overlayKey(tableName);
+        let overlay = this.overlays.get(key);
+        if (!overlay || overlay.baseGen !== format.baseGen) {
+            overlay = new TableOverlay(join(this.databasePath, tableName), format.baseGen);
+            this.overlays.set(key, overlay);
+        }
+        if (forWrite)
+            await overlay.refreshForWrite();
+        else
+            await overlay.refresh();
+        return overlay;
+    }
+    async stampFormat(tableName) {
+        let stamp = "none";
+        try {
+            const stat_ = await stat(join(this.databasePath, tableName, FORMAT_FILE));
+            stamp = `${stat_.mtimeMs}:${stat_.size}:${stat_.ino}`;
+        }
+        catch {
+            stamp = "none";
+        }
+        this.formatStamps.set(tableName, stamp);
+    }
+    /**
+     * The format-2 update log stores one encoded cell per changed *top-level
+     * simple* column; nested (object/array/table) and computed fields are not
+     * representable without a compaction step (3.9). A log table whose schema
+     * violates this is rejected at DDL time so the write path can always take
+     * the log and never needs to interleave a whole-column rewrite.
+     */
+    assertLogCompatibleSchema(schema) {
+        for (const field of schema) {
+            if (["id", "createdAt", "updatedAt"].includes(field.key))
+                continue;
+            if (typeof field.computed !== "undefined")
+                throw this.createError("UNSUPPORTED_FORMAT", `simple top-level fields (computed field "${field.key}" is not supported with log)`);
+            if (!this.isSimpleField(field.type))
+                throw this.createError("UNSUPPORTED_FORMAT", `simple top-level fields (field "${field.key}" is ${field.type})`);
+        }
+    }
+    idFieldOf(tableName) {
+        const schema = globalConfig[this.databasePath].tables?.get(tableName)?.schema ?? [];
+        return (schema.find((field) => field.key === "id") ?? { key: "id", type: "id" });
+    }
+    /** Read the internal numeric id for each requested line (stable overlay key). */
+    async overlayLineIds(tableName, lines) {
+        const map = new Map();
+        if (!lines.length)
+            return map;
+        const filePath = join(this.databasePath, tableName, `id${this.getFileExtension(tableName)}`);
+        // Streaming reader (no shell spawn): like the sparse column read path,
+        // stop at the last requested line instead of shelling out to `sed -n`,
+        // which costs ~ms per call on a log-table read.
+        const values = await File.getLines(filePath, lines, {
+            ...this.idFieldOf(tableName),
+            key: "id",
+            type: "number",
+        });
+        for (const [line, value] of Object.entries(values))
+            map.set(Number(line), Number(value));
+        return map;
+    }
+    /** Internal id -> current line number(s), built from the id column. */
+    async overlayIdToLines(tableName) {
+        const filePath = join(this.databasePath, tableName, `id${this.getFileExtension(tableName)}`);
+        const values = await File.get(filePath, undefined, {
+            ...this.idFieldOf(tableName),
+            key: "id",
+            type: "number",
+        });
+        const map = new Map();
+        for (const [line, value] of Object.entries(values)) {
+            if (typeof value === "undefined" || value === null)
+                continue;
+            const id = Number(value);
+            const bucket = map.get(id);
+            if (bucket)
+                bucket.push(Number(line));
+            else
+                map.set(id, [Number(line)]);
+        }
+        return map;
+    }
+    /**
+     * Per-line raw-cell overrides for one column, from the overlay. Used by
+     * `File.search` so criteria, uniqueness and aggregate scans see the merged
+     * (un-compacted) values. Returns undefined when the table has no overlay or
+     * the column is never overlaid.
+     */
+    async overlayColumnOverrides(tableName, columnKey) {
+        const overlay = await this.getOverlay(tableName);
+        if (!overlay || overlay.size === 0)
+            return undefined;
+        const entries = [];
+        for (const [id, entry] of overlay.byId)
+            if (Object.hasOwn(entry.cols, columnKey))
+                entries.push([Number(id), entry.cols[columnKey]]);
+        if (!entries.length)
+            return undefined;
+        const idToLines = await this.overlayIdToLines(tableName);
+        const overrides = new Map();
+        for (const [id, encoded] of entries) {
+            const lines = idToLines.get(id);
+            if (!lines)
+                continue;
+            for (const line of lines)
+                overrides.set(line, encoded);
+        }
+        return overrides.size ? overrides : undefined;
+    }
+    /**
+     * Fold every logged update into the base columns and reset the log.
+     *
+     * Safe to call on any table: legacy tables (no `.format` marker) are a
+     * no-op. Semantically neutral — it only materializes already-acknowledged
+     * updates, so a failure never makes an acknowledged `put` disappear.
+     * Returns the new `baseGen` for a compacted table, or `null` when there was
+     * nothing to fold.
+     */
+    async compact(tableName) {
+        this.validateName(tableName);
+        if (this.activeTransaction)
+            throw this.createError("MAINTENANCE_REQUIRED", `${tableName} (compaction inside a transaction)`);
+        await this.ensureDatabaseRecovered();
+        const tablePath = join(this.databasePath, tableName);
+        if (!(await File.isExists(tablePath)))
+            throw this.createError("TABLE_NOT_EXISTS", tableName);
+        const format = await this.currentFormat(tableName);
+        if (!format?.log)
+            return null; // legacy table: nothing to fold
+        await File.lock(join(tablePath, ".tmp"));
+        try {
+            return await this.compactLogTable(tableName);
+        }
+        finally {
+            await File.unlock(join(tablePath, ".tmp"));
+        }
+    }
+    /**
+     * Compaction core. The caller must hold the table writer lock. Streams each
+     * overlaid base column through `File.replace` (which handles compression),
+     * then publishes the new columns, an empty `updates.log`, and a `.format`
+     * marker with the bumped generation in one journaled commit, so a crash
+     * either rolls every file back (old base + old log + old `baseGen`) or
+     * forward (new base + empty log + new `baseGen`). No acknowledged update is
+     * ever lost.
+     */
+    async compactLogTable(tableName) {
+        const tablePath = join(this.databasePath, tableName);
+        const overlay = await this.getOverlay(tableName, true);
+        if (!overlay || overlay.size === 0)
+            return null; // already compact
+        const schema = (await this.getTable(tableName))?.schema ?? [];
+        const schemaKeys = new Set(schema.map((field) => field.key));
+        // Only columns that some frame actually overlaid need rewriting; the
+        // untouched base columns are left exactly as they are.
+        const overlaidKeys = new Set();
+        for (const entry of overlay.byId.values())
+            for (const key of Object.keys(entry.cols))
+                if (schemaKeys.has(key))
+                    overlaidKeys.add(key);
+        if (!overlaidKeys.size)
+            return null;
+        const idToLines = await this.overlayIdToLines(tableName);
+        const extension = this.getFileExtension(tableName);
+        const renameList = [];
+        for (const key of overlaidKeys) {
+            const record = {};
+            for (const [id, entry] of overlay.byId) {
+                if (!Object.hasOwn(entry.cols, key))
+                    continue;
+                const lines = idToLines.get(Number(id));
+                if (!lines)
+                    continue;
+                const encoded = entry.cols[key];
+                for (const line of lines)
+                    record[line] = encoded;
+            }
+            if (!Object.keys(record).length)
+                continue;
+            renameList.push(await File.replace(join(tablePath, `${key}${extension}`), record));
+        }
+        if (!renameList.length)
+            return null;
+        const newGen = overlay.committedGen;
+        const staging = (name) => join(tablePath, ".tmp", `${name}.compact-${process.pid}-${randomUUID().slice(0, 8)}`);
+        // Empty log (frames are now folded into base) and the bumped marker,
+        // committed atomically with the column swaps.
+        const logTmp = staging(UPDATES_LOG_NAME);
+        await File.write(logTmp, "");
+        renameList.push([logTmp, join(tablePath, UPDATES_LOG_NAME)]);
+        const fmtTmp = staging(FORMAT_FILE);
+        await File.write(fmtTmp, serializeFormat({
+            format: CURRENT_FORMAT_VERSION,
+            log: true,
+            baseGen: newGen,
+        }));
+        renameList.push([fmtTmp, join(tablePath, FORMAT_FILE)]);
+        await this.commitFiles(tablePath, renameList, null);
+        // In-memory: the base now carries every update; drop the overlay so the
+        // next access rebuilds an empty one at the new generation.
+        this.tableFormats.set(tableName, {
+            format: CURRENT_FORMAT_VERSION,
+            log: true,
+            baseGen: newGen,
+        });
+        await this.stampFormat(tableName);
+        this.overlays.delete(this.overlayKey(tableName));
+        return newGen;
+    }
     async getTableSchema(tableName) {
         this.validateName(tableName);
         const tablePath = join(this.databasePath, tableName);
@@ -811,7 +1191,7 @@ export default class Inibase {
                 if (!field)
                     continue;
                 fieldsKeys.push(field.key);
-                const [_, totalLines, lineNumbers] = await File.search(join(tablePath, `${field.key}${this.getFileExtension(tableName)}`), "[]", Array.from(values), undefined, valueObject.exclude, { ...field, databasePath: this.databasePath }, 1, undefined, false);
+                const [_, totalLines, lineNumbers] = await File.search(join(tablePath, `${field.key}${this.getFileExtension(tableName)}`), "[]", Array.from(values), undefined, valueObject.exclude, { ...field, databasePath: this.databasePath }, 1, undefined, false, undefined, await this.overlayColumnOverrides(tableName, field.key));
                 if (totalLines > 0) {
                     if (valueObject.columnsValues.size === 1 ||
                         (valueObject.columnsValues.size === index &&
@@ -982,19 +1362,39 @@ export default class Inibase {
     }
     async processSchemaData(tableName, schema, linesNumber, options, prefix) {
         const RETURN = {};
-        // Fast path: read every top-level simple column in one `paste | sed`
-        // child process instead of spawning one child process per column.
-        // Nested (prefixed) schemas keep the per-column path.
+        // Sparse path: for an explicit finite selection, stream each top-level
+        // simple column only up to the last requested line (never to EOF) and
+        // decode it independently, instead of pasting every column through a
+        // shell pipeline that always scans the whole table and can misalign on
+        // values containing a literal tab. Nested (prefixed) schemas keep the
+        // per-column path below.
         let batchedKeys = null;
         if (!prefix && linesNumber?.length) {
             const simpleFields = schema.filter((field) => this.isSimpleField(field.type));
-            const batched = await this.processSimpleFieldsBatch(tableName, simpleFields, linesNumber);
-            if (batched) {
-                batchedKeys = new Set(simpleFields.map((field) => field.key));
-                for (const [line, row] of Object.entries(batched)) {
+            const decodeID = globalConfig[this.databasePath].tables?.get(tableName)?.config
+                .decodeID === true;
+            const extension = this.getFileExtension(tableName);
+            const reads = await Promise.all(simpleFields.map(async (field) => {
+                const fieldPath = join(this.databasePath, tableName, `${field.key}${extension}`);
+                if (!(await File.isExists(fieldPath)))
+                    return null;
+                const items = await File.getLines(fieldPath, linesNumber, {
+                    ...field,
+                    type: field.key === "id" && decodeID ? "number" : field.type,
+                    databasePath: this.databasePath,
+                });
+                return { key: field.key, items };
+            }));
+            batchedKeys = new Set(simpleFields.map((field) => field.key));
+            for (const read of reads) {
+                if (!read)
+                    continue;
+                for (const [line, value] of Object.entries(read.items)) {
+                    if (typeof value === "undefined")
+                        continue;
                     if (!RETURN[line])
                         RETURN[line] = {};
-                    Object.assign(RETURN[line], row);
+                    Object.assign(RETURN[line], { [read.key]: value });
                 }
             }
         }
@@ -1019,11 +1419,58 @@ export default class Inibase {
                 await this.processTableField(tableName, field, RETURN, linesNumber, options, prefix);
             }
         }
+        await this.mergeOverlayIntoRows(tableName, schema, RETURN);
         return RETURN;
     }
-    /* -----------------------------------------------------------------------
-     * Computed fields
-     * --------------------------------------------------------------------- */
+    /**
+     * Fold the table's un-compacted update-log overlay onto decoded rows.
+     *
+     * `RETURN` is keyed by line number and its `id` is the user-facing id, so
+     * the overlay (keyed by the internal numeric id) is matched through a fresh
+     * id-column read of just the requested lines. Only columns present in
+     * `schema` (which is already narrowed to the requested `columns`) are
+     * written, so a partial projection never grows extra keys. Legacy tables
+     * (no `.format` marker) return immediately.
+     */
+    async mergeOverlayIntoRows(tableName, schema, RETURN) {
+        if (!this.isLogTable(tableName))
+            return;
+        const lines = Object.keys(RETURN).map(Number);
+        if (!lines.length)
+            return;
+        const overlay = await this.getOverlay(tableName);
+        if (!overlay || overlay.size === 0)
+            return;
+        const lineIds = await this.overlayLineIds(tableName, lines);
+        if (!lineIds.size)
+            return;
+        const allowed = new Map();
+        for (const field of schema)
+            allowed.set(field.key, field);
+        const hasUpdatedAt = allowed.has("updatedAt");
+        for (const line of lines) {
+            const id = lineIds.get(line);
+            if (id === undefined)
+                continue;
+            const entry = overlay.get(id);
+            if (!entry)
+                continue;
+            const row = RETURN[line];
+            if (!row)
+                continue;
+            for (const [key, encoded] of Object.entries(entry.cols)) {
+                const field = allowed.get(key);
+                if (!field)
+                    continue;
+                row[key] = File.decode(encoded, {
+                    ...field,
+                    databasePath: this.databasePath,
+                });
+            }
+            if (hasUpdatedAt && entry.updatedAt !== undefined)
+                row.updatedAt = entry.updatedAt;
+        }
+    }
     /**
      * Extract the raw expression from a field's `computed` property (string or
      * persisted `{ expr, ast }` spec).
@@ -1658,78 +2105,6 @@ export default class Inibase {
             }
         }
     }
-    /**
-     * Batched read for top-level simple columns: a single `paste | sed` child
-     * process returns every requested line of every column at once instead of
-     * spawning one `sed`/`gunzip|sed` child process per column. Returning null
-     * (e.g. when any column file is missing, or the shell command fails) makes
-     * the caller fall back to the existing per-column reads.
-     */
-    async processSimpleFieldsBatch(tableName, fields, linesNumber) {
-        if (!fields.length || !linesNumber.length)
-            return null;
-        const decodeID = globalConfig[this.databasePath].tables?.get(tableName)?.config
-            .decodeID === true;
-        // Decode configs are computed once per column (the per-column read path
-        // builds one config per file too), not once per cell.
-        const cols = [];
-        for (const field of fields) {
-            const path = join(this.databasePath, tableName, `${field.key}${this.getFileExtension(tableName)}`);
-            if (!(await File.isExists(path)))
-                return null;
-            cols.push({
-                path,
-                key: field.key,
-                config: {
-                    ...field,
-                    type: field.key === "id" && decodeID ? "number" : field.type,
-                    databasePath: this.databasePath,
-                },
-            });
-        }
-        const sortedLines = [...linesNumber].sort((a, b) => a - b);
-        const range = File._groupIntoRanges(sortedLines);
-        const files = cols.map(({ path }) => File.escapeShellPath(path)).join(" ");
-        const isGz = this.getFileExtension(tableName).endsWith(".gz");
-        // Each compressed column needs its own process substitution so the
-        // streams stay aligned column-by-column inside `paste`.
-        const pasteInputs = isGz
-            ? cols
-                .map(({ path }) => `<(gunzip -c ${File.escapeShellPath(path)})`)
-                .join(" ")
-            : files;
-        const command = isGz
-            ? `bash -c 'paste -d "\\t" ${pasteInputs} | sed -n "${range}"'`
-            : `paste -d'\\t' ${files} | sed -n '${range}'`;
-        let output;
-        try {
-            output = (await UtilsServer.exec(command)).stdout;
-        }
-        catch {
-            return null;
-        }
-        const outLines = output.trimEnd().split("\n");
-        const RETURN = {};
-        for (let i = 0; i < outLines.length && i < sortedLines.length; i++) {
-            const lineNo = sortedLines[i];
-            const cells = outLines[i].split("\t");
-            const row = {};
-            let added = false;
-            for (let c = 0; c < cols.length; c++) {
-                const raw = cells[c];
-                if (raw === undefined)
-                    continue;
-                const value = File.decode(raw, cols[c].config);
-                if (value !== undefined) {
-                    row[cols[c].key] = value;
-                    added = true;
-                }
-            }
-            if (added)
-                RETURN[lineNo] = row;
-        }
-        return Object.keys(RETURN).length ? RETURN : null;
-    }
     // Helper function to check if the field type is array
     isArrayField(fieldType) {
         return ((Array.isArray(fieldType) &&
@@ -2011,7 +2386,7 @@ export default class Inibase {
                 }, (options.perPage ?? 0) < 0 ? undefined : options.perPage, (options.perPage ?? 0) < 0
                     ? undefined
                     : (options.page - 1) * options.perPage +
-                        ((options.page ?? 1) > 1 ? 1 : 0), true);
+                        ((options.page ?? 1) > 1 ? 1 : 0), true, undefined, await this.overlayColumnOverrides(tableName, key));
                 if (!searchResult) {
                     if (allTrue)
                         return null;
@@ -2581,6 +2956,14 @@ export default class Inibase {
                 (Utils.isObject(where) && !Object.keys(where).length)))
             where = undefined;
         if (options.sort) {
+            // With an un-compacted overlay the shell `paste | sort | awk` path
+            // would read stale base cells. Fall back to an in-process sort of
+            // the merged rows, which is correct for every sort shape.
+            if (this.isLogTable(tableName)) {
+                const overlay = await this.getOverlay(tableName);
+                if (overlay?.size)
+                    return await this.getOnceSortedWithOverlay(tableName, schema, where, options, onlyOne);
+            }
             let sortArray;
             let awkCommand = "";
             if (Utils.isObject(options.sort) && !Array.isArray(options.sort)) {
@@ -2768,7 +3151,11 @@ export default class Inibase {
                 }
             }
             if (!lineNumbers) {
-                [lineNumbers, countItems] = await File.search(join(tablePath, `id${this.getFileExtension(tableName)}`), "[]", Ids.map((id) => Utils.isNumber(id) ? Number(id) : UtilsServer.decodeID(id)), undefined, undefined, { key: "BLABLA", type: "number" }, Ids.length, 0, !this.totalItems.has(`${tableName}-id`));
+                [lineNumbers, countItems] = await File.search(join(tablePath, `id${this.getFileExtension(tableName)}`), "[]", Ids.map((id) => Utils.isNumber(id) ? Number(id) : UtilsServer.decodeID(id)), undefined, undefined, { key: "BLABLA", type: "number" }, Ids.length, 0, 
+                // The caller only needs the matching line numbers, not a
+                // whole-table count: stop as soon as every requested id has
+                // been found instead of scanning the column to EOF.
+                false, true);
             }
             if (!lineNumbers)
                 return null;
@@ -2788,8 +3175,15 @@ export default class Inibase {
         }
         else if (Utils.isObject(where)) {
             let cachedFilePath = "";
+            // The criteria cache is versioned by the row count only; a put
+            // changes matching rows without changing the count, so a cached
+            // line set would go stale. Bypass it entirely while an overlay
+            // exists.
+            const overlayActive = this.isLogTable(tableName) &&
+                ((await this.getOverlay(tableName))?.size ?? 0) > 0;
             // Criteria
-            if (globalConfig[this.databasePath].tables?.get(tableName)?.config.cache) {
+            if (!overlayActive &&
+                globalConfig[this.databasePath].tables?.get(tableName)?.config.cache) {
                 // Cache entries are versioned by the pagination row count so a
                 // stale cache written before a post/delete (in this process or
                 // another) is detectable: the candidate filename simply stops
@@ -2824,7 +3218,8 @@ export default class Inibase {
                 for (const [key] of this.totalItems)
                     if (key.startsWith(`${tableName}-`) && key !== `${tableName}-id`)
                         this.totalItems.delete(key);
-                if (globalConfig[this.databasePath].tables?.get(tableName)?.config.cache)
+                if (!overlayActive &&
+                    globalConfig[this.databasePath].tables?.get(tableName)?.config.cache)
                     await writeFile(cachedFilePath, Object.keys(LineNumberDataObj).join(","));
             }
         }
@@ -2846,6 +3241,70 @@ export default class Inibase {
         };
         return onlyOne && Array.isArray(RETURN) ? RETURN[0] : RETURN;
     }
+    /**
+     * In-process sort fallback for log tables with a non-empty overlay: the
+     * shell `paste | sort | awk` path reads base cells and would miss puts.
+     * Reads the merged rows for the (optionally filtered) lines, sorts them by
+     * the requested keys with type-aware comparison, then paginates.
+     */
+    async getOnceSortedWithOverlay(tableName, schema, where, options, onlyOne) {
+        let sortArray;
+        if (Utils.isObject(options.sort) && !Array.isArray(options.sort))
+            sortArray = Object.entries(options.sort).map(([key, value]) => [
+                key,
+                typeof value === "string" ? value.toLowerCase() === "asc" : value > 0,
+            ]);
+        else
+            sortArray = []
+                .concat(options.sort)
+                .map((column) => [column, true]);
+        let lines;
+        if (where) {
+            const found = await this.get(tableName, where, undefined, undefined, true);
+            if (!found)
+                return null;
+            lines = Array.isArray(found) ? found : [found];
+        }
+        else {
+            const { total } = await this.resolvePagination(tableName);
+            lines = Array.from({ length: total }, (_, index) => index + 1);
+        }
+        if (!lines.length)
+            return null;
+        const rows = Object.values(await this.processSchemaData(tableName, schema, lines, options));
+        const comparator = (a, b) => {
+            for (const [key, ascending] of sortArray) {
+                const field = Utils.getField(key, Utils.flattenSchema(schema));
+                const aValue = a?.[key];
+                const bValue = b?.[key];
+                let comparison;
+                if (field && Utils.isFieldType(field, ["id", "number", "date"]))
+                    comparison = (Number(aValue) || 0) - (Number(bValue) || 0);
+                else
+                    comparison = String(aValue ?? "").localeCompare(String(bValue ?? ""));
+                if (comparison)
+                    return ascending ? comparison : -comparison;
+            }
+            // Deterministic tiebreak (legacy `sort` leaves ties unspecified).
+            return (Number(a?.id) || 0) - (Number(b?.id) || 0);
+        };
+        rows.sort(comparator);
+        const total = rows.length;
+        const perPage = options.perPage;
+        const pageRows = perPage < 0
+            ? rows
+            : rows.slice((options.page - 1) * perPage, options.page * perPage);
+        if (!pageRows.length)
+            return null;
+        this.totalItems.set(`${tableName}-*`, total);
+        this.pageInfo[tableName] = {
+            ...(({ columns, ...restOfOptions }) => restOfOptions)(options),
+            perPage: pageRows.length,
+            totalPages: perPage < 0 ? 1 : Math.ceil(total / perPage),
+            total,
+        };
+        return onlyOne ? pageRows[0] : pageRows;
+    }
     async post(tableName, data, options, returnPostedData) {
         return File.runWithLockStore(() => this._post(tableName, data, options, returnPostedData));
     }
@@ -2915,10 +3374,10 @@ export default class Inibase {
                     ? clonedData.toReversed()
                     : clonedData
                 : clonedData);
-            await Promise.allSettled(Object.entries(pathesContents).map(async ([path, content]) => renameList.push(globalConfig[this.databasePath].tables?.get(tableName)?.config
+            await settleWrites(await Promise.allSettled(Object.entries(pathesContents).map(async ([path, content]) => renameList.push(globalConfig[this.databasePath].tables?.get(tableName)?.config
                 .prepend
                 ? await File.prepend(path, content)
-                : await File.append(path, content))));
+                : await File.append(path, content)))));
             const newTotal = _totalItems + (Array.isArray(data) ? data.length : 1);
             const pagination = {
                 from: paginationFilePath,
@@ -2994,6 +3453,119 @@ export default class Inibase {
     }, returnUpdatedData, _whereIsLinesNumbers) {
         return File.runWithLockStore(() => this._put(tableName, data, where, options, returnUpdatedData, _whereIsLinesNumbers));
     }
+    /**
+     * Incremental `put` for format-2 (log) tables.
+     *
+     * Resolves the target rows to their stable internal ids, validates the
+     * payload against the merged view (uniqueness included), and appends one
+     * bounded frame instead of rewriting whole columns. The base files are left
+     * untouched; readers merge the overlay. Returns `undefined` when nothing
+     * matches, matching the legacy `put` contract.
+     */
+    async putViaLog(tableName, data, where, options = { page: 1, perPage: 15 }, returnUpdatedData, _whereIsLinesNumbers) {
+        const tablePath = join(this.databasePath, tableName);
+        const schema = globalConfig[this.databasePath].tables?.get(tableName)?.schema ?? [];
+        const payloads = Array.isArray(data) ? data : [data];
+        const isWhereLess = !where;
+        const isLines = _whereIsLinesNumbers === true;
+        const scalarWhere = !isWhereLess && !isLines && !Array.isArray(where);
+        await this.throwErrorIfTableEmpty(tableName);
+        let overlay = null;
+        let resolvedLines = [];
+        let inlineLock = false;
+        try {
+            await File.lock(join(tablePath, ".tmp"));
+            inlineLock = true;
+            // Writer-side refresh: repairs a torn tail so the next frame is
+            // never appended behind unreadable bytes, and gives us the current
+            // committed generation.
+            overlay = await this.getOverlay(tableName, true);
+            if (!overlay)
+                return undefined;
+            // Backpressure before the write: if the log or the overlay has grown
+            // past its bound, fold it into the base first. Compaction is
+            // maintenance (it only materializes acknowledged frames), so a
+            // failure surfaces as MAINTENANCE_REQUIRED without losing any
+            // committed update.
+            if (overlay.size >= maxOverlayRecords() ||
+                (await overlay.log.bytes()) >= maxLogBytes()) {
+                try {
+                    await this.compactLogTable(tableName);
+                }
+                catch (error) {
+                    throw this.createError("MAINTENANCE_REQUIRED", `${tableName} (compaction failed: ${error?.message ?? error})`);
+                }
+                overlay = await this.getOverlay(tableName, true);
+                if (overlay.size >= maxOverlayRecords())
+                    throw this.createError("MAINTENANCE_REQUIRED", `${tableName} (overlay still exceeds ${maxOverlayRecords()} rows after compaction)`);
+            }
+            if (isWhereLess) {
+                const { total } = await this.resolvePagination(tableName);
+                resolvedLines = Array.from({ length: total }, (_, index) => index + 1);
+            }
+            else if (isLines) {
+                resolvedLines = Array.isArray(where)
+                    ? where
+                    : [where];
+            }
+            else {
+                const lines = await this.get(tableName, where, undefined, undefined, true);
+                if (!lines)
+                    return undefined;
+                resolvedLines = Array.isArray(lines)
+                    ? lines
+                    : [lines];
+            }
+            if (!resolvedLines.length)
+                return undefined;
+            const lineIds = await this.overlayLineIds(tableName, resolvedLines);
+            // Validate against the merged view: `checkUnique` searches the base
+            // columns through `File.search` overrides, so this sees un-compacted
+            // puts. Required fields are skipped (partial update).
+            await this.validateTableData(tableName, structuredClone(payloads), true);
+            const now = Date.now();
+            const records = [];
+            for (let index = 0; index < resolvedLines.length; index++) {
+                const id = lineIds.get(resolvedLines[index]);
+                if (id === undefined)
+                    continue;
+                const payload = payloads.length === 1
+                    ? payloads[0]
+                    : (payloads[index % payloads.length] ?? payloads[0]);
+                const formatted = this.formatData(payload, schema, true);
+                const cols = {};
+                for (const [key, value] of Object.entries(formatted)) {
+                    if (key === "id" || key === "createdAt" || key === "updatedAt")
+                        continue;
+                    // Only columns the caller actually supplied are overlaid, so a
+                    // partial `put` never blanks untouched fields.
+                    if (!Object.hasOwn(payload, key))
+                        continue;
+                    cols[key] = File.encode(value);
+                }
+                cols.updatedAt = now;
+                records.push({ id, cols, updatedAt: now });
+            }
+            if (records.length) {
+                const frames = buildFrames(records, overlay.committedGen + 1, DEFAULT_MAX_FRAME_BYTES, overlay.log.path);
+                for (const frame of frames)
+                    await overlay.log.append(frame);
+                await overlay.applyLocal(frames);
+            }
+        }
+        finally {
+            if (inlineLock)
+                await File.unlock(join(tablePath, ".tmp"));
+        }
+        if (returnUpdatedData) {
+            if (isWhereLess)
+                return await this.get(tableName, undefined, options);
+            if (scalarWhere)
+                return await this.get(tableName, resolvedLines[0], options, true, undefined, true);
+            return await this.get(tableName, resolvedLines, options, undefined, undefined, true);
+        }
+        return undefined;
+    }
     async _put(tableName, data, where, options = {
         page: 1,
         perPage: 15,
@@ -3009,6 +3581,16 @@ export default class Inibase {
         if (!this.activeTransaction)
             await this.ensureDatabaseRecovered();
         await this.throwErrorIfTableEmpty(tableName);
+        // Format-2 tables route every put through the append-only log, which
+        // is proportional to the changed data. Transactions and DDL need a
+        // compaction that 3.9 ships, so they are refused loudly rather than
+        // silently falling back to a whole-column rewrite that would leave the
+        // overlay shadowing newer base values.
+        if (this.isLogTable(tableName)) {
+            if (this.activeTransaction)
+                throw this.createError("MAINTENANCE_REQUIRED", `${tableName} (put inside a transaction on update-log table)`);
+            return this.putViaLog(tableName, data, where, options, returnUpdatedData, _whereIsLinesNumbers);
+        }
         let clonedData = structuredClone(data);
         if (!where) {
             if (Utils.isArrayOfObjects(clonedData)) {
@@ -3060,7 +3642,7 @@ export default class Inibase {
                     const evaluated = await this.evaluateComputedRows(tableName, computedPlan, mergedRows);
                     this.mergeComputedLineRecords(tableName, computedPlan, evaluated, pathesContents, existing);
                 }
-                await Promise.allSettled(Object.entries(pathesContents).map(async ([path, content]) => renameList.push(await File.replace(path, content, total))));
+                await settleWrites(await Promise.allSettled(Object.entries(pathesContents).map(async ([path, content]) => renameList.push(await File.replace(path, content, total)))));
                 if (this.activeTransaction) {
                     // Stage instead of publishing: row count is unchanged so
                     // there is no pagination rename to journal.
@@ -3157,7 +3739,7 @@ export default class Inibase {
                     }
                     this.mergeComputedLineRecords(tableName, computedPlan, evaluated, pathesContents, existing);
                 }
-                await Promise.allSettled(Object.entries(pathesContents).map(async ([path, content]) => renameList.push(await File.replace(path, content))));
+                await settleWrites(await Promise.allSettled(Object.entries(pathesContents).map(async ([path, content]) => renameList.push(await File.replace(path, content)))));
                 if (this.activeTransaction) {
                     await this.stageTxnOp(tableName, renameList, null);
                     txnStaged = true;
@@ -3252,6 +3834,13 @@ export default class Inibase {
             await this.ensureDatabaseRecovered();
         const tablePath = join(this.databasePath, tableName);
         await this.throwErrorIfTableEmpty(tableName);
+        if (this.isLogTable(tableName) && this.activeTransaction)
+            throw this.createError("MAINTENANCE_REQUIRED", `${tableName} (delete inside a transaction on update-log table)`);
+        // Fold any pending puts into the base before a tombstone rewrite, so the
+        // log is empty and the rewritten base carries every acknowledged value.
+        // `compact` is a no-op when there is nothing to fold.
+        if (this.isLogTable(tableName))
+            await this.compact(tableName);
         if (!where) {
             let txnStaged = false;
             // Crash-atomic truncate: park every column file (pure removal
@@ -3330,7 +3919,7 @@ export default class Inibase {
                     const remaining = total - (Array.isArray(where) ? where.length : 1);
                     if (total && remaining > 0) {
                         this.idDensity.set(tableName, false);
-                        await Promise.allSettled(files.map(async (file) => renameList.push(await File.remove(join(tablePath, file), where))));
+                        await settleWrites(await Promise.allSettled(files.map(async (file) => renameList.push(await File.remove(join(tablePath, file), where)))));
                         const pagination = {
                             from: paginationFilePath,
                             to: join(tablePath, `${lastId}-${remaining}.pagination`),
@@ -3529,11 +4118,11 @@ export default class Inibase {
                     // the default page size (15): every matching row counts.
                     const lineNumbers = await this.get(tableName, where, { perPage: -1 }, undefined, true);
                     RETURN[column] = lineNumbers
-                        ? await File.sum(columnPath, lineNumbers)
+                        ? await File.sum(columnPath, lineNumbers, await this.overlayColumnOverrides(tableName, column))
                         : 0;
                 }
                 else
-                    RETURN[column] = await File.sum(columnPath);
+                    RETURN[column] = await File.sum(columnPath, undefined, await this.overlayColumnOverrides(tableName, column));
             }
         }
         return columns.length > 1 ? RETURN : Object.values(RETURN)[0];
@@ -3718,11 +4307,11 @@ export default class Inibase {
                 if (where) {
                     const lineNumbers = await this.get(tableName, where, { perPage: -1 }, undefined, true);
                     RETURN[column] = lineNumbers
-                        ? await File.avg(columnPath, lineNumbers)
+                        ? await File.avg(columnPath, lineNumbers, await this.overlayColumnOverrides(tableName, column))
                         : 0;
                 }
                 else
-                    RETURN[column] = await File.avg(columnPath);
+                    RETURN[column] = await File.avg(columnPath, undefined, await this.overlayColumnOverrides(tableName, column));
             }
         }
         return columns.length > 1 ? RETURN : Object.values(RETURN)[0];
@@ -3742,11 +4331,11 @@ export default class Inibase {
                 if (where) {
                     const lineNumbers = await this.get(tableName, where, { perPage: -1 }, undefined, true);
                     RETURN[column] = lineNumbers
-                        ? await File.max(columnPath, lineNumbers)
+                        ? await File.max(columnPath, lineNumbers, await this.overlayColumnOverrides(tableName, column))
                         : 0;
                 }
                 else
-                    RETURN[column] = await File.max(columnPath);
+                    RETURN[column] = await File.max(columnPath, undefined, await this.overlayColumnOverrides(tableName, column));
             }
         }
         return RETURN;
@@ -3766,11 +4355,11 @@ export default class Inibase {
                 if (where) {
                     const lineNumbers = await this.get(tableName, where, { perPage: -1 }, undefined, true);
                     RETURN[column] = lineNumbers
-                        ? await File.min(columnPath, lineNumbers)
+                        ? await File.min(columnPath, lineNumbers, await this.overlayColumnOverrides(tableName, column))
                         : 0;
                 }
                 else
-                    RETURN[column] = await File.min(columnPath);
+                    RETURN[column] = await File.min(columnPath, undefined, await this.overlayColumnOverrides(tableName, column));
             }
         }
         return RETURN;