@feltdb/core 0.11.6 → 0.11.8

package/dist/file-db.js

@@ -4,6 +4,33 @@ import * as nodeFs from 'fs';
 import * as nodePath from 'path';
 import { decideTransaction, embeddedResult, EmbeddedTransactionRejected, } from './embedded-transaction.js';
 import { EmbeddedQueryEngine } from './embedded-query.js';
+import { encodeRecord, scanJournal, JournalCorruptionError, PersistenceProfiler, } from './file-journal.js';
+export { JournalCorruptionError } from './file-journal.js';
+/** The newest local file snapshot format this build reads and writes. */
+export const FILE_SNAPSHOT_FORMAT_VERSION = 2;
+/**
+ * The runtime directory was written in a durable format newer than this build
+ * understands.
+ *
+ * Refused rather than recovered around: treating a newer snapshot as invalid
+ * and falling back to an older backup — or to an empty database — silently
+ * discards every commit the newer format carries. That is exactly what a
+ * package-only rollback onto upgraded data would do, so it fails closed.
+ */
+export class UnsupportedDurableFormatError extends Error {
+    statePath;
+    found;
+    code = 'UNSUPPORTED_DURABLE_FORMAT';
+    constructor(statePath, found) {
+        super(`UNSUPPORTED_DURABLE_FORMAT: ${statePath} is in local durable format ${found}; this FeltDB `
+            + `build reads formats 1-${FILE_SNAPSHOT_FORMAT_VERSION}. It was written by a newer FeltDB. `
+            + 'The database was not opened and was not modified. Upgrade FeltDB, or restore the backup '
+            + 'taken before the upgrade.');
+        this.statePath = statePath;
+        this.found = found;
+        this.name = 'UnsupportedDurableFormatError';
+    }
+}
 // Synchronous versions for sync API (will throw if called in browser)
 function getFsSync() {
     if (typeof nodeFs.existsSync === 'function')
@@ -15,11 +42,24 @@ function getPathSync() {
         return nodePath;
     throw new Error('FileJsDb is only available in Node.js environments');
 }
+const DEFAULT_MIN_JOURNAL_BYTES = 8 * 1024 * 1024;
+const JOURNAL_FILE = /^state\.(\d+)\.journal$/;
+function isMissing(error) {
+    return error?.code === 'ENOENT';
+}
+function writeFully(fd, bytes, position) {
+    let written = 0;
+    while (written < bytes.length) {
+        written += getFsSync().writeSync(fd, bytes, written, bytes.length - written, position + written);
+    }
+}
 export class FileJsDb {
     dataPath;
     statePath;
+    backupPath;
     lockPath;
     state;
+    view;
     origin;
     peers = new Set();
     lockFd = null;
@@ -27,7 +67,10 @@ export class FileJsDb {
     applied = new Set();
     queryEngine;
     indexedSequence = -1;
-    constructor(path) {
+    profiler;
+    compaction;
+    compactionScheduled = false;
+    constructor(path, options = {}) {
         const nodePath = getPathSync();
         const nodeFs = getFsSync();
         // Normalize and resolve the path to prevent directory traversal
@@ -38,7 +81,14 @@ export class FileJsDb {
         }
         this.dataPath = resolvedPath;
         this.statePath = nodePath.join(this.dataPath, 'state.json');
+        this.backupPath = `${this.statePath}.backup`;
         this.lockPath = nodePath.join(this.dataPath, 'state.lock');
+        this.profiler = new PersistenceProfiler(options.profile ?? (typeof process !== 'undefined' && process.env?.FELTDB_PERSISTENCE_PROFILE === '1'));
+        this.compaction = {
+            mode: options.compaction?.mode ?? 'auto',
+            minJournalBytes: options.compaction?.minJournalBytes ?? DEFAULT_MIN_JOURNAL_BYTES,
+            journalToSnapshotRatio: options.compaction?.journalToSnapshotRatio ?? 1,
+        };
         // Create directory if it doesn't exist
         if (!nodeFs.existsSync(this.dataPath)) {
             nodeFs.mkdirSync(this.dataPath, { recursive: true });
@@ -48,80 +98,80 @@ export class FileJsDb {
         // Clean up any stale locks from previous crashes on startup
         this.cleanupStaleLock();
         // Load or initialize state
-        this.state = this.loadState();
-        this.restorePeers();
-        for (const transactionId of this.state.applied_transactions ?? []) {
-            this.applied.add(transactionId);
-        }
+        this.state = this.initializeState();
+        this.loadDurableState();
         this.queryEngine = new EmbeddedQueryEngine(() => Object.entries(this.state.rows));
         this.indexedSequence = this.state.sequence;
     }
-    loadState() {
+    /* ------------------------------------------------------------------------
+     * Durable layout
+     * ---------------------------------------------------------------------- */
+    journalPath(generation) {
+        return getPathSync().join(this.dataPath, `state.${generation}.journal`);
+    }
+    snapshotKey() {
+        try {
+            const stat = getFsSync().statSync(this.statePath, { bigint: true });
+            return `${stat.ino}:${stat.size}:${stat.mtimeNs}`;
+        }
+        catch (error) {
+            if (isMissing(error))
+                return 'none';
+            throw error;
+        }
+    }
+    parseSnapshot(path) {
+        const content = getFsSync().readFileSync(path, 'utf-8');
+        const snapshot = JSON.parse(content);
+        if (typeof snapshot.version === 'number' && snapshot.version > FILE_SNAPSHOT_FORMAT_VERSION) {
+            throw new UnsupportedDurableFormatError(path, snapshot.version);
+        }
+        const known = snapshot.version === 1
+            || (snapshot.version === 2 && Number.isInteger(snapshot.journal?.generation) && Number.isInteger(snapshot.journal?.commit));
+        if (!known || !snapshot.meta || !snapshot.rows || !snapshot.events)
+            return null;
+        return { snapshot, bytes: Buffer.byteLength(content) };
+    }
+    /**
+     * Read the snapshot this handle should start from: the primary, or the
+     * backup when the primary cannot be read (unchanged from the single-file
+     * runtime), or nothing at all.
+     */
+    readSnapshot() {
         const nodeFs = getFsSync();
-        const backupPath = `${this.statePath}.backup`;
-        // Load atomic snapshot if it exists
         if (nodeFs.existsSync(this.statePath)) {
             try {
-                const content = nodeFs.readFileSync(this.statePath, 'utf-8');
-                const snapshot = JSON.parse(content);
-                // Validate snapshot structure
-                if (snapshot.version !== 1 || !snapshot.meta || !snapshot.rows || !snapshot.events) {
-                    console.warn(`Invalid snapshot structure at ${this.statePath}, attempting backup recovery`);
-                    return this.loadStateFromBackup();
-                }
-                // Successfully loaded valid state, preserve it as backup
-                try {
-                    nodeFs.copyFileSync(this.statePath, backupPath);
-                }
-                catch {
-                    // Backup failure is non-fatal
-                }
-                return {
-                    version: snapshot.version,
-                    instance_id: this.origin,
-                    sequence: snapshot.meta.sequence,
-                    rows: snapshot.rows,
-                    events: snapshot.events,
-                    peers: snapshot.meta.peers ?? [],
-                    sequences: snapshot.meta.sequences ?? {},
-                    applied_transactions: snapshot.applied_transactions ?? []
-                };
+                const parsed = this.parseSnapshot(this.statePath);
+                if (parsed)
+                    return { snapshot: parsed.snapshot, bytes: parsed.bytes, source: 'primary' };
+                console.warn(`Invalid snapshot structure at ${this.statePath}, attempting backup recovery`);
             }
             catch (error) {
-                console.warn(`Failed to load snapshot from ${this.statePath}: ${error}`);
-                console.warn(`Attempting to recover from backup...`);
-                return this.loadStateFromBackup();
+                if (error instanceof UnsupportedDurableFormatError)
+                    throw error;
+                if (!isMissing(error)) {
+                    console.warn(`Failed to load snapshot from ${this.statePath}: ${error}`);
+                    console.warn(`Attempting to recover from backup...`);
+                }
+                else {
+                    return { snapshot: null, bytes: 0, source: 'empty' };
+                }
             }
-        }
-        return this.initializeState();
-    }
-    loadStateFromBackup() {
-        const nodeFs = getFsSync();
-        const backupPath = `${this.statePath}.backup`;
-        if (nodeFs.existsSync(backupPath)) {
             try {
-                const content = nodeFs.readFileSync(backupPath, 'utf-8');
-                const snapshot = JSON.parse(content);
-                if (snapshot.version === 1 && snapshot.meta && snapshot.rows && snapshot.events) {
-                    console.warn(`Recovered state from backup at ${backupPath}`);
-                    return {
-                        version: snapshot.version,
-                        instance_id: this.origin,
-                        sequence: snapshot.meta.sequence,
-                        rows: snapshot.rows,
-                        events: snapshot.events,
-                        peers: snapshot.meta.peers ?? [],
-                        sequences: snapshot.meta.sequences ?? {},
-                        applied_transactions: snapshot.applied_transactions ?? []
-                    };
+                const parsed = this.parseSnapshot(this.backupPath);
+                if (parsed) {
+                    console.warn(`Recovered state from backup at ${this.backupPath}`);
+                    return { snapshot: parsed.snapshot, bytes: parsed.bytes, source: 'backup' };
                 }
             }
-            catch {
+            catch (error) {
+                if (error instanceof UnsupportedDurableFormatError)
+                    throw error;
                 // Backup is also corrupted or invalid
             }
+            console.warn(`Backup recovery failed, reinitializing state`);
         }
-        console.warn(`Backup recovery failed, reinitializing state`);
-        return this.initializeState();
+        return { snapshot: null, bytes: 0, source: 'empty' };
     }
     initializeState() {
         return {
@@ -135,11 +185,540 @@ export class FileJsDb {
             applied_transactions: []
         };
     }
-    restorePeers() {
-        for (const peer of this.state.peers) {
+    /**
+     * Materialize the durable state from scratch: snapshot, then every complete
+     * commit in its journal, in order. A journal that disappears while it is
+     * being read has been superseded by a compaction, so the read starts over
+     * from the new snapshot. One that is missing for the current snapshot is
+     * corruption.
+     */
+    loadDurableState() {
+        const started = this.profiler.now();
+        for (let attempt = 0;; attempt += 1) {
+            const snapshotKey = this.snapshotKey();
+            const { snapshot, bytes, source } = this.readSnapshot();
+            const state = snapshot ? {
+                version: snapshot.version,
+                instance_id: this.origin,
+                sequence: snapshot.meta.sequence,
+                rows: snapshot.rows,
+                events: snapshot.events,
+                peers: snapshot.meta.peers ?? [],
+                sequences: snapshot.meta.sequences ?? {},
+                applied_transactions: snapshot.applied_transactions ?? [],
+            } : this.initializeState();
+            const view = {
+                snapshotKey,
+                format: snapshot ? (snapshot.version === 2 ? 'v2' : 'v1') : 'none',
+                source,
+                generation: snapshot?.journal?.generation ?? 0,
+                commit: snapshot?.journal?.commit ?? 0,
+                journalOffset: 0,
+                journalRecords: 0,
+                snapshotBytes: bytes,
+                tornBytes: 0,
+            };
+            let records = [];
+            if (view.format === 'v2') {
+                const journalPath = this.journalPath(view.generation);
+                let journal;
+                try {
+                    journal = getFsSync().readFileSync(journalPath);
+                }
+                catch (error) {
+                    if (!isMissing(error))
+                        throw error;
+                    if (attempt < 8 && this.snapshotKey() !== snapshotKey)
+                        continue;
+                    if (source === 'backup') {
+                        console.warn(`Backup snapshot journal ${journalPath} is missing, reinitializing state`);
+                        this.adopt(this.initializeState(), { ...view, format: 'none', source: 'empty', generation: 0, commit: 0 });
+                        this.profiler.record('recovery', started);
+                        return;
+                    }
+                    throw new JournalCorruptionError(journalPath, 1, 0, 'the journal named by the current snapshot is missing');
+                }
+                let scan;
+                try {
+                    scan = scanJournal(journal, {
+                        journalPath, generation: view.generation, expectedCommit: view.commit + 1,
+                    });
+                }
+                catch (error) {
+                    // A reader holds no lock, so a writer repairing a torn tail can be
+                    // observed mid-way. Real corruption is still there on a re-read.
+                    if (attempt < 3 && error instanceof JournalCorruptionError)
+                        continue;
+                    throw error;
+                }
+                records = scan.records;
+                view.journalOffset = scan.consumed;
+                view.journalRecords = scan.records.length;
+                view.tornBytes = scan.tornBytes;
+                view.commit += scan.records.length;
+            }
+            if (source === 'backup') {
+                // The unreadable primary was the compaction of this backup and its
+                // journal, so the journals written after it continue the backup's
+                // commits exactly. Replaying them recovers everything that primary
+                // and its journal held, instead of stopping at the backup.
+                for (let generation = view.generation + 1;; generation += 1) {
+                    const journalPath = this.journalPath(generation);
+                    let journal;
+                    try {
+                        journal = getFsSync().readFileSync(journalPath);
+                    }
+                    catch (error) {
+                        if (isMissing(error))
+                            break;
+                        throw error;
+                    }
+                    const scan = scanJournal(journal, {
+                        journalPath, generation, expectedCommit: view.commit + 1,
+                    });
+                    records.push(...scan.records);
+                    view.format = 'v2';
+                    view.generation = generation;
+                    view.journalOffset = scan.consumed;
+                    view.journalRecords = scan.records.length;
+                    view.tornBytes = scan.tornBytes;
+                    view.commit += scan.records.length;
+                }
+            }
+            this.adopt(state, view);
+            for (const record of records)
+                this.applyCommit(record.body, false);
+            this.profiler.record('recovery', started);
+            return;
+        }
+    }
+    adopt(state, view) {
+        this.state = state;
+        this.view = view;
+        this.peers.clear();
+        for (const peer of state.peers)
             this.peers.add(peer);
+        this.applied.clear();
+        for (const transactionId of state.applied_transactions ?? [])
+            this.applied.add(transactionId);
+    }
+    /** Apply one commit to memory. The only way any mutation reaches memory. */
+    applyCommit(body, index) {
+        for (const mutation of body.m ?? []) {
+            const [key] = mutation;
+            if (mutation.length === 1)
+                delete this.state.rows[key];
+            else
+                this.state.rows[key] = mutation[1];
+            if (index)
+                this.queryEngine?.apply(key, mutation.length === 1 ? undefined : mutation[1]);
+        }
+        for (const event of body.e ?? [])
+            this.state.events.push(event);
+        if (body.q)
+            Object.assign(this.state.sequences, body.q);
+        if (body.p) {
+            this.state.peers = [...body.p];
+            this.peers.clear();
+            for (const peer of body.p)
+                this.peers.add(peer);
+        }
+        if (body.t !== undefined && !this.applied.has(body.t)) {
+            this.applied.add(body.t);
+            this.state.applied_transactions.push(body.t);
+        }
+        this.state.sequence = body.s;
+        if (index)
+            this.indexedSequence = this.state.sequence;
+    }
+    /**
+     * Bring memory up to date with the durable files.
+     *
+     * While the snapshot is unchanged, only journal bytes after the last
+     * complete record already applied are read, so a read costs what was
+     * committed since the previous read — not the size of all history. Any
+     * change to the snapshot, or a journal that no longer continues what was
+     * applied, falls back to a complete load.
+     */
+    refresh() {
+        const key = this.snapshotKey();
+        if (key === this.view.snapshotKey) {
+            if (this.view.format !== 'v2')
+                return;
+            if (this.tailJournal())
+                return;
+        }
+        this.loadDurableState();
+        this.queryEngine.rebuild();
+        this.indexedSequence = this.state.sequence;
+    }
+    /** Apply newly committed journal records. False when a full load is required. */
+    tailJournal() {
+        const nodeFs = getFsSync();
+        const journalPath = this.journalPath(this.view.generation);
+        let fd;
+        try {
+            fd = nodeFs.openSync(journalPath, 'r');
+        }
+        catch (error) {
+            if (isMissing(error))
+                return false;
+            throw error;
+        }
+        try {
+            for (let attempt = 0;; attempt += 1) {
+                const size = nodeFs.fstatSync(fd).size;
+                if (size < this.view.journalOffset)
+                    return false;
+                const length = size - this.view.journalOffset;
+                if (length === 0) {
+                    this.view.tornBytes = 0;
+                    return true;
+                }
+                const bytes = Buffer.allocUnsafe(length);
+                let read = 0;
+                while (read < length) {
+                    const count = nodeFs.readSync(fd, bytes, read, length - read, this.view.journalOffset + read);
+                    if (count === 0)
+                        break;
+                    read += count;
+                }
+                let scan;
+                try {
+                    scan = scanJournal(bytes.subarray(0, read), {
+                        journalPath,
+                        generation: this.view.generation,
+                        expectedCommit: this.view.commit + 1,
+                        baseOffset: this.view.journalOffset,
+                        firstRecord: this.view.journalRecords + 1,
+                    });
+                }
+                catch (error) {
+                    if (attempt < 3 && error instanceof JournalCorruptionError && this.lockToken === null)
+                        continue;
+                    throw error;
+                }
+                for (const record of scan.records)
+                    this.applyCommit(record.body, true);
+                this.view.journalOffset += scan.consumed;
+                this.view.journalRecords += scan.records.length;
+                this.view.commit += scan.records.length;
+                this.view.tornBytes = scan.tornBytes;
+                return true;
+            }
+        }
+        finally {
+            nodeFs.closeSync(fd);
+        }
+    }
+    /* ------------------------------------------------------------------------
+     * Commit protocol
+     * ---------------------------------------------------------------------- */
+    /** A commit body starting from the current sequence. */
+    stage() {
+        return { s: this.state.sequence };
+    }
+    stageRow(body, key, value) {
+        (body.m ??= []).push(value === undefined ? [key] : [key, value]);
+    }
+    stageEvent(body, collection, key, type, value, timestamp = Date.now()) {
+        body.s += 1;
+        (body.e ??= []).push({
+            sequence: body.s,
+            collection,
+            key,
+            type,
+            value,
+            timestamp,
+            origin: this.origin,
+            id: `${this.origin}:${timestamp}:${body.s}`,
+        });
+    }
+    /** Stage a row write together with the event that records it. */
+    stagePut(body, key, value) {
+        this.stageRow(body, key, value);
+        this.stageEvent(body, key.split(':')[0], key, 'put', value);
+    }
+    /**
+     * Make one commit durable, then visible. Caller holds the lock and has
+     * refreshed, so memory reflects every earlier commit.
+     *
+     *   1  body framed and written, without its terminator
+     *   2  fdatasync                    ← the record's bytes are durable
+     *   3  terminator written           ← the commit boundary exists
+     *   4  fdatasync                    ← the boundary is durable
+     *   5  applied to memory, success returned
+     *
+     * A crash before 3 leaves an unterminated record that recovery discards; it
+     * was never acknowledged. Readers only accept terminated records, so no
+     * process can observe a commit whose bytes were not already durable. A
+     * failure at any step removes whatever was written, so a failed commit is
+     * not visible after it returns either.
+     */
+    persist(body) {
+        const nodeFs = getFsSync();
+        if (this.view.format !== 'v2' || this.view.source !== 'primary') {
+            // First write to a fresh, legacy, or backup-recovered directory
+            // establishes a journal-continued snapshot of the current state.
+            this.compactLocked();
+        }
+        const commitStarted = this.profiler.now();
+        const commit = this.view.commit + 1;
+        const serializationStarted = this.profiler.now();
+        const { header, body: bodyBytes } = encodeRecord(this.view.generation, commit, body);
+        const record = Buffer.concat([header, bodyBytes]);
+        this.profiler.record('serialization', serializationStarted);
+        const offset = this.view.journalOffset;
+        const fd = nodeFs.openSync(this.journalPath(this.view.generation), 'r+');
+        try {
+            try {
+                if (this.view.tornBytes > 0) {
+                    // An earlier writer died mid-append. Its record was never
+                    // acknowledged; remove it before anything is written after it.
+                    console.warn(`FeltDB recovered an incomplete journal append at byte ${offset} `
+                        + `(${this.view.tornBytes} bytes discarded) in ${this.journalPath(this.view.generation)}`);
+                    nodeFs.ftruncateSync(fd, offset);
+                    this.timedSync(fd);
+                    this.view.tornBytes = 0;
+                }
+                let started = this.profiler.now();
+                writeFully(fd, record, offset);
+                this.profiler.record('write', started);
+                this.timedSync(fd);
+                started = this.profiler.now();
+                writeFully(fd, Buffer.from('\n'), offset + record.length);
+                this.profiler.record('write', started);
+                this.timedSync(fd);
+            }
+            catch (error) {
+                try {
+                    nodeFs.ftruncateSync(fd, offset);
+                    nodeFs.fdatasyncSync(fd);
+                }
+                catch {
+                    // The unterminated remainder is discarded by the next writer.
+                }
+                throw new Error(`Failed to save FeltDB state: ${error}`);
+            }
+        }
+        finally {
+            nodeFs.closeSync(fd);
+        }
+        this.view.journalOffset = offset + record.length + 1;
+        this.view.journalRecords += 1;
+        this.view.commit = commit;
+        this.profiler.bytes('journal', record.length + 1);
+        // Memory receives exactly the bytes that were made durable.
+        this.applyCommit(JSON.parse(bodyBytes.toString('utf8')), true);
+        this.profiler.record('commit', commitStarted);
+        this.scheduleCompaction();
+    }
+    timedSync(fd) {
+        const started = this.profiler.now();
+        getFsSync().fdatasyncSync(fd);
+        this.profiler.record('fsync', started);
+    }
+    /* ------------------------------------------------------------------------
+     * Compaction
+     * ---------------------------------------------------------------------- */
+    compactionDue() {
+        return this.view.format === 'v2'
+            && this.view.journalOffset >= this.compaction.minJournalBytes
+            && this.view.journalOffset >= this.view.snapshotBytes * this.compaction.journalToSnapshotRatio;
+    }
+    /**
+     * Compaction runs after the commit that made it due has been acknowledged,
+     * never as part of it, and under the same lock as every other writer.
+     */
+    scheduleCompaction() {
+        if (this.compaction.mode !== 'auto' || this.compactionScheduled || !this.compactionDue())
+            return;
+        this.compactionScheduled = true;
+        const timer = setTimeout(() => {
+            this.compactionScheduled = false;
+            try {
+                this.withMutationLock(() => { if (this.compactionDue())
+                    this.compactLocked(); });
+            }
+            catch (error) {
+                // Committed data is already durable in the journal; compaction only
+                // reclaims it. A failed attempt is retried after a later commit.
+                console.warn(`FeltDB journal compaction failed: ${error}`);
+            }
+        }, 0);
+        timer.unref?.();
+    }
+    /**
+     * Rewrite the current state as a new snapshot and start an empty journal.
+     * Caller holds the lock and has refreshed.
+     *
+     *   1  new empty journal G+1 created and synced
+     *   2  snapshot naming (G+1, commit C) written to a temporary file, synced
+     *   3  previous snapshot linked as the backup
+     *   4  snapshot renamed into place, directory synced   ← compaction commits
+     *   5  journals other than G and G+1 removed
+     *
+     * A crash before 4 leaves the previous snapshot and journal authoritative;
+     * an orphaned journal G+1 is never read and is removed by the next
+     * compaction. After 4 the new snapshot is complete by itself.
+     */
+    compactLocked() {
+        const nodeFs = getFsSync();
+        const started = this.profiler.now();
+        const previousGeneration = this.view.generation;
+        let generation = previousGeneration;
+        for (const name of nodeFs.readdirSync(this.dataPath)) {
+            const match = JOURNAL_FILE.exec(name);
+            if (match)
+                generation = Math.max(generation, Number(match[1]));
+        }
+        generation += 1;
+        const commit = this.view.commit;
+        const journal = nodeFs.openSync(this.journalPath(generation), 'w');
+        try {
+            nodeFs.fsyncSync(journal);
+        }
+        finally {
+            nodeFs.closeSync(journal);
+        }
+        // The journal's directory entry must be durable before a snapshot names it.
+        this.syncDirectory();
+        this.state.peers = [...this.peers];
+        const snapshot = {
+            version: FILE_SNAPSHOT_FORMAT_VERSION,
+            timestamp: Date.now(),
+            meta: {
+                instance_id: this.origin,
+                sequence: this.state.sequence,
+                peers: this.state.peers,
+                sequences: this.state.sequences,
+            },
+            rows: this.state.rows,
+            events: this.state.events,
+            applied_transactions: [...this.applied],
+            journal: { generation, commit },
+        };
+        // Compaction is profiled as one phase, so that it never inflates the
+        // serialization, write and fsync figures of ordinary commits.
+        const content = Buffer.from(JSON.stringify(snapshot), 'utf8');
+        const tempPath = `${this.statePath}.${process.pid}.${Math.random().toString(36).slice(2)}.tmp`;
+        try {
+            const handle = nodeFs.openSync(tempPath, 'w');
+            try {
+                writeFully(handle, content, 0);
+                // Force the contents to disk before the rename publishes them.
+                nodeFs.fsyncSync(handle);
+            }
+            finally {
+                nodeFs.closeSync(handle);
+            }
+            if (this.view.source === 'primary' && nodeFs.existsSync(this.statePath)) {
+                // The snapshot being replaced becomes the backup. Its journal is kept
+                // below, so the backup remains a complete state on its own.
+                const backupTemp = `${this.backupPath}.${process.pid}.tmp`;
+                try {
+                    nodeFs.unlinkSync(backupTemp);
+                }
+                catch { /* absent */ }
+                try {
+                    nodeFs.linkSync(this.statePath, backupTemp);
+                }
+                catch {
+                    nodeFs.copyFileSync(this.statePath, backupTemp);
+                }
+                nodeFs.renameSync(backupTemp, this.backupPath);
+            }
+            // Atomic rename commits the new snapshot.
+            nodeFs.renameSync(tempPath, this.statePath);
+            this.syncDirectory();
+        }
+        catch (error) {
+            try {
+                nodeFs.unlinkSync(tempPath);
+            }
+            catch { /* absent */ }
+            throw new Error(`Failed to save FeltDB state: ${error}`);
+        }
+        for (const name of nodeFs.readdirSync(this.dataPath)) {
+            const match = JOURNAL_FILE.exec(name);
+            if (!match)
+                continue;
+            const kept = Number(match[1]);
+            if (kept === generation || kept === previousGeneration)
+                continue;
+            try {
+                nodeFs.unlinkSync(getPathSync().join(this.dataPath, name));
+            }
+            catch { /* already gone */ }
         }
+        this.view = {
+            snapshotKey: this.snapshotKey(),
+            format: 'v2',
+            source: 'primary',
+            generation,
+            commit,
+            journalOffset: 0,
+            journalRecords: 0,
+            snapshotBytes: content.length,
+            tornBytes: 0,
+        };
+        this.profiler.bytes('snapshot', content.length);
+        this.profiler.record('compaction', started);
+    }
+    syncDirectory() {
+        const nodeFs = getFsSync();
+        // Persist the rename itself, so the commit survives power loss.
+        try {
+            const directory = nodeFs.openSync(this.dataPath, 'r');
+            try {
+                nodeFs.fsyncSync(directory);
+            }
+            finally {
+                nodeFs.closeSync(directory);
+            }
+        }
+        catch {
+            // Directory fsync is not available on every platform. The rename is
+            // still atomic; only the durability of the rename itself is weaker.
+        }
+    }
+    /**
+     * Fold the journal into a new snapshot now. Logical state is unchanged;
+     * only its physical representation is. Normal commits never do this.
+     */
+    compact() {
+        this.withMutationLock(() => this.compactLocked());
+        return this.persistence_status();
     }
+    /** How the durable state is currently represented on disk. */
+    persistence_status() {
+        this.refresh();
+        const view = this.view;
+        return {
+            format: view.format === 'v2' ? 'snapshot+journal' : view.format === 'v1' ? 'legacy-snapshot' : 'none',
+            source: view.source,
+            generation: view.generation,
+            commit: view.commit,
+            sequence: this.state.sequence,
+            snapshotBytes: view.snapshotBytes,
+            journalBytes: view.journalOffset,
+            journalRecords: view.journalRecords,
+            recovery: view.tornBytes > 0
+                ? { tornTail: { byteOffset: view.journalOffset, discardedBytes: view.tornBytes } }
+                : 'clean',
+        };
+    }
+    /**
+     * Persistence timings collected by this handle: mutation, serialization,
+     * write, fsync, commit, compaction and recovery. Empty unless profiling was
+     * enabled with the `profile` option or `FELTDB_PERSISTENCE_PROFILE=1`.
+     */
+    persistence_profile(options = {}) {
+        return this.profiler.summary(options.reset ?? false);
+    }
+    /* ------------------------------------------------------------------------
+     * Cross-process lock
+     * ---------------------------------------------------------------------- */
     isStalelock() {
         const nodeFs = getFsSync();
         const STALE_LOCK_THRESHOLD_MS = 5000;
@@ -264,110 +843,32 @@ export class FileJsDb {
     }
     withMutationLock(operation) {
         this.acquireLockSync();
+        const started = this.profiler.now();
         try {
             this.refresh();
             return operation();
         }
         finally {
+            this.profiler.record('mutation', started);
             this.releaseLockAsync();
         }
     }
-    refresh() {
-        this.state = this.loadState();
-        this.peers.clear();
-        this.restorePeers();
-        this.applied.clear();
-        for (const transactionId of this.state.applied_transactions ?? [])
-            this.applied.add(transactionId);
-        if (this.queryEngine && this.indexedSequence !== this.state.sequence) {
-            this.queryEngine.rebuild();
-            this.indexedSequence = this.state.sequence;
-        }
-    }
-    saveState() {
-        const nodeFs = getFsSync();
-        const nodePathModule = getPathSync();
-        try {
-            // Update peers in state
-            this.state.peers = [...this.peers];
-            // Serialize complete snapshot (single logical state)
-            const snapshot = {
-                version: 1,
-                timestamp: Date.now(),
-                meta: {
-                    instance_id: this.origin,
-                    sequence: this.state.sequence,
-                    peers: this.state.peers,
-                    sequences: this.state.sequences
-                },
-                rows: this.state.rows,
-                events: this.state.events,
-                applied_transactions: [...this.applied]
-            };
-            const content = JSON.stringify(snapshot, null, 2);
-            // Write to temporary file
-            const tempPath = `${this.statePath}.${process.pid}.${Math.random().toString(36).slice(2)}.tmp`;
-            const handle = nodeFs.openSync(tempPath, 'w');
-            try {
-                nodeFs.writeFileSync(handle, content, 'utf-8');
-                // Force the contents to disk before the rename publishes them.
-                // A rename is a metadata operation: without this, a crash can leave
-                // the new name pointing at a file whose contents never landed.
-                nodeFs.fsyncSync(handle);
-            }
-            finally {
-                nodeFs.closeSync(handle);
-            }
-            // Atomic rename commits the new state
-            nodeFs.renameSync(tempPath, this.statePath);
-            // Persist the rename itself, so the commit survives power loss.
-            try {
-                const directory = nodeFs.openSync(nodePathModule.dirname(this.statePath), 'r');
-                try {
-                    nodeFs.fsyncSync(directory);
-                }
-                finally {
-                    nodeFs.closeSync(directory);
-                }
-            }
-            catch {
-                // Directory fsync is not available on every platform. The rename is
-                // still atomic; only the durability of the rename itself is weaker.
-            }
-        }
-        catch (error) {
-            throw new Error(`Failed to save FeltDB state: ${error}`);
-        }
-    }
-    event(collection, key, type, value) {
-        this.state.sequence += 1;
-        const timestamp = Date.now();
-        this.state.events.push({
-            sequence: this.state.sequence,
-            collection,
-            key,
-            type,
-            value,
-            timestamp,
-            origin: this.origin,
-            id: `${this.origin}:${timestamp}:${this.state.sequence}`
-        });
-        this.queryEngine?.apply(key, type === 'put' ? this.state.rows[key] : undefined);
-        this.indexedSequence = this.state.sequence;
-    }
+    /* ------------------------------------------------------------------------
+     * JsDb
+     * ---------------------------------------------------------------------- */
     /**
      * Commit several operations atomically and durably.
      *
-     * The snapshot is published by writing a temporary file, fsyncing it, and
-     * renaming it over the previous one. The rename is atomic, so a reader sees
-     * either the whole transaction or none of it, and a crash at any point
-     * leaves one of those two states on disk — never a mixture.
+     * The whole transaction is one journal record, and a record is either
+     * terminated — committed, every mutation in it — or discarded on recovery.
+     * A reader or a restart sees the whole transaction or none of it.
      *
-     * Order matters here. The next state is built and persisted *before* it is
-     * adopted in memory, so a failed write cannot leave memory ahead of disk.
+     * Order matters here. The commit is made durable *before* it is applied in
+     * memory, so a failed write cannot leave memory ahead of disk.
      */
     async commit_transaction(request) {
         await this.acquireLockAsync();
+        const started = this.profiler.now();
         try {
             this.refresh();
             const before = this.state.sequence;
@@ -380,53 +881,22 @@ export class FileJsDb {
                 return embeddedResult(request, true, before, this.state.sequence);
             if (plan.kind === 'reject')
                 throw new EmbeddedTransactionRejected(plan.reason, plan.failure);
-            // Build the post-transaction state without touching the live one.
-            // Critical: Deep-copy rows to prevent stale references after refresh
-            const rows = {};
-            for (const [key, value] of Object.entries(this.state.rows)) {
-                rows[key] = value;
-            }
-            const events = [...this.state.events];
-            let sequence = this.state.sequence;
+            const body = this.stage();
             const timestamp = Date.now();
             for (const mutation of plan.mutations) {
-                sequence += 1;
-                if (mutation.value === undefined) {
-                    delete rows[mutation.key];
-                }
-                else {
-                    rows[mutation.key] = mutation.value;
-                }
-                events.push({
-                    sequence,
-                    collection: mutation.collection,
-                    key: mutation.key,
-                    type: mutation.value === undefined ? 'delete' : 'put',
-                    value: mutation.value,
-                    timestamp,
-                    origin: this.origin,
-                    id: `${this.origin}:${timestamp}:${sequence}`,
-                });
-            }
-            const previous = this.state;
-            this.state = { ...this.state, rows, events, sequence };
-            // The transaction id has to be part of the snapshot being written, not
-            // added after it. Recording it afterwards would leave a committed
-            // transaction whose identity did not survive a restart, and redelivery
-            // would then apply it a second time.
-            this.applied.add(request.transactionId);
-            try {
-                this.saveState();
-            }
-            catch (error) {
-                // The commit did not become durable, so it must not be visible either.
-                this.state = previous;
-                this.applied.delete(request.transactionId);
-                throw error;
+                this.stageRow(body, mutation.key, mutation.value);
+                this.stageEvent(body, mutation.collection, mutation.key, mutation.value === undefined ? 'delete' : 'put', mutation.value, timestamp);
             }
-            return embeddedResult(request, false, before, sequence);
+            // The transaction id is part of the commit being written, not added
+            // after it. Recording it afterwards would leave a committed transaction
+            // whose identity did not survive a restart, and redelivery would then
+            // apply it a second time.
+            body.t = request.transactionId;
+            this.persist(body);
+            return embeddedResult(request, false, before, body.s);
         }
         finally {
+            this.profiler.record('mutation', started);
             this.releaseLockAsync();
         }
     }
@@ -434,10 +904,9 @@ export class FileJsDb {
         try {
             const parsed = JSON.parse(value);
             return this.withMutationLock(() => {
-                this.state.rows[key] = parsed;
-                const collection = key.split(':')[0];
-                this.event(collection, key, 'put', parsed);
-                this.saveState();
+                const body = this.stage();
+                this.stagePut(body, key, parsed);
+                this.persist(body);
                 return { success: true, data: key };
             });
         }
@@ -451,9 +920,9 @@ export class FileJsDb {
             return this.withMutationLock(() => {
                 if (!(key in this.state.rows))
                     return { success: false, error: `Record ${key} not found` };
-                this.state.rows[key] = parsed;
-                this.event(key.split(':')[0], key, 'put', parsed);
-                this.saveState();
+                const body = this.stage();
+                this.stagePut(body, key, parsed);
+                this.persist(body);
                 return { success: true, data: key };
             });
         }
@@ -473,9 +942,10 @@ export class FileJsDb {
             return this.withMutationLock(() => {
                 if (!(key in this.state.rows))
                     return { success: false, error: `Record ${key} not found` };
-                delete this.state.rows[key];
-                this.event(key.split(':')[0], key, 'delete');
-                this.saveState();
+                const body = this.stage();
+                this.stageRow(body, key, undefined);
+                this.stageEvent(body, key.split(':')[0], key, 'delete');
+                this.persist(body);
                 return { success: true, data: key };
             });
         }
@@ -531,10 +1001,20 @@ export class FileJsDb {
         return { success: true };
     }
     add_sync_peer(peerId) {
-        return this.withMutationLock(() => { this.peers.add(peerId); this.saveState(); return { success: true }; });
+        return this.withMutationLock(() => {
+            const body = this.stage();
+            body.p = [...new Set([...this.peers, peerId])];
+            this.persist(body);
+            return { success: true };
+        });
     }
     remove_sync_peer(peerId) {
-        return this.withMutationLock(() => { this.peers.delete(peerId); this.saveState(); return { success: true }; });
+        return this.withMutationLock(() => {
+            const body = this.stage();
+            body.p = [...this.peers].filter(peer => peer !== peerId);
+            this.persist(body);
+            return { success: true };
+        });
     }
     acknowledge_peer_operations() {
         return { success: true };
@@ -593,62 +1073,63 @@ export class FileJsDb {
         return this.withMutationLock(() => {
             let applied = 0;
             let ignored = 0;
+            const body = this.stage();
+            const staged = () => body.e ?? [];
             for (const operation of operations) {
                 // Skip if we've already applied this operation
-                if (this.state.events.some(event => event.id === operation.id)) {
+                if (staged().some(event => event.id === operation.id)
+                    || this.state.events.some(event => event.id === operation.id)) {
                     ignored++;
                     continue;
                 }
                 // Find the current state of this key
-                const current = [...this.state.events].reverse().find(event => event.key === operation.key);
+                const current = [...staged()].reverse().find(event => event.key === operation.key)
+                    ?? [...this.state.events].reverse().find(event => event.key === operation.key);
                 // Use lamport-clock-style ordering: timestamp, then origin, then id
                 const incomingOrder = `${String(operation.timestamp).padStart(16, '0')}:${operation.origin}:${operation.id}`;
                 const currentOrder = current ? `${String(current.timestamp).padStart(16, '0')}:${current.origin}:${current.id}` : '';
-                this.state.sequence += 1;
-                this.state.events.push({ ...operation, sequence: this.state.sequence });
+                body.s += 1;
+                (body.e ??= []).push({ ...operation, sequence: body.s });
                 // Apply if incoming is newer
                 if (incomingOrder >= currentOrder) {
-                    if (operation.type === 'delete') {
-                        delete this.state.rows[operation.key];
-                    }
-                    else {
-                        this.state.rows[operation.key] = operation.value;
-                    }
+                    this.stageRow(body, operation.key, operation.type === 'delete' ? undefined : operation.value);
                     applied++;
                 }
                 else {
                     ignored++;
                 }
             }
-            // Save state after applying remote operations
+            // Persist after applying remote operations
             if (applied > 0) {
-                this.saveState();
+                this.persist(body);
             }
             return { applied, ignored };
         });
     }
     async allocateSequence(params) {
         await this.acquireLockAsync();
+        const started = this.profiler.now();
         try {
             this.refresh();
             const key = `${params.scope}:${params.scopeId}`;
             const next = (this.state.sequences[key] ?? 0) + 1;
-            this.state.sequences[key] = next;
-            this.saveState();
+            const body = this.stage();
+            body.q = { [key]: next };
+            this.persist(body);
             return next;
         }
         finally {
+            this.profiler.record('mutation', started);
             this.releaseLockAsync();
         }
     }
     async putIfAbsent(key, value) {
-        const nodeFs = getFsSync();
         try {
             await this.acquireLockAsync();
             try {
                 const parsed = JSON.parse(value);
-                // Reload state from disk to ensure we have the latest
-                this.state = this.loadState();
+                // Bring state up to date with disk to ensure we have the latest
+                this.refresh();
                 // Check if key exists (atomic read within lock boundary)
                 if (key in this.state.rows) {
                     // Already exists, return the existing value
@@ -659,10 +1140,9 @@ export class FileJsDb {
                     };
                 }
                 // Key doesn't exist, insert it atomically within lock
-                this.state.rows[key] = parsed;
-                const collection = key.split(':')[0];
-                this.event(collection, key, 'put', parsed);
-                this.saveState();
+                const body = this.stage();
+                this.stagePut(body, key, parsed);
+                this.persist(body);
                 return {
                     inserted: true,
                     value: value
@@ -696,7 +1176,6 @@ export class FileJsDb {
             metadata: input.metadata,
             authorityScope: input.authorityScope,
         };
-        // Use putIfAbsent to atomically store the operation
         // Key format: "operation:{idempotencyKey}" for idempotency lookup
         // Key format: "op-id:{operationId}" for direct operation lookup
         const idempotencyKey = `operation:${input.idempotencyKey}`;
@@ -710,11 +1189,10 @@ export class FileJsDb {
                 ? { inserted: true, value: operationValue }
                 : { inserted: false, value: JSON.stringify(present) };
             if (result.inserted) {
-                this.state.rows[idempotencyKey] = durableOperation;
-                this.state.rows[operationIdKey] = durableOperation;
-                this.event('operation', idempotencyKey, 'put', durableOperation);
-                this.event('op-id', operationIdKey, 'put', durableOperation);
-                this.saveState();
+                const body = this.stage();
+                this.stagePut(body, idempotencyKey, durableOperation);
+                this.stagePut(body, operationIdKey, durableOperation);
+                this.persist(body);
             }
             // Parse the existing or newly inserted operation
             const existingOperation = JSON.parse(result.value);
@@ -749,8 +1227,8 @@ export class FileJsDb {
         try {
             await this.acquireLockAsync();
             try {
-                // Reload state from disk to ensure we have latest
-                this.state = this.loadState();
+                // Bring state up to date with disk to ensure we have latest
+                this.refresh();
                 // Look up operation by its operationId
                 const operationIdKey = `op-id:${input.operationId}`;
                 // Get current operation state
@@ -797,14 +1275,15 @@ export class FileJsDb {
                     error: input.error ?? currentOperation.error,
                 };
                 // Persist the transition atomically within lock (update both indices)
-                this.state.rows[operationIdKey] = transitionedOperation;
+                const body = this.stage();
+                this.stageRow(body, operationIdKey, transitionedOperation);
                 const idempotencyIndexKey = `operation:${currentOperation.idempotencyKey}`;
                 if (idempotencyIndexKey in this.state.rows) {
-                    this.state.rows[idempotencyIndexKey] = transitionedOperation;
+                    this.stageRow(body, idempotencyIndexKey, transitionedOperation);
                 }
                 const collection = operationIdKey.split(':')[0];
-                this.event(collection, operationIdKey, 'put', transitionedOperation);
-                this.saveState();
+                this.stageEvent(body, collection, operationIdKey, 'put', transitionedOperation);
+                this.persist(body);
                 return {
                     transitioned: true,
                     operation: transitionedOperation,
@@ -831,11 +1310,11 @@ export class FileJsDb {
             console.log(`[CAS-${casId}] lock acquired`);
         try {
             const parsed = JSON.parse(params.value);
-            // CRITICAL: Reload state from disk while holding lock
+            // CRITICAL: Bring state up to date with disk while holding lock
             // This ensures we see the authoritative version, not just our in-memory cache
             if (debugMode)
                 console.log(`[CAS-${casId}] reloading state from disk...`);
-            this.state = this.loadState();
+            this.refresh();
             if (debugMode)
                 console.log(`[CAS-${casId}] state reloaded`);
             // Check if key exists
@@ -883,13 +1362,12 @@ export class FileJsDb {
             if (debugMode)
                 console.log(`[CAS-${casId}] version match - updating from ${currentVersion} to ${params.expectedVersion + 1}`);
             const newValue = { ...parsed, __version: params.expectedVersion + 1 };
-            this.state.rows[params.key] = newValue;
-            const collection = params.key.split(':')[0];
-            this.event(collection, params.key, 'put', newValue);
-            // Persist atomically (temp file + rename)
+            const body = this.stage();
+            this.stagePut(body, params.key, newValue);
+            // Persist durably (journal commit)
             if (debugMode)
                 console.log(`[CAS-${casId}] saving state to disk...`);
-            this.saveState();
+            this.persist(body);
             if (debugMode)
                 console.log(`[CAS-${casId}] state saved, returning SUCCESS`);
             return {