@optimystic/db-p2p-storage-fs 1.14.0 → 1.16.0

package/dist/src/file-write-log.js

@@ -0,0 +1,848 @@
+import { promises as fs } from 'fs';
+import * as path from 'path';
+// NOTE: `zlib.crc32` exists from Node 22.2 (and 20.15); the repository declares no engine floor,
+// and this import is what sets it for this package. On an older Node it is `undefined` and the
+// first commit throws a TypeError. If an older Node must be supported, compute the CRC here.
+import { crc32 } from 'zlib';
+import { armUnrefTimer } from '@optimystic/db-core';
+import { StoreFailedError, SyncDeadlines } from '@optimystic/db-p2p';
+import { createLogger } from './logger.js';
+import { acquireStoreLock, StoreLockedError } from './store-lock.js';
+const log = createLogger('write-log');
+export const WRITE_LOG_FILENAME = 'write-ahead.log';
+/**
+ * The log a checkpoint has sealed and is syncing the tree under. Present on disk only between a
+ * checkpoint's seal and its truncate; an open replays it ahead of {@link WRITE_LOG_FILENAME}.
+ */
+export const SEALED_WRITE_LOG_FILENAME = 'write-ahead.log.sealed';
+/** {@link FileWriteLogOptions.checkpointBytes}'s default: 16 MiB. */
+export const DEFAULT_CHECKPOINT_BYTES = 16 * 1024 * 1024;
+/** {@link FileWriteLogOptions.checkpointIdleMs}'s default. */
+export const DEFAULT_CHECKPOINT_IDLE_MS = 5000;
+/** How many files, then directories, a checkpoint syncs at once. */
+const CHECKPOINT_SYNC_CONCURRENCY = 16;
+/**
+ * The log's fixed header: the magic bytes `OPTIMYSTIC-WAL`, then one format-version byte. A file
+ * that does not start with exactly this is not a log this build wrote, and replay applies nothing
+ * from it.
+ */
+export const WRITE_LOG_FORMAT_VERSION = 1;
+// `TextEncoder` is fine at module scope everywhere (see the Hermes note in eslint.config.js); a
+// `TextDecoder` is built on first use, like every other one in library source.
+const encoder = new TextEncoder();
+let decoderInstance;
+const decoder = () => decoderInstance ??= new TextDecoder();
+export const WRITE_LOG_HEADER = concat([encoder.encode('OPTIMYSTIC-WAL'), Uint8Array.of(WRITE_LOG_FORMAT_VERSION)]);
+const KIND_PUT = 1;
+const KIND_DELETE = 2;
+const RECORD_HEAD_BYTES = 8;
+const ENTRY_HEAD_BYTES = 1 + 2 + 4;
+const EMPTY = new Uint8Array(0);
+// --- record codec ---
+/**
+ * One record: `[u32 payload length][u32 CRC-32 of the payload][payload]`, little-endian, where the
+ * payload is the batch's entries in order, each `[u8 kind][u16 path length][path, UTF-8][u32 value
+ * length][value]`. A delete carries a zero-length value.
+ */
+export function encodeWriteLogRecord(entries) {
+    const payload = encodePayload(entries);
+    const record = new Uint8Array(RECORD_HEAD_BYTES + payload.length);
+    const view = new DataView(record.buffer);
+    view.setUint32(0, payload.length, true);
+    view.setUint32(4, crc32(payload), true);
+    record.set(payload, RECORD_HEAD_BYTES);
+    return record;
+}
+function encodePayload(entries) {
+    const encoded = entries.map(entry => {
+        const pathBytes = encoder.encode(entry.path);
+        if (pathBytes.length > 0xffff) {
+            throw new Error(`write-ahead log path is ${pathBytes.length} bytes, over the 65535 the record format holds: ${entry.path}`);
+        }
+        return { kind: entry.kind === 'put' ? KIND_PUT : KIND_DELETE, pathBytes, value: entry.kind === 'put' ? entry.value : EMPTY };
+    });
+    const payload = new Uint8Array(encoded.reduce((total, { pathBytes, value }) => total + ENTRY_HEAD_BYTES + pathBytes.length + value.length, 0));
+    const view = new DataView(payload.buffer);
+    let offset = 0;
+    for (const { kind, pathBytes, value } of encoded) {
+        payload[offset] = kind;
+        view.setUint16(offset + 1, pathBytes.length, true);
+        payload.set(pathBytes, offset + 3);
+        view.setUint32(offset + 3 + pathBytes.length, value.length, true);
+        payload.set(value, offset + ENTRY_HEAD_BYTES + pathBytes.length);
+        offset += ENTRY_HEAD_BYTES + pathBytes.length + value.length;
+    }
+    return payload;
+}
+/**
+ * Read a log strictly: a header that is not ours, a record whose length runs past the end of the
+ * file, a CRC that does not match, or an entry that does not parse stops the read before that
+ * record. What precedes it is returned in order.
+ */
+export function decodeWriteLog(bytes) {
+    if (!startsWith(bytes, WRITE_LOG_HEADER)) {
+        return { batches: [], stoppedAt: 0, stoppedBy: 'bad-header' };
+    }
+    const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
+    const batches = [];
+    let offset = WRITE_LOG_HEADER.length;
+    const stop = (stoppedBy) => ({ batches, stoppedAt: offset, stoppedBy });
+    while (offset < bytes.length) {
+        if (bytes.length - offset < RECORD_HEAD_BYTES)
+            return stop('torn-record');
+        const length = view.getUint32(offset, true);
+        const crc = view.getUint32(offset + 4, true);
+        if (bytes.length - offset - RECORD_HEAD_BYTES < length)
+            return stop('torn-record');
+        const payload = bytes.subarray(offset + RECORD_HEAD_BYTES, offset + RECORD_HEAD_BYTES + length);
+        if (crc32(payload) !== crc)
+            return stop('crc-mismatch');
+        const entries = decodePayload(payload);
+        if (entries === undefined)
+            return stop('bad-entry');
+        batches.push(entries);
+        offset += RECORD_HEAD_BYTES + length;
+    }
+    return { batches, stoppedAt: offset };
+}
+/** The payload's entries, or `undefined` when they do not exactly fill it. */
+function decodePayload(payload) {
+    const view = new DataView(payload.buffer, payload.byteOffset, payload.byteLength);
+    const entries = [];
+    let offset = 0;
+    while (offset < payload.length) {
+        if (payload.length - offset < ENTRY_HEAD_BYTES)
+            return undefined;
+        const kind = payload[offset];
+        const pathLength = view.getUint16(offset + 1, true);
+        if (payload.length - offset - 3 < pathLength + 4)
+            return undefined;
+        const entryPath = decoder().decode(payload.subarray(offset + 3, offset + 3 + pathLength));
+        const valueLength = view.getUint32(offset + 3 + pathLength, true);
+        const valueStart = offset + ENTRY_HEAD_BYTES + pathLength;
+        if (payload.length - valueStart < valueLength)
+            return undefined;
+        if (kind === KIND_PUT) {
+            entries.push({ kind: 'put', path: entryPath, value: payload.slice(valueStart, valueStart + valueLength) });
+        }
+        else if (kind === KIND_DELETE && valueLength === 0) {
+            entries.push({ kind: 'delete', path: entryPath });
+        }
+        else {
+            return undefined;
+        }
+        offset = valueStart + valueLength;
+    }
+    return entries;
+}
+function concat(parts) {
+    const out = new Uint8Array(parts.reduce((total, part) => total + part.length, 0));
+    let offset = 0;
+    for (const part of parts) {
+        out.set(part, offset);
+        offset += part.length;
+    }
+    return out;
+}
+function startsWith(bytes, prefix) {
+    if (bytes.length < prefix.length)
+        return false;
+    for (let i = 0; i < prefix.length; i++) {
+        if (bytes[i] !== prefix[i])
+            return false;
+    }
+    return true;
+}
+// --- filesystem helpers shared with the backend ---
+/** The file's bytes, or `undefined` when there is no such file. Any other error propagates. */
+export async function readFileIfExists(filePath) {
+    try {
+        return await fs.readFile(filePath);
+    }
+    catch (err) {
+        if (err.code === 'ENOENT')
+            return undefined;
+        throw err;
+    }
+}
+/**
+ * Best-effort fsync of a directory, so an entry added to, removed from or renamed within it
+ * survives power loss. POSIX needs this; platforms that reject opening a directory for fsync
+ * throw, and the throw is ignored rather than failing the write.
+ *
+ * Skipped outright on win32: there `fs.open(dir, 'r')` SUCCEEDS and only the following
+ * `handle.sync()` fails with EPERM, so the open/close pair was guaranteed wasted work. Durability
+ * is unchanged on win32 (the sync never succeeded there).
+ *
+ * NOTE: every error is swallowed, an I/O error included, and a checkpoint discards its sealed log
+ * on the strength of these syncs. A directory sync that fails with EIO therefore lets the log go
+ * while a directory entry it covered may not be durable. If that is ever seen, swallow only the
+ * codes a filesystem uses to refuse a directory sync (EINVAL, ENOTSUP, EISDIR, EPERM, EBADF) and
+ * ENOENT, and let the rest fail the checkpoint.
+ */
+export async function fsyncDir(dir) {
+    if (process.platform === 'win32')
+        return;
+    let handle;
+    try {
+        handle = await fs.open(dir, 'r');
+        await handle.sync();
+    }
+    catch {
+        // Directory fsync unsupported here — nothing to do.
+    }
+    finally {
+        if (handle)
+            await handle.close().catch(() => { });
+    }
+}
+/** fsync a file in place; a file that no longer exists is skipped. Opened `r+`: Windows refuses to flush a read-only handle. */
+export async function fsyncFile(filePath) {
+    let handle;
+    try {
+        handle = await fs.open(filePath, 'r+');
+    }
+    catch (err) {
+        if (err.code === 'ENOENT')
+            return;
+        throw err;
+    }
+    try {
+        await handle.sync();
+    }
+    finally {
+        await handle.close();
+    }
+}
+/**
+ * The identity every store over `basePath` reports (`RawStoreDriver.storeIdentity`): `file:` and
+ * the directory resolved once, here. `path.resolve` is cwd-dependent, and resolving at construction
+ * is the correct reading — two stores built from the same RELATIVE path under different cwds
+ * genuinely address different directories and must not compare equal. An empty basePath resolves
+ * to the cwd; nothing special-cases it. (A whitespace-only basePath is NOT the cwd — `path.resolve`
+ * is pure string work, so `' '` names a child directory called `' '`.)
+ *
+ * Lowercased on win32 because Windows paths are case-insensitive: `C:\Foo` and `C:\foo` are one
+ * directory and must produce one identity.
+ *
+ * NOTE: aliases of one directory that this does NOT collapse, and so read as two identities:
+ * symlinks, junctions, UNC-versus-mapped-drive spellings, and case-differing spellings on a
+ * case-INSENSITIVE non-Windows volume (the default on macOS) — platform is the only signal
+ * available synchronously, and case-folding every darwin path would be wrong on a case-sensitive
+ * volume. `fs.realpath` resolves all of these, but it is async (a store's constructor is not) and
+ * requires the directory to already exist. Revisit if an alias ever bites. All of these fail in
+ * the SAFE direction for the read cache: a missed alias reports two identities for one directory,
+ * so an identity-keyed consumer declines to merge and gets today's behavior. For the write-ahead
+ * log they fail less safely: two stores over two spellings of one directory open two logs, and
+ * the second is refused by the lock.
+ * NOTE: the win32 fold is the one branch that can err in the UNSAFE direction — Windows 10+
+ * supports per-directory case sensitivity (`fsutil file setCaseSensitiveInfo`), and on such a
+ * directory `X:\p\Foo` and `X:\p\foo` are two directories folded onto ONE identity. The merging
+ * consumers — `withReadCache` (one read cache per identity) and the log registry below (one log
+ * per identity) — would then read and write two directories through one cache and one log. Not
+ * addressable synchronously either (the flag is a per-directory filesystem query). Revisit if
+ * Optimystic ever targets a case-sensitive-enabled Windows directory; until then the fold is
+ * right for every ordinary Windows volume.
+ */
+export function fileStoreIdentity(basePath) {
+    const resolved = path.resolve(basePath);
+    return `file:${process.platform === 'win32' ? resolved.toLowerCase() : resolved}`;
+}
+/** Throws a `RangeError` for a checkpoint setting out of range; a store's constructor calls it, so a bad setting fails there rather than at the first write. */
+export function assertFileWriteLogOptions(options) {
+    checkpointSettings(options);
+}
+function checkpointSettings(options) {
+    const checkpointBytes = options.checkpointBytes ?? DEFAULT_CHECKPOINT_BYTES;
+    const checkpointIdleMs = options.checkpointIdleMs ?? DEFAULT_CHECKPOINT_IDLE_MS;
+    if (!(checkpointBytes > 0))
+        throw new RangeError(`checkpointBytes must be positive, got ${checkpointBytes}`);
+    if (!(checkpointIdleMs >= 0) || !Number.isFinite(checkpointIdleMs)) {
+        throw new RangeError(`checkpointIdleMs must be a finite number of milliseconds, got ${checkpointIdleMs}`);
+    }
+    return { checkpointBytes, checkpointIdleMs, setTimer: options.setTimer ?? armUnrefTimer };
+}
+const noop = () => { };
+const emptyDirty = () => ({ files: new Set(), directories: new Set() });
+/**
+ * The write-ahead log of one store directory: `<basePath>/write-ahead.log`, every batch of writes
+ * appended as one record, synced by the earliest deadline any batch was given, and applied to the
+ * readable file tree once it is durable.
+ *
+ * The write-ahead rule: a tree file only ever holds content whose log record is durable. So what
+ * survives a crash is the log's durable prefix plus whatever the tree already held, and {@link open}
+ * reconstructs it by replaying the log in order (puts and deletes are idempotent) before anything
+ * is served. A replay of a put over a newer tree file cannot happen: every tree write has a log
+ * record, and the log is replayed in order, so the last record for a path is the newest content
+ * the tree was ever given. A store written by a build before the log has no log file and opens
+ * with nothing to replay.
+ *
+ * A record reaches the operating system in its commit, so it survives a crash of this process from
+ * then on: the next open's replay applies it. What the sync adds is surviving a power cut, and a
+ * {@link commit} allowed to wait for that (`syncWithinMs` above 0) resolves without it, so that one
+ * sync, made by the earliest deadline, covers every record appended since the last. The rules are
+ * at {@link commit} and {@link syncUnsynced}.
+ *
+ * A {@link checkpoint} makes the tree as durable as the log and then discards the log, so the log
+ * never grows without bound: it runs when the log passes `checkpointBytes`, `checkpointIdleMs` after
+ * the last append, and on the last {@link release}. Its three steps are under {@link checkpointOnce}.
+ *
+ * One instance per directory per process, shared by reference count through {@link acquire}, so
+ * every store over one directory in one process — `FileRawStorage` and `FileKVStore` alike — shares
+ * one log, one lock and one checkpoint, rather than deadlocking on the lock or interleaving two
+ * writers into one file. Each keeps its own overlay above this, which is why a second store opened
+ * beside a first cannot see the first's unflushed writes.
+ */
+export class FileWriteLog {
+    basePath;
+    identity;
+    settings;
+    predecessorClosed;
+    static registry = new Map();
+    /** Logs whose last reference was released and whose handle and lock are still being let go of. */
+    static closing = new Map();
+    /**
+     * The shared log for the directory, counted once more. Synchronous, so two acquirers in one
+     * tick get one instance. Balance every call with one {@link release}.
+     *
+     * The checkpoint settings are the first acquirer's: a store opened over a directory whose log is
+     * already open shares that log's settings, and a setting it names that differs is logged and
+     * not applied.
+     */
+    static acquire(basePath, options = {}) {
+        const identity = fileStoreIdentity(basePath);
+        let shared = FileWriteLog.registry.get(identity);
+        if (shared) {
+            shared.noteDifferingSettings(options);
+        }
+        else {
+            shared = new FileWriteLog(path.resolve(basePath), identity, checkpointSettings(options), FileWriteLog.closing.get(identity));
+            FileWriteLog.registry.set(identity, shared);
+        }
+        shared.refs += 1;
+        return shared;
+    }
+    logPath;
+    sealedPath;
+    /** What the tree has been given since the log was last sealed; see {@link DirtyPaths}. Swapped for an empty set by each seal. */
+    dirty = emptyDirty();
+    /** Bytes of records in the current log, the size threshold's measure. Zero means there is nothing to checkpoint. */
+    appendedBytes = 0;
+    refs = 0;
+    opening;
+    handle;
+    lock;
+    /** Commits and seals run one at a time, in order, on this chain; it never rejects. */
+    queue = Promise.resolve();
+    fault;
+    closed = false;
+    /** The checkpoint running now, if any, which also runs every one asked for while it ran. */
+    checkpointRun;
+    /** A checkpoint was asked for while {@link checkpointRun} was running: run one more after it. */
+    checkpointAgain = false;
+    cancelIdleCheckpoint;
+    /**
+     * Records appended and not yet synced, in log order, each held with the entries the sync step
+     * writes to the tree and the `durable` it settles; with them the earliest deadline, the timer
+     * armed for it, and the `window-overrun` episode. Emptied by every sync step and by the latch.
+     * Bounded by the window: it holds what was committed since the last sync, and the earliest
+     * deadline among them forces one.
+     */
+    unsynced;
+    constructor(basePath, identity, settings, 
+    /** A previous log over this directory still releasing its handle and lock; opening waits for it. */
+    predecessorClosed) {
+        this.basePath = basePath;
+        this.identity = identity;
+        this.settings = settings;
+        this.predecessorClosed = predecessorClosed;
+        this.logPath = path.join(basePath, WRITE_LOG_FILENAME);
+        this.sealedPath = path.join(basePath, SEALED_WRITE_LOG_FILENAME);
+        this.unsynced = new SyncDeadlines({
+            onDue: firedAt => this.syncInBackground(firedAt),
+            setTimer: settings.setTimer,
+            log,
+            subject: basePath,
+        });
+    }
+    /** True once an open, an append, an apply or a checkpoint failed; every later commit then throws {@link StoreFailedError}. */
+    get failed() {
+        return this.fault !== undefined;
+    }
+    /**
+     * Take the lock, replay the log into the tree, truncate it and hold it open. Memoized: the first
+     * operation opens, every later one waits for that. A directory another live process holds throws
+     * {@link StoreLockedError} from every operation; an I/O failure latches the log failed and throws
+     * {@link StoreFailedError} from every operation. Either outcome is kept for the life of the
+     * instance, and the instance is shared by every store over the directory until each has released
+     * it: to retry after the holder has exited, close those stores and open a new one.
+     */
+    open() {
+        return this.opening ??= this.openOnce();
+    }
+    /**
+     * Append the batch as one record and resolve with its `durable`, which settles once the record is
+     * synced and the batch written to the tree. Batches commit one at a time in the order they were
+     * submitted, and their `durable` settle in that order. `WriteGroupSink.commit`'s contract:
+     *
+     * - `syncWithinMs` 0 (**strict**): the sync step runs before this resolves, so the returned
+     *   `durable` is already settled and every earlier batch is durable too.
+     * - Otherwise (**relaxed**): the record is in the operating system and the batch waits in
+     *   {@link unsynced} with the deadline `syncWithinMs` after this was CALLED, not after the queue got
+     *   to it, so a commit that waited in line past its own deadline is treated as overdue. A timer
+     *   armed for the earliest deadline runs the sync step; a strict commit or a checkpoint meanwhile
+     *   runs it sooner, for every batch waiting.
+     * - **Backpressure:** a commit that finds the earliest unsynced deadline already passed runs the
+     *   sync step before it resolves, whatever its own window. When the disk falls behind, writers
+     *   wait for it instead of the window quietly growing.
+     *
+     * The deadlines, the timer and the backpressure test are the rules of `SyncDeadlines`, from
+     * `@optimystic/db-p2p`, which every store that defers its sync shares.
+     *
+     * The tree is written at the sync, not here, so a reader of the tree never sees a batch whose
+     * record a power cut could still take; the overlay above each store serves the batch until its
+     * `durable` resolves.
+     */
+    commit(entries, { syncWithinMs }) {
+        if (!(syncWithinMs >= 0) || !Number.isFinite(syncWithinMs)) {
+            return Promise.reject(new RangeError(`syncWithinMs must be a finite number of milliseconds at or above 0, got ${syncWithinMs}`));
+        }
+        const deadline = this.unsynced.deadlineFor(syncWithinMs);
+        return this.enqueue(async () => {
+            await this.open();
+            if (this.closed)
+                throw new Error(`write-ahead log at ${this.basePath} is closed`);
+            this.throwIfFailed();
+            await this.append(entries);
+            // A checkpoint syncs the tree outside the queue, so its failure can latch the log while the
+            // append is in flight; the commit is then refused like every later one, its record left to the
+            // next open's replay. Checked with no await before `defer`, which throws once latched.
+            this.throwIfFailed();
+            const durable = this.unsynced.defer(entries, deadline);
+            if (this.unsynced.mustSyncNow(deadline))
+                await this.syncUnsynced();
+            this.scheduleCheckpoint();
+            return { durable };
+        });
+    }
+    /**
+     * Hasten: make every record appended before this call durable, and write its batch to the tree.
+     * Queued, so it covers exactly those records; with nothing unsynced it touches no disk.
+     * `WriteGroupSink.sync`'s contract.
+     */
+    sync() {
+        return this.enqueue(async () => {
+            this.throwIfFailed();
+            await this.syncUnsynced();
+        });
+    }
+    /**
+     * Make the tree as durable as the log, then discard the log. One at a time: asked for while one
+     * runs, it runs once more after that one, and resolves when that has. Skipped when the log has
+     * failed, was never opened, or holds no record. A failure latches the log failed, like a rejected
+     * commit: a half-synced tree under a discarded log cannot be recovered.
+     */
+    checkpoint() {
+        if (this.checkpointRun) {
+            this.checkpointAgain = true;
+            return this.checkpointRun;
+        }
+        return this.checkpointRun = this.runCheckpoints();
+    }
+    /**
+     * Give back one reference. The last release waits for the commit in flight and for a checkpoint
+     * in flight, runs a final checkpoint, closes the log handle and removes the lock; a log acquired
+     * for the directory meanwhile opens after that. Rejects only when that final checkpoint fails,
+     * and lets go of the handle and the lock either way.
+     */
+    async release() {
+        if (this.refs === 0)
+            throw new Error(`write-ahead log at ${this.basePath} was released more often than acquired`);
+        this.refs -= 1;
+        if (this.refs > 0)
+            return;
+        FileWriteLog.registry.delete(this.identity);
+        const done = this.shutDown();
+        const tracked = done.catch(noop).finally(() => {
+            if (FileWriteLog.closing.get(this.identity) === tracked)
+                FileWriteLog.closing.delete(this.identity);
+        });
+        FileWriteLog.closing.set(this.identity, tracked);
+        await done;
+    }
+    throwIfFailed() {
+        if (this.fault)
+            throw new StoreFailedError(this.fault.cause);
+    }
+    enqueue(task) {
+        const run = this.queue.then(task);
+        this.queue = run.catch(noop);
+        return run;
+    }
+    noteDifferingSettings(options) {
+        const differing = ['checkpointBytes', 'checkpointIdleMs']
+            .filter(name => options[name] !== undefined && options[name] !== this.settings[name]);
+        if (differing.length > 0) {
+            log('a store over %s asked for %o; the log already open there keeps %o', this.basePath, Object.fromEntries(differing.map(name => [name, options[name]])), Object.fromEntries(differing.map(name => [name, this.settings[name]])));
+        }
+    }
+    // --- open and replay ---
+    async openOnce() {
+        try {
+            await this.predecessorClosed;
+            await fs.mkdir(this.basePath, { recursive: true });
+            this.lock = await acquireStoreLock(this.basePath);
+            await this.replay();
+            this.handle = await this.createEmptyLog();
+        }
+        catch (err) {
+            await this.releaseResources().catch(closeErr => log('releasing after a failed open: %o', closeErr));
+            if (err instanceof StoreLockedError)
+                throw err;
+            throw this.latch(err);
+        }
+    }
+    /**
+     * Apply the sealed segment's records, then the current log's, in order; make what they touched
+     * durable; then remove the sealed segment. The two files are one log: a sealed segment that
+     * stops short ends the replay there, and the current log is discarded with the rest of it.
+     *
+     * Re-applying a sealed segment whose checkpoint already synced the tree is harmless, because the
+     * current log holds every later write of the same paths, in order, and is applied after it. What
+     * must never happen is the sealed segment outliving the current log's truncation, since replaying
+     * it alone would put older content over newer: so the sealed segment is removed, and the
+     * directory synced, before {@link openOnce} truncates the current log. (On win32, where
+     * {@link fsyncDir} is skipped, NTFS commits the unlink and the later truncation through its
+     * metadata journal in order.)
+     */
+    async replay() {
+        const touched = emptyDirty();
+        const sealed = await this.replaySegment(this.sealedPath, touched);
+        if (sealed === 'stopped-short') {
+            // NOTE: the current log is discarded only by the truncation after the sealed segment is
+            // removed, so a crash between the two replays the current log alone on the next open. A
+            // sealed segment holds only synced records, so it stops short only when its bytes were
+            // damaged on disk; if that is ever seen, truncate the current log before removing it.
+            log('replay of %s stopped short, so %s was not applied', this.sealedPath, this.logPath);
+        }
+        else {
+            await this.replaySegment(this.logPath, touched);
+        }
+        await syncDirtyPaths(touched);
+        if (sealed !== 'absent') {
+            await fs.unlink(this.sealedPath);
+            await fsyncDir(this.basePath);
+        }
+    }
+    /** Apply one log file's intact records, in order, into `touched`. */
+    async replaySegment(filePath, touched) {
+        // NOTE: a log file is read into memory for one decode. A checkpoint bounds the current log to
+        // about `checkpointBytes` plus what is appended while a checkpoint runs, and a sealed segment
+        // to one such log; if a log is ever found much larger than that, decode it record by record
+        // from a stream instead.
+        const bytes = await readFileIfExists(filePath);
+        if (bytes === undefined)
+            return 'absent';
+        const decoded = decodeWriteLog(bytes);
+        for (const batch of decoded.batches) {
+            await this.applyBatch(batch, touched);
+        }
+        if (!decoded.stoppedBy)
+            return 'complete';
+        log('replay of %s stopped at byte %d of %d (%s): %d batch(es) before it applied, the rest discarded', filePath, decoded.stoppedAt, bytes.length, decoded.stoppedBy, decoded.batches.length);
+        return 'stopped-short';
+    }
+    /** Create (or truncate) the current log to its header, and make it and its directory entry durable before any record goes in. */
+    async createEmptyLog() {
+        const handle = await fs.open(this.logPath, 'w');
+        try {
+            await writeAll(handle, WRITE_LOG_HEADER);
+            await handle.sync();
+            await fsyncDir(this.basePath);
+        }
+        catch (err) {
+            await handle.close().catch(noop);
+            throw err;
+        }
+        return handle;
+    }
+    // --- append and sync ---
+    /**
+     * Append the batch as one record. The record has reached the operating system when this
+     * resolves, so it survives a crash of this process: the next open's replay applies it. A failed
+     * append latches the log, and rejects every batch still waiting for its sync, since no sync will
+     * ever be made for them.
+     */
+    async append(entries) {
+        try {
+            const record = encodeWriteLogRecord(entries);
+            await writeAll(this.handle, record);
+            this.appendedBytes += record.length;
+        }
+        catch (err) {
+            throw this.latch(err);
+        }
+    }
+    /**
+     * The sync step: one sync of the log makes every record appended so far durable; then each waiting
+     * batch is written to the tree, in order, and its `durable` resolved, in order. Called only from a
+     * queued task — a commit, {@link sync}, the timer's task or a checkpoint's seal — so it covers
+     * exactly the records appended before that task began, and a sealed segment only ever holds
+     * synced records whose tree writes are done. Nothing waiting: nothing is touched.
+     *
+     * A failure anywhere latches the log and rejects every covered `durable`, the ones whose tree
+     * write succeeded included: the records are durable and the next open's replay rebuilds the tree
+     * from them, but a `durable` is a promise that this process finished the work, and it did not.
+     *
+     * `owedSince` is the moment the timer fired, when the timer's task runs this: `SyncDeadlines.take`
+     * counts the sync's lateness from it rather than from a deadline the timer's own slop missed, and
+     * the cover says `window-overrun` once per episode of syncs that started late.
+     */
+    async syncUnsynced(owedSince) {
+        const cover = this.unsynced.take(owedSince);
+        if (cover.items.length === 0)
+            return;
+        try {
+            await this.handle.sync();
+            for (const entries of cover.items)
+                await this.applyBatch(entries, this.dirty);
+        }
+        catch (err) {
+            const failure = this.latch(err);
+            cover.fail(err);
+            throw failure;
+        }
+        cover.settle();
+    }
+    /** The timer fired: run the sync step in the queue. Its failure has latched the log, so it is only logged here, like a background checkpoint's. */
+    syncInBackground(firedAt) {
+        this.enqueue(() => this.syncUnsynced(firedAt)).catch((err) => {
+            log('deferred sync of %s failed; the store refuses every later write: %o', this.basePath, err);
+        });
+    }
+    /**
+     * Latch the log failed on its first fault and reject every batch waiting for a sync that will now
+     * never be made, so no caller waits on a `durable` for ever. Returns the error to throw.
+     */
+    latch(err) {
+        this.fault ??= { cause: err };
+        this.unsynced.failAll(err);
+        return new StoreFailedError(err);
+    }
+    // --- apply ---
+    /**
+     * Write each entry to its canonical path, creating missing directories; record what the tree was given.
+     *
+     * NOTE: the write goes straight to the canonical path, so a reader of that path can see it
+     * half-written. The committing store's own readers are served from its overlay until the batch's
+     * `durable` resolves, and the two store kinds over one directory never share a path, so only a
+     * second store of the same kind over the same directory can read a torn file — a shape Invariant
+     * 5 already excludes. If that shape is ever supported, write through a temp file and rename.
+     */
+    async applyBatch(entries, touched) {
+        for (const entry of entries) {
+            const target = path.join(this.basePath, entry.path);
+            const dir = path.dirname(target);
+            if (entry.kind === 'put') {
+                const created = await fs.mkdir(dir, { recursive: true });
+                if (created !== undefined)
+                    this.noteCreated(dir, touched);
+                await fs.writeFile(target, entry.value);
+                touched.files.add(target);
+            }
+            else {
+                await unlinkIfExists(target);
+                touched.files.delete(target);
+            }
+            touched.directories.add(dir);
+        }
+    }
+    /**
+     * `mkdir` created at least `dir`'s topmost missing ancestor, whose parent's entry is new too, so
+     * every directory from `dir` up to the store's root is marked — a superset of what was created,
+     * at most the layout's depth of extra directory syncs per checkpoint. (The created path `mkdir`
+     * returns is not compared against: on win32 it comes back in `\\?\` long-path form.)
+     */
+    noteCreated(dir, touched) {
+        for (let current = dir;; current = path.dirname(current)) {
+            touched.directories.add(current);
+            if (current === this.basePath || path.dirname(current) === current)
+                return;
+        }
+    }
+    // --- checkpoint ---
+    /**
+     * After an append: re-arm the idle checkpoint, and start one now when the log has passed its size
+     * threshold. Nothing once the last release has begun, since it runs the final checkpoint itself.
+     */
+    scheduleCheckpoint() {
+        if (this.closed)
+            return;
+        this.disarmIdleCheckpoint();
+        this.cancelIdleCheckpoint = this.settings.setTimer(() => {
+            this.cancelIdleCheckpoint = undefined;
+            this.checkpointInBackground('idle');
+        }, this.settings.checkpointIdleMs);
+        if (this.appendedBytes >= this.settings.checkpointBytes)
+            this.checkpointInBackground('size');
+    }
+    /** Never awaited by a commit: a commit holds the queue the checkpoint's seal waits in. Its failure has latched the log. */
+    checkpointInBackground(trigger) {
+        this.checkpoint().catch((err) => {
+            log('checkpoint (%s) of %s failed; the store refuses every later write: %o', trigger, this.basePath, err);
+        });
+    }
+    disarmIdleCheckpoint() {
+        this.cancelIdleCheckpoint?.();
+        this.cancelIdleCheckpoint = undefined;
+    }
+    async runCheckpoints() {
+        try {
+            do {
+                this.checkpointAgain = false;
+                await this.checkpointOnce();
+            } while (this.checkpointAgain);
+        }
+        finally {
+            // Cleared before this promise settles, so a checkpoint asked for from here on starts afresh.
+            this.checkpointRun = undefined;
+        }
+    }
+    /**
+     * 1. **Seal**, in the commit queue so no apply is in flight: run the sync step for whatever is
+     *    still waiting, rename the log to {@link SEALED_WRITE_LOG_FILENAME} and start an empty one, and
+     *    take the dirty set, leaving an empty one. Commits resume on the new log at once; only this
+     *    step waits in the queue.
+     * 2. **Sync**, off the commit path: every file in the taken set, then every directory. A file
+     *    deleted since is skipped; one rewritten since is synced with its newer content, which the
+     *    new log covers too.
+     * 3. **Truncate**: unlink the sealed segment and sync the directory.
+     *
+     * A crash anywhere in this leaves the sealed segment, the current log or both, and the next open
+     * replays them in that order (see {@link replay}).
+     */
+    async checkpointOnce() {
+        if (!this.opening || this.fault)
+            return;
+        await this.opening.catch(noop);
+        const sealed = await this.seal();
+        if (!sealed)
+            return;
+        try {
+            await syncDirtyPaths(sealed);
+            await fs.unlink(this.sealedPath);
+            await fsyncDir(this.basePath);
+        }
+        catch (err) {
+            throw this.latch(err);
+        }
+    }
+    /**
+     * Step 1 of {@link checkpointOnce}: the dirty set the sealed segment covers, or `undefined` when
+     * there is nothing to seal. The sync step runs first, so the sealed segment holds only synced
+     * records whose tree writes are in the set the checkpoint goes on to sync. An unsynced record
+     * always has bytes, so `appendedBytes` is never 0 while one waits: the early return cannot skip
+     * one, and the final checkpoint on the last release syncs whatever is still waiting.
+     */
+    seal() {
+        return this.enqueue(async () => {
+            if (this.fault || this.appendedBytes === 0)
+                return undefined;
+            await this.syncUnsynced();
+            try {
+                const handle = this.handle;
+                this.handle = undefined;
+                await handle.close();
+                await fs.rename(this.logPath, this.sealedPath);
+                this.handle = await this.createEmptyLog();
+            }
+            catch (err) {
+                throw this.latch(err);
+            }
+            const sealed = this.dirty;
+            this.dirty = emptyDirty();
+            this.appendedBytes = 0;
+            return sealed;
+        });
+    }
+    // --- close ---
+    async shutDown() {
+        this.closed = true;
+        this.disarmIdleCheckpoint();
+        // The final checkpoint's seal syncs whatever is still waiting (see `seal`), which disarms the
+        // sync timer; one that fires before then enqueues a sync step that covers it sooner.
+        await this.queue;
+        try {
+            // A failure of the checkpoint in flight has been logged by whatever started it, and has
+            // latched the log, which skips the final one.
+            await this.checkpointRun?.catch(noop);
+            await this.checkpoint();
+        }
+        finally {
+            await this.releaseResources();
+        }
+    }
+    async releaseResources() {
+        const handle = this.handle;
+        this.handle = undefined;
+        if (handle)
+            await handle.close();
+        const lock = this.lock;
+        this.lock = undefined;
+        if (lock)
+            await lock.release();
+    }
+}
+/**
+ * Sync every file, then every directory, {@link CHECKPOINT_SYNC_CONCURRENCY} at a time.
+ *
+ * NOTE: a checkpoint's file sync holds a handle open while later commits may delete and re-create
+ * the same path. libuv opens with delete sharing and, on Windows 10 1809 and later, unlinks with
+ * POSIX semantics, so the name goes at once; on an older Windows the re-create would fail with
+ * EPERM while the deleted name lingers, latching the store. If that is ever seen, retry the apply's
+ * write on EPERM for the length of one sync.
+ */
+async function syncDirtyPaths(dirty) {
+    await forEachConcurrently(dirty.files, CHECKPOINT_SYNC_CONCURRENCY, fsyncFile);
+    await forEachConcurrently(dirty.directories, CHECKPOINT_SYNC_CONCURRENCY, fsyncDir);
+}
+/** Run `fn` over `items` with at most `limit` in flight; after the first failure no more start, and it is thrown once all in flight have settled. */
+async function forEachConcurrently(items, limit, fn) {
+    const iterator = items[Symbol.iterator]();
+    let failure;
+    const worker = async () => {
+        while (!failure) {
+            const next = iterator.next();
+            if (next.done)
+                return;
+            try {
+                await fn(next.value);
+            }
+            catch (err) {
+                failure ??= { cause: err };
+            }
+        }
+    };
+    await Promise.all(Array.from({ length: limit }, worker));
+    if (failure)
+        throw failure.cause;
+}
+async function writeAll(handle, bytes) {
+    let written = 0;
+    while (written < bytes.length) {
+        const { bytesWritten } = await handle.write(bytes, written, bytes.length - written);
+        if (bytesWritten <= 0)
+            throw new Error('write-ahead log append wrote nothing');
+        written += bytesWritten;
+    }
+}
+async function unlinkIfExists(filePath) {
+    try {
+        await fs.unlink(filePath);
+    }
+    catch (err) {
+        if (err.code !== 'ENOENT')
+            throw err;
+    }
+}
+//# sourceMappingURL=file-write-log.js.map