@backloghq/opslog 0.1.4 → 0.2.0

package/README.md

@@ -131,14 +131,42 @@ await store.open(dir, {
   checkpointOnClose: true,        // Checkpoint when close() is called (default: true)
   version: 1,                     // Schema version
   migrate: (record, fromVersion) => record, // Migration function
+  readOnly: false,                // Open in read-only mode (default: false)
 });
 ```
 
+## Read-Only Mode
+
+Open a store for reading without acquiring the write lock. Useful for dashboards, backup processes, or multiple readers alongside a single writer.
+
+```typescript
+const reader = new Store();
+await reader.open("./data", { readOnly: true });
+
+// All reads work
+const tasks = reader.all();
+const active = reader.filter((t) => t.status === "active");
+
+// All mutations throw
+await reader.set("x", value); // Error: Store is read-only
+```
+
+Read-only stores load the latest snapshot and replay ops on open. They do not checkpoint on close. Multiple read-only stores can open the same directory concurrently alongside one writer.
+
+## Concurrency
+
+All state-mutating operations (`set`, `delete`, `batch`, `undo`, `compact`, `archive`) are serialized through an internal async mutex. This prevents interleaving of concurrent mutations — e.g., `compact()` swapping the ops file while `set()` is appending, or `undo()` truncating while `set()` is writing.
+
+Read operations (`get`, `all`, `filter`, `count`, `has`, `entries`) are synchronous and lock-free.
+
+An advisory directory write lock (`.lock` file with PID) prevents two processes from opening the same store. Stale locks from crashed processes are automatically recovered.
+
 ## Crash Safety
 
 - **Ops file**: append-only writes. A crash mid-append loses at most the last operation. Malformed lines are skipped on recovery.
 - **Snapshots**: immutable. Written to a temp file, then atomically renamed.
 - **Manifest**: atomically replaced via temp-file-rename. Always points to a valid snapshot.
+- **Undo**: uses `ftruncate()` — a single atomic POSIX syscall. O(1) regardless of file size.
 
 No data corruption on crash. At most one in-flight operation is lost.
 

package/dist/index.d.ts

@@ -1,2 +1,3 @@
 export { Store } from "./store.js";
+export { acquireLock, releaseLock } from "./lock.js";
 export type { Operation, Snapshot, Manifest, ManifestStats, ArchiveSegment, StoreOptions, StoreStats, } from "./types.js";

package/dist/index.js

@@ -1 +1,2 @@
 export { Store } from "./store.js";
+export { acquireLock, releaseLock } from "./lock.js";

package/dist/lock.d.ts

@@ -0,0 +1,12 @@
+import type { FileHandle } from "node:fs/promises";
+/**
+ * Acquire an advisory write lock on a directory.
+ * Returns a FileHandle that must be passed to releaseLock() on close.
+ * Throws if another live process holds the lock.
+ * Automatically recovers stale locks from crashed processes.
+ */
+export declare function acquireLock(dir: string): Promise<FileHandle>;
+/**
+ * Release the advisory write lock.
+ */
+export declare function releaseLock(dir: string, fh: FileHandle): Promise<void>;

package/dist/lock.js

@@ -0,0 +1,64 @@
+import { open, readFile, unlink } from "node:fs/promises";
+import { join } from "node:path";
+const LOCK_FILE = ".lock";
+function isProcessAlive(pid) {
+    try {
+        process.kill(pid, 0);
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+/**
+ * Acquire an advisory write lock on a directory.
+ * Returns a FileHandle that must be passed to releaseLock() on close.
+ * Throws if another live process holds the lock.
+ * Automatically recovers stale locks from crashed processes.
+ */
+export async function acquireLock(dir) {
+    const lockPath = join(dir, LOCK_FILE);
+    // Try exclusive create
+    try {
+        const fh = await open(lockPath, "wx");
+        await fh.writeFile(String(process.pid), "utf-8");
+        return fh;
+    }
+    catch (err) {
+        if (err.code !== "EEXIST")
+            throw err;
+    }
+    // Lock file exists — check if the holder is still alive
+    let content;
+    try {
+        content = await readFile(lockPath, "utf-8");
+    }
+    catch {
+        // File disappeared between our open and read — retry
+        return acquireLock(dir);
+    }
+    const pid = parseInt(content, 10);
+    if (!isNaN(pid) && isProcessAlive(pid)) {
+        throw new Error(`Store is locked by process ${pid}. If this is stale, delete ${lockPath}`);
+    }
+    // Stale lock — remove and retry
+    try {
+        await unlink(lockPath);
+    }
+    catch {
+        // Another process may have already cleaned it up
+    }
+    return acquireLock(dir);
+}
+/**
+ * Release the advisory write lock.
+ */
+export async function releaseLock(dir, fh) {
+    await fh.close();
+    try {
+        await unlink(join(dir, LOCK_FILE));
+    }
+    catch {
+        // Already cleaned up
+    }
+}

package/dist/store.d.ts

@@ -12,6 +12,15 @@ export declare class Store<T = Record<string, unknown>> {
     private archivedRecordCount;
     private batching;
     private batchOps;
+    private _lock;
+    private lockFh;
+    /**
+     * Serialize all state-mutating operations through a promise chain.
+     * This prevents interleaving of async mutations (e.g. compact + set,
+     * undo + set) which could corrupt the WAL or in-memory state.
+     * Read operations remain synchronous and lock-free.
+     */
+    private serialize;
     open(dir: string, options?: StoreOptions): Promise<void>;
     close(): Promise<void>;
     get(id: string): T | undefined;
@@ -28,10 +37,19 @@ export declare class Store<T = Record<string, unknown>> {
     getOps(since?: string): Operation<T>[];
     compact(): Promise<void>;
     archive(predicate: (value: T, id: string) => boolean, segment?: string): Promise<number>;
-    loadArchive(segment: string): Promise<Map<string, T>>;
     listArchiveSegments(): string[];
+    loadArchive(segment: string): Promise<Map<string, T>>;
     stats(): StoreStats;
+    private _set;
+    private _setSync;
+    private _delete;
+    private _deleteSync;
+    private _batch;
+    private _undo;
+    private _compact;
+    private _archive;
     private ensureOpen;
+    private ensureWritable;
     private applyOp;
     private reverseOp;
     private persistOp;

package/dist/store.js

@@ -4,6 +4,7 @@ import { appendOp, appendOps, readOps, truncateLastOp } from "./wal.js";
 import { loadSnapshot, writeSnapshot } from "./snapshot.js";
 import { createDefaultManifest, readManifest, writeManifest, } from "./manifest.js";
 import { loadArchiveSegment, writeArchiveSegment, } from "./archive.js";
+import { acquireLock, releaseLock } from "./lock.js";
 export class Store {
     dir = "";
     records = new Map();
@@ -18,20 +19,43 @@ export class Store {
         checkpointOnClose: true,
         version: 1,
         migrate: (r) => r,
+        readOnly: false,
     };
     archivedRecordCount = 0;
     batching = false;
     batchOps = [];
+    _lock = Promise.resolve();
+    lockFh = null;
+    /**
+     * Serialize all state-mutating operations through a promise chain.
+     * This prevents interleaving of async mutations (e.g. compact + set,
+     * undo + set) which could corrupt the WAL or in-memory state.
+     * Read operations remain synchronous and lock-free.
+     */
+    serialize(fn) {
+        const prev = this._lock;
+        let resolve;
+        this._lock = new Promise((r) => {
+            resolve = r;
+        });
+        return prev.then(fn).finally(() => resolve());
+    }
     async open(dir, options) {
         this.dir = dir;
         if (options) {
             this.options = { ...this.options, ...options };
         }
-        await mkdir(join(dir, "snapshots"), { recursive: true });
-        await mkdir(join(dir, "ops"), { recursive: true });
-        await mkdir(join(dir, "archive"), { recursive: true });
+        if (!this.options.readOnly) {
+            await mkdir(join(dir, "snapshots"), { recursive: true });
+            await mkdir(join(dir, "ops"), { recursive: true });
+            await mkdir(join(dir, "archive"), { recursive: true });
+            this.lockFh = await acquireLock(dir);
+        }
         const manifest = await readManifest(dir);
         if (!manifest) {
+            if (this.options.readOnly) {
+                throw new Error("Cannot open in readOnly mode: no existing store found");
+            }
             // Fresh store — create empty snapshot and manifest
             const snapshotPath = await writeSnapshot(dir, new Map(), this.options.version);
             const opsFilename = `ops-${Date.now()}.jsonl`;
@@ -82,8 +106,12 @@ export class Store {
     }
     async close() {
         this.ensureOpen();
-        if (this.options.checkpointOnClose && this.ops.length > 0) {
-            await this.compact();
+        if (!this.options.readOnly && this.options.checkpointOnClose && this.ops.length > 0) {
+            await this.serialize(() => this._compact());
+        }
+        if (this.lockFh) {
+            await releaseLock(this.dir, this.lockFh);
+            this.lockFh = null;
         }
         this.opened = false;
     }
@@ -93,39 +121,21 @@ export class Store {
     }
     set(id, value) {
         this.ensureOpen();
-        const prev = this.records.get(id) ?? null;
-        const op = {
-            ts: new Date().toISOString(),
-            op: "set",
-            id,
-            data: value,
-            prev,
-        };
-        this.records.set(id, value);
+        this.ensureWritable();
         if (this.batching) {
-            this.batchOps.push(op);
+            this._setSync(id, value);
             return;
         }
-        return this.persistOp(op);
+        return this.serialize(() => this._set(id, value));
     }
     delete(id) {
         this.ensureOpen();
-        const prev = this.records.get(id);
-        if (prev === undefined) {
-            throw new Error(`Record '${id}' not found`);
-        }
-        const op = {
-            ts: new Date().toISOString(),
-            op: "delete",
-            id,
-            prev,
-        };
-        this.records.delete(id);
+        this.ensureWritable();
         if (this.batching) {
-            this.batchOps.push(op);
+            this._deleteSync(id);
             return;
         }
-        return this.persistOp(op);
+        return this.serialize(() => this._delete(id));
     }
     has(id) {
         this.ensureOpen();
@@ -161,6 +171,107 @@ export class Store {
     }
     async batch(fn) {
         this.ensureOpen();
+        this.ensureWritable();
+        return this.serialize(() => this._batch(fn));
+    }
+    async undo() {
+        this.ensureOpen();
+        this.ensureWritable();
+        return this.serialize(() => this._undo());
+    }
+    getHistory(id) {
+        this.ensureOpen();
+        return this.ops.filter((op) => op.id === id);
+    }
+    getOps(since) {
+        this.ensureOpen();
+        if (!since)
+            return [...this.ops];
+        return this.ops.filter((op) => op.ts > since);
+    }
+    async compact() {
+        this.ensureOpen();
+        this.ensureWritable();
+        return this.serialize(() => this._compact());
+    }
+    async archive(predicate, segment) {
+        this.ensureOpen();
+        this.ensureWritable();
+        return this.serialize(() => this._archive(predicate, segment));
+    }
+    listArchiveSegments() {
+        this.ensureOpen();
+        return [...this.archiveSegments];
+    }
+    async loadArchive(segment) {
+        this.ensureOpen();
+        const segmentPath = this.archiveSegments.find((s) => s === `archive/archive-${segment}.json`) || this.archiveSegments.find((s) => s.includes(segment));
+        if (!segmentPath)
+            throw new Error(`Archive segment '${segment}' not found`);
+        return loadArchiveSegment(this.dir, segmentPath);
+    }
+    stats() {
+        this.ensureOpen();
+        return {
+            activeRecords: this.records.size,
+            opsCount: this.ops.length,
+            archiveSegments: this.archiveSegments.length,
+        };
+    }
+    // --- Private mutation implementations ---
+    async _set(id, value) {
+        const prev = this.records.get(id) ?? null;
+        const op = {
+            ts: new Date().toISOString(),
+            op: "set",
+            id,
+            data: value,
+            prev,
+        };
+        this.records.set(id, value);
+        await this.persistOp(op);
+    }
+    _setSync(id, value) {
+        const prev = this.records.get(id) ?? null;
+        const op = {
+            ts: new Date().toISOString(),
+            op: "set",
+            id,
+            data: value,
+            prev,
+        };
+        this.records.set(id, value);
+        this.batchOps.push(op);
+    }
+    async _delete(id) {
+        const prev = this.records.get(id);
+        if (prev === undefined) {
+            throw new Error(`Record '${id}' not found`);
+        }
+        const op = {
+            ts: new Date().toISOString(),
+            op: "delete",
+            id,
+            prev,
+        };
+        this.records.delete(id);
+        await this.persistOp(op);
+    }
+    _deleteSync(id) {
+        const prev = this.records.get(id);
+        if (prev === undefined) {
+            throw new Error(`Record '${id}' not found`);
+        }
+        const op = {
+            ts: new Date().toISOString(),
+            op: "delete",
+            id,
+            prev,
+        };
+        this.records.delete(id);
+        this.batchOps.push(op);
+    }
+    async _batch(fn) {
         this.batching = true;
         this.batchOps = [];
         try {
@@ -170,7 +281,7 @@ export class Store {
                 await appendOps(join(this.dir, this.activeOpsPath), this.batchOps);
                 this.ops.push(...this.batchOps);
                 if (this.ops.length >= this.options.checkpointThreshold) {
-                    await this.compact();
+                    await this._compact();
                 }
             }
         }
@@ -191,8 +302,7 @@ export class Store {
             this.batchOps = [];
         }
     }
-    async undo() {
-        this.ensureOpen();
+    async _undo() {
         if (this.ops.length === 0)
             return false;
         const lastOp = this.ops[this.ops.length - 1];
@@ -201,18 +311,7 @@ export class Store {
         await truncateLastOp(join(this.dir, this.activeOpsPath));
         return true;
     }
-    getHistory(id) {
-        this.ensureOpen();
-        return this.ops.filter((op) => op.id === id);
-    }
-    getOps(since) {
-        this.ensureOpen();
-        if (!since)
-            return [...this.ops];
-        return this.ops.filter((op) => op.ts > since);
-    }
-    async compact() {
-        this.ensureOpen();
+    async _compact() {
         const snapshotPath = await writeSnapshot(this.dir, this.records, this.version);
         const opsFilename = `ops-${Date.now()}.jsonl`;
         const opsPath = `ops/${opsFilename}`;
@@ -234,8 +333,7 @@ export class Store {
         this.activeOpsPath = opsPath;
         this.ops = [];
     }
-    async archive(predicate, segment) {
-        this.ensureOpen();
+    async _archive(predicate, segment) {
         const toArchive = new Map();
         for (const [id, value] of this.records) {
             if (predicate(value, id))
@@ -252,32 +350,18 @@ export class Store {
             this.records.delete(id);
         }
         this.archivedRecordCount += toArchive.size;
-        await this.compact();
+        await this._compact();
         return toArchive.size;
     }
-    async loadArchive(segment) {
-        this.ensureOpen();
-        const segmentPath = this.archiveSegments.find((s) => s === `archive/archive-${segment}.json`) || this.archiveSegments.find((s) => s.includes(segment));
-        if (!segmentPath)
-            throw new Error(`Archive segment '${segment}' not found`);
-        return loadArchiveSegment(this.dir, segmentPath);
-    }
-    listArchiveSegments() {
-        this.ensureOpen();
-        return [...this.archiveSegments];
-    }
-    stats() {
-        this.ensureOpen();
-        return {
-            activeRecords: this.records.size,
-            opsCount: this.ops.length,
-            archiveSegments: this.archiveSegments.length,
-        };
-    }
+    // --- Helpers ---
     ensureOpen() {
         if (!this.opened)
             throw new Error("Store is not open. Call open() first.");
     }
+    ensureWritable() {
+        if (this.options.readOnly)
+            throw new Error("Store is read-only. Cannot perform mutations.");
+    }
     applyOp(op) {
         if (op.op === "set" && op.data !== undefined) {
             this.records.set(op.id, op.data);
@@ -304,7 +388,7 @@ export class Store {
         await appendOp(join(this.dir, this.activeOpsPath), op);
         this.ops.push(op);
         if (this.ops.length >= this.options.checkpointThreshold) {
-            await this.compact();
+            await this._compact();
         }
     }
     defaultPeriod() {

package/dist/types.d.ts

@@ -9,6 +9,8 @@ export interface Operation<T = Record<string, unknown>> {
     data?: T;
     /** Previous value (null for creates, full record for updates/deletes) */
     prev: T | null;
+    /** Encoding format for prev field. Omitted or "full" = full record. "delta" = JSON Patch (future). */
+    encoding?: "full" | "delta";
 }
 export interface Snapshot<T = Record<string, unknown>> {
     version: number;
@@ -44,6 +46,8 @@ export interface StoreOptions {
     version?: number;
     /** Migration function: called if stored version < current version */
     migrate?: (record: unknown, fromVersion: number) => unknown;
+    /** Open in read-only mode: skips directory lock, rejects all mutations. */
+    readOnly?: boolean;
 }
 export interface StoreStats {
     activeRecords: number;

package/dist/validate.js

@@ -21,6 +21,9 @@ export function validateOp(raw) {
         throw new Error("Invalid operation: delete op must have non-null prev");
     if (obj.op === "delete" && "data" in obj)
         throw new Error("Invalid operation: delete op must not have data field");
+    if ("encoding" in obj && obj.encoding !== "full" && obj.encoding !== "delta") {
+        throw new Error(`Invalid operation: encoding must be "full" or "delta", got "${obj.encoding}"`);
+    }
     return raw;
 }
 export function validateManifest(raw) {

package/dist/wal.js

@@ -1,4 +1,4 @@
-import { appendFile, readFile, writeFile } from "node:fs/promises";
+import { appendFile, readFile, open } from "node:fs/promises";
 import { validateOp } from "./validate.js";
 export async function appendOp(path, op) {
     await appendFile(path, JSON.stringify(op) + "\n", "utf-8");
@@ -32,18 +32,45 @@ export async function readOps(path) {
     return ops;
 }
 export async function truncateLastOp(path) {
-    let content;
+    let fh;
     try {
-        content = await readFile(path, "utf-8");
+        fh = await open(path, "r+");
     }
     catch {
         return false;
     }
-    const lines = content.trim().split("\n").filter(Boolean);
-    if (lines.length === 0)
-        return false;
-    lines.pop();
-    const newContent = lines.length > 0 ? lines.join("\n") + "\n" : "";
-    await writeFile(path, newContent, "utf-8");
-    return true;
+    try {
+        const { size } = await fh.stat();
+        if (size === 0)
+            return false;
+        // Read the tail of the file to find the second-to-last newline.
+        // 4KB handles operations up to ~4KB. For larger ops, read in chunks.
+        let readSize = Math.min(4096, size);
+        let readPos = size - readSize;
+        let lastNl = -1;
+        while (true) {
+            const buf = Buffer.alloc(readSize);
+            await fh.read(buf, 0, readSize, readPos);
+            const text = buf.toString("utf-8", 0, readSize);
+            // Find the second-to-last newline (skip trailing newline)
+            lastNl = text.lastIndexOf("\n", text.length - 2);
+            if (lastNl !== -1) {
+                await fh.truncate(readPos + lastNl + 1);
+                return true;
+            }
+            // No newline found in this chunk — need to read further back
+            if (readPos === 0) {
+                // Only one line in the entire file — truncate to empty
+                await fh.truncate(0);
+                return true;
+            }
+            // Read the next chunk further back
+            const nextSize = Math.min(4096, readPos);
+            readPos -= nextSize;
+            readSize = nextSize;
+        }
+    }
+    finally {
+        await fh.close();
+    }
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@backloghq/opslog",
-  "version": "0.1.4",
+  "version": "0.2.0",
   "description": "Embedded event-sourced document store. Append-only operation log with immutable snapshots, zero native dependencies.",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",