@backloghq/opslog 0.2.0 → 0.4.0

package/README.md

@@ -54,13 +54,14 @@ State survives restarts — reopen the same directory and everything is there.
 
 ```
 data/
-  manifest.json              # Points to current snapshot + ops file
+  manifest.json                    # Points to current snapshot + ops file(s)
   snapshots/
-    snap-<timestamp>.json    # Immutable full-state capture
+    snap-<timestamp>.json          # Immutable full-state capture
   ops/
-    ops-<timestamp>.jsonl    # Append-only operation log
+    ops-<timestamp>.jsonl          # Append-only operation log (single-writer)
+    agent-<id>-<timestamp>.jsonl   # Per-agent operation log (multi-writer)
   archive/
-    archive-<period>.json    # Old records, lazy-loaded
+    archive-<period>.json          # Old records, lazy-loaded
 ```
 
 **Writes** append an operation (one JSON line) to the ops file. **Reads** come from an in-memory map built from the latest snapshot + ops replay. **Checkpoints** materialize current state as a new immutable snapshot.
@@ -121,6 +122,7 @@ await store.archive(predicate)    // Move matching records to archive
 await store.loadArchive(segment)  // Lazy-load archived records
 store.listArchiveSegments()       // List available archive files
 store.stats()                     // { activeRecords, opsCount, archiveSegments }
+await store.refresh()             // Reload from all agent WALs (multi-writer only)
 ```
 
 ## Options
@@ -132,9 +134,80 @@ await store.open(dir, {
   version: 1,                     // Schema version
   migrate: (record, fromVersion) => record, // Migration function
   readOnly: false,                // Open in read-only mode (default: false)
+  agentId: "agent-A",             // Enable multi-writer mode (optional)
+  backend: new FsBackend(),       // Custom storage backend (optional, default: FsBackend)
 });
 ```
 
+## Multi-Writer Mode
+
+Multiple agents can write to the same store concurrently. Each agent gets its own WAL file — no write contention.
+
+```typescript
+// Agent A (process 1 / machine 1)
+const storeA = new Store<Task>();
+await storeA.open("./data", { agentId: "agent-A" });
+await storeA.set("task-1", { title: "Build API", status: "active" });
+await storeA.close();
+
+// Agent B (process 2 / machine 2)
+const storeB = new Store<Task>();
+await storeB.open("./data", { agentId: "agent-B" });
+// B sees A's writes on open
+storeB.get("task-1"); // { title: "Build API", status: "active" }
+await storeB.set("task-2", { title: "Write tests", status: "active" });
+await storeB.close();
+```
+
+### How it works
+
+- Each agent writes to `ops/agent-{id}-{timestamp}.jsonl` — separate files, no locking needed for writes
+- Operations carry a [Lamport clock](https://en.wikipedia.org/wiki/Lamport_timestamp) for ordering
+- On `open()`, all agent WAL files are merge-sorted by `(clock, agentId)` for a deterministic total order
+- Conflicts (two agents write the same key) are resolved with **last-writer-wins** by clock value
+- `undo()` only undoes the calling agent's last operation
+- `compact()` acquires a compaction lock, snapshots the merged state, and resets all WAL files
+- `refresh()` re-reads all agent WALs to pick up other agents' writes
+
+### Conflict resolution
+
+When two agents modify the same key, the operation with the higher Lamport clock wins. If clocks are equal, the lexicographically higher agent ID wins. This is deterministic — all agents arrive at the same state regardless of replay order.
+
+```typescript
+// Agent A sets "shared" (clock=1)
+await storeA.set("shared", { value: "from-A" });
+
+// Agent B opens (sees clock=1), sets "shared" (clock=2)
+await storeB.set("shared", { value: "from-B" });
+
+// B wins — higher clock
+store.get("shared"); // { value: "from-B" }
+```
+
+## Custom Storage Backend
+
+opslog uses a pluggable `StorageBackend` interface for all I/O. The default is `FsBackend` (local filesystem). You can implement your own backend for S3, databases, or other storage systems.
+
+```typescript
+import { Store, FsBackend } from "@backloghq/opslog";
+import type { StorageBackend } from "@backloghq/opslog";
+
+// Use the default filesystem backend (implicit)
+const store = new Store();
+await store.open("./data");
+
+// Or pass a custom backend explicitly
+const store = new Store();
+await store.open("./data", { backend: new FsBackend() });
+
+// Or implement your own
+class S3Backend implements StorageBackend {
+  // ... implement all methods
+}
+const store = new Store();
+await store.open("s3://bucket/prefix", { backend: new S3Backend() });
+```
+
 ## Read-Only Mode
 
 Open a store for reading without acquiring the write lock. Useful for dashboards, backup processes, or multiple readers alongside a single writer.

package/dist/backend.d.ts

@@ -0,0 +1,30 @@
+import type { LockHandle, Manifest, Operation, StorageBackend } from "./types.js";
+/** Filesystem-backed storage backend. Default backend for opslog. */
+export declare class FsBackend implements StorageBackend {
+    private dir;
+    initialize(dir: string, opts: {
+        readOnly: boolean;
+    }): Promise<void>;
+    shutdown(): Promise<void>;
+    readManifest(): Promise<Manifest | null>;
+    writeManifest(manifest: Manifest): Promise<void>;
+    writeSnapshot(records: Map<string, unknown>, version: number): Promise<string>;
+    loadSnapshot(relativePath: string): Promise<{
+        records: Map<string, unknown>;
+        version: number;
+    }>;
+    appendOps(relativePath: string, ops: Operation[]): Promise<void>;
+    readOps(relativePath: string): Promise<Operation[]>;
+    truncateLastOp(relativePath: string): Promise<boolean>;
+    createOpsFile(): Promise<string>;
+    writeArchiveSegment(period: string, records: Map<string, unknown>): Promise<string>;
+    loadArchiveSegment(relativePath: string): Promise<Map<string, unknown>>;
+    listArchiveSegments(): Promise<string[]>;
+    acquireLock(): Promise<LockHandle>;
+    releaseLock(handle: LockHandle): Promise<void>;
+    createAgentOpsFile(agentId: string): Promise<string>;
+    listOpsFiles(): Promise<string[]>;
+    acquireCompactionLock(): Promise<LockHandle>;
+    releaseCompactionLock(handle: LockHandle): Promise<void>;
+    getManifestVersion(): Promise<string | null>;
+}

package/dist/backend.js

@@ -0,0 +1,134 @@
+import { mkdir, open, readdir, stat, unlink, writeFile } from "node:fs/promises";
+import { join } from "node:path";
+import { appendOp, appendOps, readOps, truncateLastOp } from "./wal.js";
+import { loadSnapshot, writeSnapshot } from "./snapshot.js";
+import { readManifest, writeManifest } from "./manifest.js";
+import { loadArchiveSegment, writeArchiveSegment, listArchiveSegments as fsListArchiveSegments, } from "./archive.js";
+import { acquireLock as fsAcquireLock, releaseLock as fsReleaseLock, } from "./lock.js";
+class FsLockHandle {
+    fh;
+    dir;
+    constructor(fh, dir) {
+        this.fh = fh;
+        this.dir = dir;
+    }
+}
+/** Filesystem-backed storage backend. Default backend for opslog. */
+export class FsBackend {
+    dir = "";
+    async initialize(dir, opts) {
+        this.dir = dir;
+        if (!opts.readOnly) {
+            await mkdir(join(dir, "snapshots"), { recursive: true });
+            await mkdir(join(dir, "ops"), { recursive: true });
+            await mkdir(join(dir, "archive"), { recursive: true });
+        }
+    }
+    async shutdown() {
+        // No-op for filesystem backend
+    }
+    // -- Manifest --
+    async readManifest() {
+        return readManifest(this.dir);
+    }
+    async writeManifest(manifest) {
+        return writeManifest(this.dir, manifest);
+    }
+    // -- Snapshots --
+    async writeSnapshot(records, version) {
+        return writeSnapshot(this.dir, records, version);
+    }
+    async loadSnapshot(relativePath) {
+        return loadSnapshot(this.dir, relativePath);
+    }
+    // -- WAL --
+    async appendOps(relativePath, ops) {
+        const fullPath = join(this.dir, relativePath);
+        if (ops.length === 1) {
+            return appendOp(fullPath, ops[0]);
+        }
+        return appendOps(fullPath, ops);
+    }
+    async readOps(relativePath) {
+        return readOps(join(this.dir, relativePath));
+    }
+    async truncateLastOp(relativePath) {
+        return truncateLastOp(join(this.dir, relativePath));
+    }
+    async createOpsFile() {
+        const filename = `ops-${Date.now()}.jsonl`;
+        const relativePath = `ops/${filename}`;
+        await writeFile(join(this.dir, relativePath), "", "utf-8");
+        return relativePath;
+    }
+    // -- Archive --
+    async writeArchiveSegment(period, records) {
+        return writeArchiveSegment(this.dir, period, records);
+    }
+    async loadArchiveSegment(relativePath) {
+        return loadArchiveSegment(this.dir, relativePath);
+    }
+    async listArchiveSegments() {
+        return fsListArchiveSegments(this.dir);
+    }
+    // -- Locking (single-writer) --
+    async acquireLock() {
+        const fh = await fsAcquireLock(this.dir);
+        return new FsLockHandle(fh, this.dir);
+    }
+    async releaseLock(handle) {
+        const fsHandle = handle;
+        return fsReleaseLock(fsHandle.dir, fsHandle.fh);
+    }
+    // -- Multi-writer extensions --
+    async createAgentOpsFile(agentId) {
+        const filename = `agent-${agentId}-${Date.now()}.jsonl`;
+        const relativePath = `ops/${filename}`;
+        await writeFile(join(this.dir, relativePath), "", "utf-8");
+        return relativePath;
+    }
+    async listOpsFiles() {
+        const opsDir = join(this.dir, "ops");
+        try {
+            const files = await readdir(opsDir);
+            return files.filter((f) => f.endsWith(".jsonl")).map((f) => `ops/${f}`);
+        }
+        catch {
+            return [];
+        }
+    }
+    async acquireCompactionLock() {
+        const lockPath = join(this.dir, ".compact-lock");
+        let fh;
+        try {
+            fh = await open(lockPath, "wx");
+        }
+        catch (err) {
+            if (err.code === "EEXIST") {
+                throw new Error("Compaction lock held by another agent", { cause: err });
+            }
+            throw err;
+        }
+        await fh.writeFile(String(process.pid), "utf-8");
+        return new FsLockHandle(fh, this.dir);
+    }
+    async releaseCompactionLock(handle) {
+        const fsHandle = handle;
+        await fsHandle.fh.close();
+        try {
+            await unlink(join(fsHandle.dir, ".compact-lock"));
+        }
+        catch {
+            // Already cleaned up
+        }
+    }
+    async getManifestVersion() {
+        try {
+            const s = await stat(join(this.dir, "manifest.json"));
+            return s.mtimeMs.toString();
+        }
+        catch {
+            return null;
+        }
+    }
+}

package/dist/clock.d.ts

@@ -0,0 +1,16 @@
+/**
+ * Lamport logical clock for multi-writer operation ordering.
+ * Each agent maintains its own clock. On local events, tick().
+ * On receiving remote events, merge(received) to stay ahead.
+ * Ties are broken by agent ID (lexicographic) for deterministic total order.
+ */
+export declare class LamportClock {
+    private counter;
+    constructor(initial?: number);
+    /** Increment and return the new value (for local events). */
+    tick(): number;
+    /** Merge with a received clock value and increment. */
+    merge(received: number): number;
+    /** Current clock value without incrementing. */
+    get current(): number;
+}

package/dist/clock.js

@@ -0,0 +1,25 @@
+/**
+ * Lamport logical clock for multi-writer operation ordering.
+ * Each agent maintains its own clock. On local events, tick().
+ * On receiving remote events, merge(received) to stay ahead.
+ * Ties are broken by agent ID (lexicographic) for deterministic total order.
+ */
+export class LamportClock {
+    counter;
+    constructor(initial = 0) {
+        this.counter = initial;
+    }
+    /** Increment and return the new value (for local events). */
+    tick() {
+        return ++this.counter;
+    }
+    /** Merge with a received clock value and increment. */
+    merge(received) {
+        this.counter = Math.max(this.counter, received) + 1;
+        return this.counter;
+    }
+    /** Current clock value without incrementing. */
+    get current() {
+        return this.counter;
+    }
+}

package/dist/delta.d.ts

@@ -0,0 +1,30 @@
+/**
+ * Delta encoding for operations.
+ * Instead of storing the full previous record, store only the diff.
+ *
+ * Uses a simplified JSON Patch-like format:
+ * - Only tracks changed/added/removed top-level keys
+ * - prev becomes a patch object: { $set: {...}, $unset: [...] }
+ */
+export interface DeltaPatch {
+    /** Fields that were changed or added (old values). */
+    $set?: Record<string, unknown>;
+    /** Fields that were removed. */
+    $unset?: string[];
+}
+/**
+ * Create a delta patch from an old record to a new record.
+ * The patch, when applied to the new record, produces the old record.
+ * This is the "reverse patch" — stored as `prev` so undo can apply it.
+ */
+export declare function createDelta(oldRecord: Record<string, unknown> | null, newRecord: Record<string, unknown>): DeltaPatch | null;
+/**
+ * Apply a delta patch to a record to produce the previous version.
+ * Used during undo: apply the reverse patch to current record → get old record.
+ */
+export declare function applyDelta(record: Record<string, unknown>, patch: DeltaPatch): Record<string, unknown>;
+/**
+ * Check if a delta patch is smaller than the full record.
+ * Used to decide whether to use delta or full encoding.
+ */
+export declare function isDeltaSmaller(patch: DeltaPatch | null, fullRecord: Record<string, unknown> | null): boolean;

package/dist/delta.js

@@ -0,0 +1,75 @@
+/**
+ * Delta encoding for operations.
+ * Instead of storing the full previous record, store only the diff.
+ *
+ * Uses a simplified JSON Patch-like format:
+ * - Only tracks changed/added/removed top-level keys
+ * - prev becomes a patch object: { $set: {...}, $unset: [...] }
+ */
+/**
+ * Create a delta patch from an old record to a new record.
+ * The patch, when applied to the new record, produces the old record.
+ * This is the "reverse patch" — stored as `prev` so undo can apply it.
+ */
+export function createDelta(oldRecord, newRecord) {
+    if (oldRecord === null)
+        return null; // Create operation — no previous
+    const patch = {};
+    const $set = {};
+    const $unset = [];
+    // Fields in old that differ from new (changed or removed in new)
+    for (const [key, oldVal] of Object.entries(oldRecord)) {
+        if (!(key in newRecord)) {
+            // Field was removed in new → to restore old, we need to $set it
+            $set[key] = oldVal;
+        }
+        else if (JSON.stringify(oldVal) !== JSON.stringify(newRecord[key])) {
+            // Field changed → store old value
+            $set[key] = oldVal;
+        }
+    }
+    // Fields in new that weren't in old (added in new)
+    for (const key of Object.keys(newRecord)) {
+        if (!(key in oldRecord)) {
+            // Field was added in new → to restore old, we need to $unset it
+            $unset.push(key);
+        }
+    }
+    if (Object.keys($set).length > 0)
+        patch.$set = $set;
+    if ($unset.length > 0)
+        patch.$unset = $unset;
+    // If no changes, return empty patch
+    if (!patch.$set && !patch.$unset)
+        return null;
+    return patch;
+}
+/**
+ * Apply a delta patch to a record to produce the previous version.
+ * Used during undo: apply the reverse patch to current record → get old record.
+ */
+export function applyDelta(record, patch) {
+    const result = { ...record };
+    if (patch.$set) {
+        for (const [key, val] of Object.entries(patch.$set)) {
+            result[key] = val;
+        }
+    }
+    if (patch.$unset) {
+        for (const key of patch.$unset) {
+            delete result[key];
+        }
+    }
+    return result;
+}
+/**
+ * Check if a delta patch is smaller than the full record.
+ * Used to decide whether to use delta or full encoding.
+ */
+export function isDeltaSmaller(patch, fullRecord) {
+    if (patch === null || fullRecord === null)
+        return false;
+    const patchSize = JSON.stringify(patch).length;
+    const fullSize = JSON.stringify(fullRecord).length;
+    return patchSize < fullSize;
+}

package/dist/index.d.ts

@@ -1,3 +1,8 @@
 export { Store } from "./store.js";
+export { FsBackend } from "./backend.js";
+export { LamportClock } from "./clock.js";
+export { createDelta, applyDelta, isDeltaSmaller } from "./delta.js";
+export type { DeltaPatch } from "./delta.js";
 export { acquireLock, releaseLock } from "./lock.js";
-export type { Operation, Snapshot, Manifest, ManifestStats, ArchiveSegment, StoreOptions, StoreStats, } from "./types.js";
+export { validateOp, validateManifest, validateSnapshot, validateArchiveSegment, } from "./validate.js";
+export type { Operation, Snapshot, Manifest, ManifestStats, ArchiveSegment, StoreOptions, StoreStats, StorageBackend, LockHandle, } from "./types.js";

package/dist/index.js

@@ -1,2 +1,6 @@
 export { Store } from "./store.js";
+export { FsBackend } from "./backend.js";
+export { LamportClock } from "./clock.js";
+export { createDelta, applyDelta, isDeltaSmaller } from "./delta.js";
 export { acquireLock, releaseLock } from "./lock.js";
+export { validateOp, validateManifest, validateSnapshot, validateArchiveSegment, } from "./validate.js";

package/dist/store.d.ts

@@ -8,20 +8,26 @@ export declare class Store<T = Record<string, unknown>> {
     private version;
     private activeOpsPath;
     private created;
-    private options;
+    private coreOpts;
     private archivedRecordCount;
     private batching;
     private batchOps;
     private _lock;
-    private lockFh;
+    private lockHandle;
+    private backend;
+    private agentId?;
+    private clock;
+    private manifestVersion;
     /**
      * Serialize all state-mutating operations through a promise chain.
-     * This prevents interleaving of async mutations (e.g. compact + set,
-     * undo + set) which could corrupt the WAL or in-memory state.
-     * Read operations remain synchronous and lock-free.
+     * Prevents interleaving of async mutations. Reads remain synchronous and lock-free.
      */
     private serialize;
+    private isMultiWriter;
     open(dir: string, options?: StoreOptions): Promise<void>;
+    private initFreshStore;
+    private loadExistingStore;
+    private loadMultiWriterOps;
     close(): Promise<void>;
     get(id: string): T | undefined;
     set(id: string, value: T): Promise<void> | void;
@@ -40,14 +46,42 @@ export declare class Store<T = Record<string, unknown>> {
     listArchiveSegments(): string[];
     loadArchive(segment: string): Promise<Map<string, T>>;
     stats(): StoreStats;
+    /**
+     * Reload state from the backend (multi-writer mode).
+     * Re-reads the manifest, snapshot, and all agent WAL files.
+     * Use this to pick up writes from other agents.
+     */
+    refresh(): Promise<void>;
+    private watchTimer;
+    private watchCallback;
+    /**
+     * Tail the WAL for new operations. Re-reads the active ops file
+     * and replays any new operations since the last known count.
+     * Returns the newly applied operations.
+     * Works in any mode (single-writer readOnly, multi-writer, etc).
+     */
+    tail(): Promise<Operation<T>[]>;
+    /**
+     * Watch for new operations on an interval.
+     * Calls the callback with new operations whenever they appear.
+     * @param callback Called with new operations
+     * @param intervalMs Polling interval in milliseconds (default: 1000)
+     */
+    watch(callback: (ops: Operation<T>[]) => void, intervalMs?: number): void;
+    /** Stop watching for new operations. */
+    unwatch(): void;
+    private makeOp;
     private _set;
     private _setSync;
     private _delete;
     private _deleteSync;
     private _batch;
     private _undo;
+    private _undoMultiWriter;
     private _compact;
+    private _compactMultiWriter;
     private _archive;
+    private _refresh;
     private ensureOpen;
     private ensureWritable;
     private applyOp;

package/dist/store.js

@@ -1,10 +1,7 @@
-import { mkdir, writeFile } from "node:fs/promises";
-import { join } from "node:path";
-import { appendOp, appendOps, readOps, truncateLastOp } from "./wal.js";
-import { loadSnapshot, writeSnapshot } from "./snapshot.js";
-import { createDefaultManifest, readManifest, writeManifest, } from "./manifest.js";
-import { loadArchiveSegment, writeArchiveSegment, } from "./archive.js";
-import { acquireLock, releaseLock } from "./lock.js";
+import { createDefaultManifest } from "./manifest.js";
+import { FsBackend } from "./backend.js";
+import { LamportClock } from "./clock.js";
+import { createDelta, applyDelta, isDeltaSmaller } from "./delta.js";
 export class Store {
     dir = "";
     records = new Map();
@@ -14,7 +11,7 @@ export class Store {
     version = 1;
     activeOpsPath = "";
     created = "";
-    options = {
+    coreOpts = {
         checkpointThreshold: 100,
         checkpointOnClose: true,
         version: 1,
@@ -25,12 +22,15 @@ export class Store {
     batching = false;
     batchOps = [];
     _lock = Promise.resolve();
-    lockFh = null;
+    lockHandle = null;
+    backend;
+    // Multi-writer state
+    agentId;
+    clock = null;
+    manifestVersion = null;
     /**
      * Serialize all state-mutating operations through a promise chain.
-     * This prevents interleaving of async mutations (e.g. compact + set,
-     * undo + set) which could corrupt the WAL or in-memory state.
-     * Read operations remain synchronous and lock-free.
+     * Prevents interleaving of async mutations. Reads remain synchronous and lock-free.
      */
     serialize(fn) {
         const prev = this._lock;
@@ -40,79 +40,159 @@ export class Store {
         });
         return prev.then(fn).finally(() => resolve());
     }
+    isMultiWriter() {
+        return this.agentId !== undefined;
+    }
     async open(dir, options) {
         this.dir = dir;
         if (options) {
-            this.options = { ...this.options, ...options };
+            const { backend, agentId, ...rest } = options;
+            this.coreOpts = { ...this.coreOpts, ...rest };
+            if (backend)
+                this.backend = backend;
+            if (agentId)
+                this.agentId = agentId;
         }
-        if (!this.options.readOnly) {
-            await mkdir(join(dir, "snapshots"), { recursive: true });
-            await mkdir(join(dir, "ops"), { recursive: true });
-            await mkdir(join(dir, "archive"), { recursive: true });
-            this.lockFh = await acquireLock(dir);
+        this.backend ??= new FsBackend();
+        await this.backend.initialize(dir, { readOnly: this.coreOpts.readOnly });
+        // Acquire write lock (single-writer only, not readOnly)
+        if (!this.coreOpts.readOnly && !this.isMultiWriter()) {
+            this.lockHandle = await this.backend.acquireLock();
         }
-        const manifest = await readManifest(dir);
+        const manifest = await this.backend.readManifest();
         if (!manifest) {
-            if (this.options.readOnly) {
+            if (this.coreOpts.readOnly) {
                 throw new Error("Cannot open in readOnly mode: no existing store found");
             }
-            // Fresh store — create empty snapshot and manifest
-            const snapshotPath = await writeSnapshot(dir, new Map(), this.options.version);
-            const opsFilename = `ops-${Date.now()}.jsonl`;
-            const opsPath = `ops/${opsFilename}`;
-            await writeFile(join(dir, opsPath), "", "utf-8");
-            const newManifest = createDefaultManifest(snapshotPath, opsPath);
-            await writeManifest(dir, newManifest);
-            this.version = this.options.version;
-            this.activeOpsPath = opsPath;
-            this.created = newManifest.stats.created;
-            this.archiveSegments = [];
+            await this.initFreshStore();
         }
         else {
-            // Load existing state
-            let snapshotData;
-            try {
-                snapshotData = await loadSnapshot(dir, manifest.currentSnapshot);
-            }
-            catch (err) {
-                const isNotFound = err instanceof Error && "code" in err && err.code === "ENOENT";
-                if (isNotFound) {
-                    throw new Error(`Snapshot file not found: ${manifest.currentSnapshot}. The data directory may be corrupted.`, { cause: err });
-                }
-                throw err;
+            await this.loadExistingStore(manifest);
+        }
+        this.manifestVersion = await this.backend.getManifestVersion();
+        this.opened = true;
+    }
+    async initFreshStore() {
+        const snapshotPath = await this.backend.writeSnapshot(new Map(), this.coreOpts.version);
+        let opsPath;
+        if (this.isMultiWriter()) {
+            opsPath = await this.backend.createAgentOpsFile(this.agentId);
+        }
+        else {
+            opsPath = await this.backend.createOpsFile();
+        }
+        const newManifest = createDefaultManifest(snapshotPath, opsPath);
+        if (this.isMultiWriter()) {
+            newManifest.activeAgentOps = { [this.agentId]: opsPath };
+        }
+        await this.backend.writeManifest(newManifest);
+        this.version = this.coreOpts.version;
+        this.activeOpsPath = opsPath;
+        this.created = newManifest.stats.created;
+        this.archiveSegments = [];
+        if (this.isMultiWriter()) {
+            this.clock = new LamportClock(0);
+        }
+    }
+    async loadExistingStore(manifest) {
+        // Load snapshot
+        let snapshotData;
+        try {
+            snapshotData = await this.backend.loadSnapshot(manifest.currentSnapshot);
+        }
+        catch (err) {
+            const isNotFound = err instanceof Error &&
+                "code" in err &&
+                err.code === "ENOENT";
+            if (isNotFound) {
+                throw new Error(`Snapshot file not found: ${manifest.currentSnapshot}. The data directory may be corrupted.`, { cause: err });
             }
-            const { records, version: storedVersion } = snapshotData;
-            this.records = records;
-            this.version = storedVersion;
-            this.activeOpsPath = manifest.activeOps;
-            this.created = manifest.stats.created;
-            this.archiveSegments = manifest.archiveSegments;
-            this.archivedRecordCount = manifest.stats.archivedRecords;
-            // Migrate if needed
-            if (storedVersion < this.options.version) {
-                for (const [id, record] of this.records) {
-                    this.records.set(id, this.options.migrate(record, storedVersion));
-                }
-                this.version = this.options.version;
+            throw err;
+        }
+        const { records, version: storedVersion } = snapshotData;
+        this.records = records;
+        this.version = storedVersion;
+        this.created = manifest.stats.created;
+        this.archiveSegments = manifest.archiveSegments;
+        this.archivedRecordCount = manifest.stats.archivedRecords;
+        // Migrate if needed
+        if (storedVersion < this.coreOpts.version) {
+            for (const [id, record] of this.records) {
+                this.records.set(id, this.coreOpts.migrate(record, storedVersion));
             }
-            // Replay ops
-            const ops = await readOps(join(dir, manifest.activeOps));
+            this.version = this.coreOpts.version;
+        }
+        if (this.isMultiWriter()) {
+            await this.loadMultiWriterOps(manifest);
+        }
+        else {
+            // Single-writer: replay ops from active ops file
+            const ops = (await this.backend.readOps(manifest.activeOps));
             for (const op of ops) {
                 this.applyOp(op);
             }
             this.ops = ops;
+            this.activeOpsPath = manifest.activeOps;
+        }
+    }
+    async loadMultiWriterOps(manifest) {
+        const allOps = [];
+        // Read all agent ops files
+        if (manifest.activeAgentOps) {
+            for (const opsPath of Object.values(manifest.activeAgentOps)) {
+                const ops = (await this.backend.readOps(opsPath));
+                allOps.push(...ops);
+            }
+        }
+        // Also read legacy single-writer ops for backward compat
+        if (manifest.activeOps && !manifest.activeAgentOps) {
+            const ops = (await this.backend.readOps(manifest.activeOps));
+            allOps.push(...ops);
+        }
+        // Merge-sort by (clock, agent) for deterministic total order
+        allOps.sort((a, b) => {
+            const clockDiff = (a.clock ?? 0) - (b.clock ?? 0);
+            if (clockDiff !== 0)
+                return clockDiff;
+            return (a.agent ?? "").localeCompare(b.agent ?? "");
+        });
+        for (const op of allOps) {
+            this.applyOp(op);
+        }
+        this.ops = allOps;
+        // Initialize Lamport clock from max seen value
+        const maxClock = allOps.reduce((max, op) => Math.max(max, op.clock ?? 0), 0);
+        this.clock = new LamportClock(maxClock);
+        // Find or create our agent's ops file
+        if (manifest.activeAgentOps?.[this.agentId]) {
+            this.activeOpsPath = manifest.activeAgentOps[this.agentId];
+        }
+        else {
+            // Register this agent in the manifest
+            this.activeOpsPath = await this.backend.createAgentOpsFile(this.agentId);
+            const updatedManifest = {
+                ...manifest,
+                activeAgentOps: {
+                    ...(manifest.activeAgentOps ?? {}),
+                    [this.agentId]: this.activeOpsPath,
+                },
+            };
+            await this.backend.writeManifest(updatedManifest);
         }
-        this.opened = true;
     }
     async close() {
         this.ensureOpen();
-        if (!this.options.readOnly && this.options.checkpointOnClose && this.ops.length > 0) {
+        this.unwatch();
+        if (!this.coreOpts.readOnly &&
+            this.coreOpts.checkpointOnClose &&
+            this.ops.length > 0) {
             await this.serialize(() => this._compact());
         }
-        if (this.lockFh) {
-            await releaseLock(this.dir, this.lockFh);
-            this.lockFh = null;
+        if (this.lockHandle) {
+            await this.backend.releaseLock(this.lockHandle);
+            this.lockHandle = null;
         }
+        await this.backend.shutdown();
         this.opened = false;
     }
     get(id) {
@@ -208,7 +288,7 @@ export class Store {
         const segmentPath = this.archiveSegments.find((s) => s === `archive/archive-${segment}.json`) || this.archiveSegments.find((s) => s.includes(segment));
         if (!segmentPath)
             throw new Error(`Archive segment '${segment}' not found`);
-        return loadArchiveSegment(this.dir, segmentPath);
+        return this.backend.loadArchiveSegment(segmentPath);
     }
     stats() {
         this.ensureOpen();
@@ -218,28 +298,105 @@ export class Store {
             archiveSegments: this.archiveSegments.length,
         };
     }
+    /**
+     * Reload state from the backend (multi-writer mode).
+     * Re-reads the manifest, snapshot, and all agent WAL files.
+     * Use this to pick up writes from other agents.
+     */
+    async refresh() {
+        this.ensureOpen();
+        if (!this.isMultiWriter()) {
+            throw new Error("refresh() is only available in multi-writer mode");
+        }
+        return this.serialize(() => this._refresh());
+    }
+    // --- WAL tailing ---
+    watchTimer = null;
+    watchCallback = null;
+    /**
+     * Tail the WAL for new operations. Re-reads the active ops file
+     * and replays any new operations since the last known count.
+     * Returns the newly applied operations.
+     * Works in any mode (single-writer readOnly, multi-writer, etc).
+     */
+    async tail() {
+        this.ensureOpen();
+        const prevCount = this.ops.length;
+        // Re-read the ops file for new entries
+        const allOps = (await this.backend.readOps(this.activeOpsPath));
+        if (allOps.length <= prevCount)
+            return [];
+        const newOps = allOps.slice(prevCount);
+        for (const op of newOps) {
+            this.applyOp(op);
+        }
+        this.ops.push(...newOps);
+        return newOps;
+    }
+    /**
+     * Watch for new operations on an interval.
+     * Calls the callback with new operations whenever they appear.
+     * @param callback Called with new operations
+     * @param intervalMs Polling interval in milliseconds (default: 1000)
+     */
+    watch(callback, intervalMs = 1000) {
+        this.ensureOpen();
+        if (this.watchTimer)
+            this.unwatch();
+        this.watchCallback = callback;
+        this.watchTimer = setInterval(async () => {
+            try {
+                const newOps = await this.tail();
+                if (newOps.length > 0 && this.watchCallback) {
+                    this.watchCallback(newOps);
+                }
+            }
+            catch {
+                // Silently ignore tail errors during watch
+            }
+        }, intervalMs);
+    }
+    /** Stop watching for new operations. */
+    unwatch() {
+        if (this.watchTimer) {
+            clearInterval(this.watchTimer);
+            this.watchTimer = null;
+        }
+        this.watchCallback = null;
+    }
     // --- Private mutation implementations ---
-    async _set(id, value) {
-        const prev = this.records.get(id) ?? null;
+    makeOp(type, id, data, prev) {
         const op = {
             ts: new Date().toISOString(),
-            op: "set",
+            op: type,
             id,
-            data: value,
             prev,
         };
+        if (type === "set")
+            op.data = data;
+        if (this.agentId) {
+            op.agent = this.agentId;
+            op.clock = this.clock.tick();
+        }
+        // Try delta encoding for updates (not creates or deletes)
+        if (type === "set" && prev !== null && data !== undefined) {
+            const delta = createDelta(prev, data);
+            if (delta && isDeltaSmaller(delta, prev)) {
+                op.prev = delta;
+                op.encoding = "delta";
+            }
+        }
+        return op;
+    }
+    async _set(id, value) {
+        const prev = this.records.get(id) ?? null;
+        const op = this.makeOp("set", id, value, prev);
         this.records.set(id, value);
         await this.persistOp(op);
     }
     _setSync(id, value) {
         const prev = this.records.get(id) ?? null;
-        const op = {
-            ts: new Date().toISOString(),
-            op: "set",
-            id,
-            data: value,
-            prev,
-        };
+        const op = this.makeOp("set", id, value, prev);
         this.records.set(id, value);
         this.batchOps.push(op);
     }
@@ -248,12 +405,7 @@ export class Store {
         if (prev === undefined) {
             throw new Error(`Record '${id}' not found`);
         }
-        const op = {
-            ts: new Date().toISOString(),
-            op: "delete",
-            id,
-            prev,
-        };
+        const op = this.makeOp("delete", id, undefined, prev);
         this.records.delete(id);
         await this.persistOp(op);
     }
@@ -262,12 +414,7 @@ export class Store {
         if (prev === undefined) {
             throw new Error(`Record '${id}' not found`);
         }
-        const op = {
-            ts: new Date().toISOString(),
-            op: "delete",
-            id,
-            prev,
-        };
+        const op = this.makeOp("delete", id, undefined, prev);
         this.records.delete(id);
         this.batchOps.push(op);
     }
@@ -276,17 +423,15 @@ export class Store {
         this.batchOps = [];
         try {
             fn();
-            // Empty batches are no-ops — no I/O if fn() didn't call set/delete
             if (this.batchOps.length > 0) {
-                await appendOps(join(this.dir, this.activeOpsPath), this.batchOps);
+                await this.backend.appendOps(this.activeOpsPath, this.batchOps);
                 this.ops.push(...this.batchOps);
-                if (this.ops.length >= this.options.checkpointThreshold) {
+                if (this.ops.length >= this.coreOpts.checkpointThreshold) {
                     await this._compact();
                 }
             }
         }
         catch (err) {
-            // Rollback in-memory changes on failure
             for (const op of this.batchOps.reverse()) {
                 try {
                     this.reverseOp(op);
@@ -303,19 +448,37 @@ export class Store {
         }
     }
     async _undo() {
+        if (this.isMultiWriter()) {
+            return this._undoMultiWriter();
+        }
+        // Single-writer: O(1) undo
         if (this.ops.length === 0)
             return false;
         const lastOp = this.ops[this.ops.length - 1];
         this.reverseOp(lastOp);
         this.ops.pop();
-        await truncateLastOp(join(this.dir, this.activeOpsPath));
+        await this.backend.truncateLastOp(this.activeOpsPath);
+        return true;
+    }
+    async _undoMultiWriter() {
+        // Find last op from this agent
+        const myOps = this.ops.filter((op) => op.agent === this.agentId);
+        if (myOps.length === 0)
+            return false;
+        // Truncate our WAL file
+        await this.backend.truncateLastOp(this.activeOpsPath);
+        // Re-derive state from scratch (correct but O(n))
+        await this._refresh();
         return true;
     }
     async _compact() {
-        const snapshotPath = await writeSnapshot(this.dir, this.records, this.version);
-        const opsFilename = `ops-${Date.now()}.jsonl`;
-        const opsPath = `ops/${opsFilename}`;
-        await writeFile(join(this.dir, opsPath), "", "utf-8");
+        if (this.isMultiWriter()) {
+            await this._compactMultiWriter();
+            return;
+        }
+        // Single-writer compaction
+        const snapshotPath = await this.backend.writeSnapshot(this.records, this.version);
+        const opsPath = await this.backend.createOpsFile();
         const updatedManifest = {
             version: this.version,
             currentSnapshot: snapshotPath,
@@ -329,10 +492,45 @@ export class Store {
                 lastCheckpoint: new Date().toISOString(),
             },
         };
-        await writeManifest(this.dir, updatedManifest);
+        await this.backend.writeManifest(updatedManifest);
         this.activeOpsPath = opsPath;
         this.ops = [];
     }
+    async _compactMultiWriter() {
+        let compactLock;
+        try {
+            compactLock = await this.backend.acquireCompactionLock();
+        }
+        catch {
+            // Another agent is compacting — skip
+            return;
+        }
+        try {
+            const snapshotPath = await this.backend.writeSnapshot(this.records, this.version);
+            const opsPath = await this.backend.createAgentOpsFile(this.agentId);
+            const updatedManifest = {
+                version: this.version,
+                currentSnapshot: snapshotPath,
+                activeOps: opsPath,
+                activeAgentOps: { [this.agentId]: opsPath },
+                archiveSegments: this.archiveSegments,
+                stats: {
+                    activeRecords: this.records.size,
+                    archivedRecords: this.archivedRecordCount,
+                    opsCount: 0,
+                    created: this.created,
+                    lastCheckpoint: new Date().toISOString(),
+                },
+            };
+            await this.backend.writeManifest(updatedManifest);
+            this.activeOpsPath = opsPath;
+            this.ops = [];
+            this.manifestVersion = await this.backend.getManifestVersion();
+        }
+        finally {
+            await this.backend.releaseCompactionLock(compactLock);
+        }
+    }
     async _archive(predicate, segment) {
         const toArchive = new Map();
         for (const [id, value] of this.records) {
@@ -342,7 +540,7 @@ export class Store {
         if (toArchive.size === 0)
             return 0;
         const period = segment ?? this.defaultPeriod();
-        const segmentPath = await writeArchiveSegment(this.dir, period, toArchive);
+        const segmentPath = await this.backend.writeArchiveSegment(period, toArchive);
         if (!this.archiveSegments.includes(segmentPath)) {
             this.archiveSegments.push(segmentPath);
         }
@@ -353,13 +551,67 @@ export class Store {
         await this._compact();
         return toArchive.size;
     }
+    async _refresh() {
+        const manifest = await this.backend.readManifest();
+        if (!manifest)
+            throw new Error("Manifest not found during refresh");
+        const { records, version } = await this.backend.loadSnapshot(manifest.currentSnapshot);
+        this.records = records;
+        this.version = version;
+        this.archiveSegments = manifest.archiveSegments;
+        this.archivedRecordCount = manifest.stats.archivedRecords;
+        this.created = manifest.stats.created;
+        // Read all agent ops
+        const allOps = [];
+        if (manifest.activeAgentOps) {
+            for (const opsPath of Object.values(manifest.activeAgentOps)) {
+                const ops = await this.backend.readOps(opsPath);
+                allOps.push(...ops);
+            }
+        }
+        // Legacy single-writer ops
+        if (manifest.activeOps && !manifest.activeAgentOps) {
+            const ops = await this.backend.readOps(manifest.activeOps);
+            allOps.push(...ops);
+        }
+        // Merge-sort
+        allOps.sort((a, b) => {
+            const clockDiff = (a.clock ?? 0) - (b.clock ?? 0);
+            if (clockDiff !== 0)
+                return clockDiff;
+            return (a.agent ?? "").localeCompare(b.agent ?? "");
+        });
+        for (const op of allOps)
+            this.applyOp(op);
+        this.ops = allOps;
+        // Update clock
+        const maxClock = allOps.reduce((max, op) => Math.max(max, op.clock ?? 0), 0);
+        this.clock = new LamportClock(maxClock);
+        // Update our ops path if manifest changed
+        if (manifest.activeAgentOps?.[this.agentId]) {
+            this.activeOpsPath = manifest.activeAgentOps[this.agentId];
+        }
+        else {
+            // Our ops file is not in the manifest (compaction happened)
+            this.activeOpsPath = await this.backend.createAgentOpsFile(this.agentId);
+            const updatedManifest = {
+                ...manifest,
+                activeAgentOps: {
+                    ...(manifest.activeAgentOps ?? {}),
+                    [this.agentId]: this.activeOpsPath,
+                },
+            };
+            await this.backend.writeManifest(updatedManifest);
+        }
+        this.manifestVersion = await this.backend.getManifestVersion();
+    }
     // --- Helpers ---
     ensureOpen() {
         if (!this.opened)
             throw new Error("Store is not open. Call open() first.");
     }
     ensureWritable() {
-        if (this.options.readOnly)
+        if (this.coreOpts.readOnly)
             throw new Error("Store is read-only. Cannot perform mutations.");
     }
     applyOp(op) {
@@ -375,19 +627,27 @@ export class Store {
             // Was a create — reverse by deleting
             this.records.delete(op.id);
         }
+        else if (op.encoding === "delta") {
+            // Delta-encoded: apply the reverse patch to the current record
+            const current = this.records.get(op.id);
+            if (current) {
+                const restored = applyDelta(current, op.prev);
+                this.records.set(op.id, restored);
+            }
+        }
         else if (op.op === "delete") {
-            // Was a delete — reverse by restoring
+            // Was a delete — reverse by restoring full prev
             this.records.set(op.id, op.prev);
         }
         else {
-            // Was an update — reverse by restoring prev
+            // Was an update — reverse by restoring full prev
             this.records.set(op.id, op.prev);
         }
     }
     async persistOp(op) {
-        await appendOp(join(this.dir, this.activeOpsPath), op);
+        await this.backend.appendOps(this.activeOpsPath, [op]);
         this.ops.push(op);
-        if (this.ops.length >= this.options.checkpointThreshold) {
+        if (this.ops.length >= this.coreOpts.checkpointThreshold) {
             await this._compact();
         }
     }

package/dist/types.d.ts

@@ -11,6 +11,10 @@ export interface Operation<T = Record<string, unknown>> {
     prev: T | null;
     /** Encoding format for prev field. Omitted or "full" = full record. "delta" = JSON Patch (future). */
     encoding?: "full" | "delta";
+    /** Agent ID (present in multi-writer mode) */
+    agent?: string;
+    /** Lamport clock value (present in multi-writer mode) */
+    clock?: number;
 }
 export interface Snapshot<T = Record<string, unknown>> {
     version: number;
@@ -21,6 +25,8 @@ export interface Manifest {
     version: number;
     currentSnapshot: string;
     activeOps: string;
+    /** Per-agent ops file paths (multi-writer mode). Keys are agent IDs. */
+    activeAgentOps?: Record<string, string>;
     archiveSegments: string[];
     stats: ManifestStats;
 }
@@ -48,9 +54,46 @@ export interface StoreOptions {
     migrate?: (record: unknown, fromVersion: number) => unknown;
     /** Open in read-only mode: skips directory lock, rejects all mutations. */
     readOnly?: boolean;
+    /** Storage backend implementation (default: FsBackend). */
+    backend?: StorageBackend;
+    /** Agent ID for multi-writer mode. Enables per-agent WAL streams and LWW conflict resolution. */
+    agentId?: string;
 }
 export interface StoreStats {
     activeRecords: number;
     opsCount: number;
     archiveSegments: number;
 }
+/** Opaque lock handle returned by StorageBackend locking methods. */
+export interface LockHandle {
+}
+/** Pluggable storage backend for opslog. */
+export interface StorageBackend {
+    /** Initialize the backend (create directories, etc.). Called once during store.open(). */
+    initialize(dir: string, opts: {
+        readOnly: boolean;
+    }): Promise<void>;
+    /** Shut down the backend. Called during store.close(). */
+    shutdown(): Promise<void>;
+    readManifest(): Promise<Manifest | null>;
+    writeManifest(manifest: Manifest): Promise<void>;
+    writeSnapshot(records: Map<string, unknown>, version: number): Promise<string>;
+    loadSnapshot(relativePath: string): Promise<{
+        records: Map<string, unknown>;
+        version: number;
+    }>;
+    appendOps(relativePath: string, ops: Operation[]): Promise<void>;
+    readOps(relativePath: string): Promise<Operation[]>;
+    truncateLastOp(relativePath: string): Promise<boolean>;
+    createOpsFile(): Promise<string>;
+    writeArchiveSegment(period: string, records: Map<string, unknown>): Promise<string>;
+    loadArchiveSegment(relativePath: string): Promise<Map<string, unknown>>;
+    listArchiveSegments(): Promise<string[]>;
+    acquireLock(): Promise<LockHandle>;
+    releaseLock(handle: LockHandle): Promise<void>;
+    createAgentOpsFile(agentId: string): Promise<string>;
+    listOpsFiles(): Promise<string[]>;
+    acquireCompactionLock(): Promise<LockHandle>;
+    releaseCompactionLock(handle: LockHandle): Promise<void>;
+    getManifestVersion(): Promise<string | null>;
+}

package/dist/validate.js

@@ -24,6 +24,14 @@ export function validateOp(raw) {
     if ("encoding" in obj && obj.encoding !== "full" && obj.encoding !== "delta") {
         throw new Error(`Invalid operation: encoding must be "full" or "delta", got "${obj.encoding}"`);
     }
+    if ("agent" in obj && (typeof obj.agent !== "string" || obj.agent.length === 0)) {
+        throw new Error("Invalid operation: agent must be a non-empty string");
+    }
+    if ("clock" in obj) {
+        if (typeof obj.clock !== "number" || !Number.isFinite(obj.clock) || !Number.isInteger(obj.clock) || obj.clock < 0) {
+            throw new Error("Invalid operation: clock must be a non-negative integer");
+        }
+    }
     return raw;
 }
 export function validateManifest(raw) {
@@ -58,6 +66,16 @@ export function validateManifest(raw) {
         throw new Error("Invalid manifest: stats.created must be a non-empty string");
     if (typeof stats.lastCheckpoint !== "string" || stats.lastCheckpoint.length === 0)
         throw new Error("Invalid manifest: stats.lastCheckpoint must be a non-empty string");
+    if ("activeAgentOps" in obj && obj.activeAgentOps !== undefined) {
+        if (typeof obj.activeAgentOps !== "object" || obj.activeAgentOps === null || Array.isArray(obj.activeAgentOps)) {
+            throw new Error("Invalid manifest: activeAgentOps must be an object");
+        }
+        for (const [, val] of Object.entries(obj.activeAgentOps)) {
+            if (typeof val !== "string" || val.length === 0) {
+                throw new Error("Invalid manifest: activeAgentOps values must be non-empty strings");
+            }
+        }
+    }
     return raw;
 }
 export function validateSnapshot(raw) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@backloghq/opslog",
-  "version": "0.2.0",
+  "version": "0.4.0",
   "description": "Embedded event-sourced document store. Append-only operation log with immutable snapshots, zero native dependencies.",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",