@backloghq/opslog 0.4.0 → 0.5.0

package/README.md

@@ -134,11 +134,41 @@ await store.open(dir, {
   version: 1,                     // Schema version
   migrate: (record, fromVersion) => record, // Migration function
   readOnly: false,                // Open in read-only mode (default: false)
+  writeMode: "immediate",         // "immediate" (default) or "group" (buffered, ~12x faster)
+  groupCommitSize: 50,            // Group: flush after N ops (default: 50)
+  groupCommitMs: 100,             // Group: flush after N ms (default: 100)
   agentId: "agent-A",             // Enable multi-writer mode (optional)
   backend: new FsBackend(),       // Custom storage backend (optional, default: FsBackend)
 });
 ```
 
+## Group Commit
+
+Buffer writes in memory and flush as a single disk write. ~12x faster for sustained writes.
+
+```typescript
+const store = new Store();
+await store.open("./data", {
+  writeMode: "group",     // Buffer ops, flush periodically
+  groupCommitSize: 50,    // Flush when buffer has 50 ops
+  groupCommitMs: 100,     // Or after 100ms idle
+});
+
+// Writes are buffered — no fsync per op
+await store.set("a", valueA);
+await store.set("b", valueB);  // Both flushed together
+
+// Explicit flush if needed
+await store.flush();
+
+// close() always flushes before shutting down
+await store.close();
+```
+
+**Safety:** Forced to `"immediate"` when `agentId` is set (multi-writer mode). Other agents can't see buffered ops, so group commit is single-writer only.
+
+**Tradeoff:** A crash can lose ops that haven't been flushed yet (up to `groupCommitMs` milliseconds of data). Use `"immediate"` (default) when every write must be durable.
+
 ## Multi-Writer Mode
 
 Multiple agents can write to the same store concurrently. Each agent gets its own WAL file — no write contention.

package/dist/store.d.ts

@@ -17,6 +17,11 @@ export declare class Store<T = Record<string, unknown>> {
     private backend;
     private agentId?;
     private clock;
+    private groupCommit;
+    private groupBuffer;
+    private groupSize;
+    private groupMs;
+    private groupTimer;
     private manifestVersion;
     /**
      * Serialize all state-mutating operations through a promise chain.
@@ -47,18 +52,19 @@ export declare class Store<T = Record<string, unknown>> {
     loadArchive(segment: string): Promise<Map<string, T>>;
     stats(): StoreStats;
     /**
-     * Reload state from the backend (multi-writer mode).
-     * Re-reads the manifest, snapshot, and all agent WAL files.
-     * Use this to pick up writes from other agents.
+     * Reload state from the backend.
+     * In multi-writer mode: re-reads manifest, snapshot, and all agent WAL files.
+     * In single-writer/readOnly mode: re-reads the active ops file for new entries.
+     * Use this to pick up writes from other agents or processes.
      */
     refresh(): Promise<void>;
     private watchTimer;
     private watchCallback;
     /**
-     * Tail the WAL for new operations. Re-reads the active ops file
-     * and replays any new operations since the last known count.
+     * Tail the WAL for new operations.
+     * In single-writer/readOnly: re-reads the active ops file for new entries.
+     * In multi-writer: re-reads ALL agent WAL files from the manifest.
      * Returns the newly applied operations.
-     * Works in any mode (single-writer readOnly, multi-writer, etc).
      */
     tail(): Promise<Operation<T>[]>;
     /**
@@ -87,5 +93,8 @@ export declare class Store<T = Record<string, unknown>> {
     private applyOp;
     private reverseOp;
     private persistOp;
+    /** Flush buffered group commit ops to disk in a single write. */
+    flush(): Promise<void>;
+    private flushGroupBuffer;
     private defaultPeriod;
 }

package/dist/store.js

@@ -27,6 +27,12 @@ export class Store {
     // Multi-writer state
     agentId;
     clock = null;
+    // Group commit state
+    groupCommit = false;
+    groupBuffer = [];
+    groupSize = 50;
+    groupMs = 100;
+    groupTimer = null;
     manifestVersion = null;
     /**
      * Serialize all state-mutating operations through a promise chain.
@@ -46,12 +52,25 @@ export class Store {
     async open(dir, options) {
         this.dir = dir;
         if (options) {
-            const { backend, agentId, ...rest } = options;
+            const { backend, agentId, writeMode, groupCommitSize, groupCommitMs, ...rest } = options;
             this.coreOpts = { ...this.coreOpts, ...rest };
             if (backend)
                 this.backend = backend;
             if (agentId)
                 this.agentId = agentId;
+            // Group commit: enabled when writeMode is "group" AND not multi-writer
+            if (writeMode === "group") {
+                if (agentId) {
+                    console.error("opslog: writeMode 'group' is not compatible with multi-writer (agentId). Using 'immediate'.");
+                }
+                else {
+                    this.groupCommit = true;
+                    if (groupCommitSize)
+                        this.groupSize = groupCommitSize;
+                    if (groupCommitMs)
+                        this.groupMs = groupCommitMs;
+                }
+            }
         }
         this.backend ??= new FsBackend();
         await this.backend.initialize(dir, { readOnly: this.coreOpts.readOnly });
@@ -183,6 +202,10 @@ export class Store {
     async close() {
         this.ensureOpen();
         this.unwatch();
+        // Flush any buffered group commit ops before checkpoint
+        if (this.groupCommit && this.groupBuffer.length > 0) {
+            await this.serialize(() => this.flushGroupBuffer());
+        }
         if (!this.coreOpts.readOnly &&
             this.coreOpts.checkpointOnClose &&
             this.ops.length > 0) {
@@ -299,30 +322,41 @@ export class Store {
         };
     }
     /**
-     * Reload state from the backend (multi-writer mode).
-     * Re-reads the manifest, snapshot, and all agent WAL files.
-     * Use this to pick up writes from other agents.
+     * Reload state from the backend.
+     * In multi-writer mode: re-reads manifest, snapshot, and all agent WAL files.
+     * In single-writer/readOnly mode: re-reads the active ops file for new entries.
+     * Use this to pick up writes from other agents or processes.
      */
     async refresh() {
         this.ensureOpen();
-        if (!this.isMultiWriter()) {
-            throw new Error("refresh() is only available in multi-writer mode");
+        if (this.isMultiWriter()) {
+            return this.serialize(() => this._refresh());
         }
-        return this.serialize(() => this._refresh());
+        // Single-writer / readOnly: just tail the active ops file
+        await this.tail();
     }
     // --- WAL tailing ---
     watchTimer = null;
     watchCallback = null;
     /**
-     * Tail the WAL for new operations. Re-reads the active ops file
-     * and replays any new operations since the last known count.
+     * Tail the WAL for new operations.
+     * In single-writer/readOnly: re-reads the active ops file for new entries.
+     * In multi-writer: re-reads ALL agent WAL files from the manifest.
      * Returns the newly applied operations.
-     * Works in any mode (single-writer readOnly, multi-writer, etc).
      */
     async tail() {
         this.ensureOpen();
         const prevCount = this.ops.length;
-        // Re-read the ops file for new entries
+        if (this.isMultiWriter()) {
+            // Multi-writer: full refresh to pick up all agents' writes
+            await this.serialize(() => this._refresh());
+            // Return the difference
+            if (this.ops.length > prevCount) {
+                return this.ops.slice(prevCount);
+            }
+            return [];
+        }
+        // Single-writer / readOnly: just re-read our ops file
         const allOps = (await this.backend.readOps(this.activeOpsPath));
         if (allOps.length <= prevCount)
             return [];
@@ -472,6 +506,9 @@ export class Store {
         return true;
     }
     async _compact() {
+        // Flush group buffer before checkpoint
+        if (this.groupCommit)
+            await this.flushGroupBuffer();
         if (this.isMultiWriter()) {
             await this._compactMultiWriter();
             return;
@@ -645,12 +682,43 @@ export class Store {
         }
     }
     async persistOp(op) {
-        await this.backend.appendOps(this.activeOpsPath, [op]);
         this.ops.push(op);
+        if (this.groupCommit) {
+            // Buffer the op, flush when buffer is full or timer fires
+            this.groupBuffer.push(op);
+            if (this.groupBuffer.length >= this.groupSize) {
+                await this.flushGroupBuffer();
+            }
+            else if (!this.groupTimer) {
+                this.groupTimer = setTimeout(() => {
+                    this.serialize(() => this.flushGroupBuffer()).catch(() => { });
+                }, this.groupMs);
+            }
+        }
+        else {
+            // Immediate: write to disk now
+            await this.backend.appendOps(this.activeOpsPath, [op]);
+        }
         if (this.ops.length >= this.coreOpts.checkpointThreshold) {
             await this._compact();
         }
     }
+    /** Flush buffered group commit ops to disk in a single write. */
+    async flush() {
+        if (!this.groupCommit || this.groupBuffer.length === 0)
+            return;
+        return this.serialize(() => this.flushGroupBuffer());
+    }
+    async flushGroupBuffer() {
+        if (this.groupBuffer.length === 0)
+            return;
+        if (this.groupTimer) {
+            clearTimeout(this.groupTimer);
+            this.groupTimer = null;
+        }
+        await this.backend.appendOps(this.activeOpsPath, this.groupBuffer);
+        this.groupBuffer = [];
+    }
     defaultPeriod() {
         const now = new Date();
         const q = Math.ceil((now.getMonth() + 1) / 3);

package/dist/types.d.ts

@@ -58,6 +58,12 @@ export interface StoreOptions {
     backend?: StorageBackend;
     /** Agent ID for multi-writer mode. Enables per-agent WAL streams and LWW conflict resolution. */
     agentId?: string;
+    /** Write mode: "immediate" flushes every op (default, safe for multi-writer). "group" buffers ops and flushes periodically (~50x faster writes). Forced to "immediate" when agentId is set. */
+    writeMode?: "immediate" | "group";
+    /** Group commit: max ops to buffer before flush (default: 50). Only used when writeMode is "group". */
+    groupCommitSize?: number;
+    /** Group commit: max milliseconds before flush (default: 100). Only used when writeMode is "group". */
+    groupCommitMs?: number;
 }
 export interface StoreStats {
     activeRecords: number;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@backloghq/opslog",
-  "version": "0.4.0",
+  "version": "0.5.0",
   "description": "Embedded event-sourced document store. Append-only operation log with immutable snapshots, zero native dependencies.",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",