@backloghq/opslog 0.4.1 → 0.5.1

package/README.md

@@ -134,11 +134,66 @@ await store.open(dir, {
   version: 1,                     // Schema version
   migrate: (record, fromVersion) => record, // Migration function
   readOnly: false,                // Open in read-only mode (default: false)
+  writeMode: "immediate",         // "immediate" (default), "group" (~12x faster), or "async" (~50x faster, lossy on crash)
+  groupCommitSize: 50,            // Group: flush after N ops (default: 50)
+  groupCommitMs: 100,             // Group: flush after N ms (default: 100)
   agentId: "agent-A",             // Enable multi-writer mode (optional)
   backend: new FsBackend(),       // Custom storage backend (optional, default: FsBackend)
 });
 ```
 
+## Group Commit
+
+Buffer writes in memory and flush as a single disk write. ~12x faster for sustained writes.
+
+```typescript
+const store = new Store();
+await store.open("./data", {
+  writeMode: "group",     // Buffer ops, flush periodically
+  groupCommitSize: 50,    // Flush when buffer has 50 ops
+  groupCommitMs: 100,     // Or after 100ms idle
+});
+
+// Writes are buffered — no fsync per op
+await store.set("a", valueA);
+await store.set("b", valueB);  // Both flushed together
+
+// Explicit flush if needed
+await store.flush();
+
+// close() always flushes before shutting down
+await store.close();
+```
+
+### Async Mode
+
+For maximum write throughput, use `writeMode: "async"`. Writes resolve immediately after buffering in memory — no disk I/O on the hot path. ~50x faster than immediate mode.
+
+```typescript
+const store = new Store();
+await store.open("./data", {
+  writeMode: "async",
+  groupCommitSize: 50,
+  groupCommitMs: 100,
+});
+
+await store.set("a", valueA);  // Returns instantly — buffered in memory
+await store.set("b", valueB);  // Same
+
+// Ensure durability before exit
+await store.sync();
+await store.close();
+```
+
+**Crash semantics:** Data buffered since the last flush is **lost** on unclean shutdown. Call `sync()` before process exit for durability. `close()` always flushes.
+
+**Safety:** Forced to `"immediate"` when `agentId` is set (multi-writer mode). Other agents can't see buffered ops, so group/async commit is single-writer only.
+
+**When to use which mode:**
+- `"immediate"` (default) — every write is durable. Use when data loss is unacceptable.
+- `"group"` — writes are batched but caller still waits for flush. ~12x faster. Crash loses up to `groupCommitMs` ms of data.
+- `"async"` — writes return instantly. ~50x faster. Crash loses all unflushed data. Best for high-throughput, latency-sensitive, crash-tolerant workloads.
+
 ## Multi-Writer Mode
 
 Multiple agents can write to the same store concurrently. Each agent gets its own WAL file — no write contention.

package/dist/store.d.ts

@@ -17,6 +17,12 @@ export declare class Store<T = Record<string, unknown>> {
     private backend;
     private agentId?;
     private clock;
+    private groupCommit;
+    private asyncMode;
+    private groupBuffer;
+    private groupSize;
+    private groupMs;
+    private groupTimer;
     private manifestVersion;
     /**
      * Serialize all state-mutating operations through a promise chain.
@@ -88,5 +94,18 @@ export declare class Store<T = Record<string, unknown>> {
     private applyOp;
     private reverseOp;
     private persistOp;
+    /**
+     * Flush buffered ops to disk in a single write.
+     * In group/async mode: drains the in-memory buffer to disk.
+     * Safe to call at any time. No-op if nothing is buffered.
+     */
+    flush(): Promise<void>;
+    /**
+     * Ensure all buffered operations are durably persisted to disk.
+     * Alias for flush(). Use before process exit when using async write mode
+     * to prevent data loss.
+     */
+    sync(): Promise<void>;
+    private flushGroupBuffer;
     private defaultPeriod;
 }

package/dist/store.js

@@ -27,6 +27,13 @@ export class Store {
     // Multi-writer state
     agentId;
     clock = null;
+    // Group commit state
+    groupCommit = false;
+    asyncMode = false;
+    groupBuffer = [];
+    groupSize = 50;
+    groupMs = 100;
+    groupTimer = null;
     manifestVersion = null;
     /**
      * Serialize all state-mutating operations through a promise chain.
@@ -46,12 +53,26 @@ export class Store {
     async open(dir, options) {
         this.dir = dir;
         if (options) {
-            const { backend, agentId, ...rest } = options;
+            const { backend, agentId, writeMode, groupCommitSize, groupCommitMs, ...rest } = options;
             this.coreOpts = { ...this.coreOpts, ...rest };
             if (backend)
                 this.backend = backend;
             if (agentId)
                 this.agentId = agentId;
+            // Group/async commit: enabled when writeMode is "group"/"async" AND not multi-writer
+            if (writeMode === "group" || writeMode === "async") {
+                if (agentId) {
+                    console.error(`opslog: writeMode '${writeMode}' is not compatible with multi-writer (agentId). Using 'immediate'.`);
+                }
+                else {
+                    this.groupCommit = true;
+                    this.asyncMode = writeMode === "async";
+                    if (groupCommitSize)
+                        this.groupSize = groupCommitSize;
+                    if (groupCommitMs)
+                        this.groupMs = groupCommitMs;
+                }
+            }
         }
         this.backend ??= new FsBackend();
         await this.backend.initialize(dir, { readOnly: this.coreOpts.readOnly });
@@ -183,6 +204,10 @@ export class Store {
     async close() {
         this.ensureOpen();
         this.unwatch();
+        // Flush any buffered group commit ops before checkpoint
+        if (this.groupCommit && this.groupBuffer.length > 0) {
+            await this.serialize(() => this.flushGroupBuffer());
+        }
         if (!this.coreOpts.readOnly &&
             this.coreOpts.checkpointOnClose &&
             this.ops.length > 0) {
@@ -483,6 +508,9 @@ export class Store {
         return true;
     }
     async _compact() {
+        // Flush group buffer before checkpoint
+        if (this.groupCommit)
+            await this.flushGroupBuffer();
         if (this.isMultiWriter()) {
             await this._compactMultiWriter();
             return;
@@ -656,12 +684,64 @@ export class Store {
         }
     }
     async persistOp(op) {
-        await this.backend.appendOps(this.activeOpsPath, [op]);
         this.ops.push(op);
+        if (this.groupCommit) {
+            // Buffer the op, flush when buffer is full or timer fires
+            this.groupBuffer.push(op);
+            if (this.groupBuffer.length >= this.groupSize) {
+                if (this.asyncMode) {
+                    // Async: trigger flush in background, don't await
+                    this.serialize(() => this.flushGroupBuffer()).catch(() => { });
+                }
+                else {
+                    await this.flushGroupBuffer();
+                }
+            }
+            else if (!this.groupTimer) {
+                this.groupTimer = setTimeout(() => {
+                    this.serialize(() => this.flushGroupBuffer()).catch(() => { });
+                }, this.groupMs);
+            }
+            // Async mode: return immediately without waiting for disk I/O
+            if (this.asyncMode)
+                return;
+        }
+        else {
+            // Immediate: write to disk now
+            await this.backend.appendOps(this.activeOpsPath, [op]);
+        }
         if (this.ops.length >= this.coreOpts.checkpointThreshold) {
             await this._compact();
         }
     }
+    /**
+     * Flush buffered ops to disk in a single write.
+     * In group/async mode: drains the in-memory buffer to disk.
+     * Safe to call at any time. No-op if nothing is buffered.
+     */
+    async flush() {
+        if (!this.groupCommit || this.groupBuffer.length === 0)
+            return;
+        return this.serialize(() => this.flushGroupBuffer());
+    }
+    /**
+     * Ensure all buffered operations are durably persisted to disk.
+     * Alias for flush(). Use before process exit when using async write mode
+     * to prevent data loss.
+     */
+    async sync() {
+        return this.flush();
+    }
+    async flushGroupBuffer() {
+        if (this.groupBuffer.length === 0)
+            return;
+        if (this.groupTimer) {
+            clearTimeout(this.groupTimer);
+            this.groupTimer = null;
+        }
+        await this.backend.appendOps(this.activeOpsPath, this.groupBuffer);
+        this.groupBuffer = [];
+    }
     defaultPeriod() {
         const now = new Date();
         const q = Math.ceil((now.getMonth() + 1) / 3);

package/dist/types.d.ts

@@ -58,6 +58,12 @@ export interface StoreOptions {
     backend?: StorageBackend;
     /** Agent ID for multi-writer mode. Enables per-agent WAL streams and LWW conflict resolution. */
     agentId?: string;
+    /** Write mode: "immediate" flushes every op (default, safe for multi-writer). "group" buffers ops and flushes periodically (~12x faster writes). "async" buffers ops and resolves immediately without waiting for flush (~50x faster, data lost on crash). Forced to "immediate" when agentId is set. */
+    writeMode?: "immediate" | "group" | "async";
+    /** Group commit: max ops to buffer before flush (default: 50). Only used when writeMode is "group". */
+    groupCommitSize?: number;
+    /** Group commit: max milliseconds before flush (default: 100). Only used when writeMode is "group". */
+    groupCommitMs?: number;
 }
 export interface StoreStats {
     activeRecords: number;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@backloghq/opslog",
-  "version": "0.4.1",
+  "version": "0.5.1",
   "description": "Embedded event-sourced document store. Append-only operation log with immutable snapshots, zero native dependencies.",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",