@backloghq/opslog 0.6.0 → 0.7.1

package/README.md

@@ -56,7 +56,7 @@ State survives restarts — reopen the same directory and everything is there.
 data/
   manifest.json                    # Points to current snapshot + ops file(s)
   snapshots/
-    snap-<timestamp>.json          # Immutable full-state capture
+    snap-<timestamp>.jsonl          # Immutable full-state capture (JSONL: header + one line per record)
   ops/
     ops-<timestamp>.jsonl          # Append-only operation log (single-writer)
     agent-<id>-<timestamp>.jsonl   # Per-agent operation log (multi-writer)
@@ -93,6 +93,21 @@ store.all()                       // All records
 store.entries()                   // All [id, record] pairs
 store.filter(predicate)           // Records matching predicate
 store.count(predicate?)           // Count (all or matching)
+store.getManifest()               // Read-only ManifestInfo (snapshot/WAL paths, stats)
+```
+
+### Streaming (for external consumers)
+
+```typescript
+// Stream snapshot records without loading all into memory
+for await (const [id, record] of store.streamSnapshot()) {
+  process(id, record);
+}
+
+// Read WAL operations (optionally since a timestamp)
+for await (const op of store.getWalOps(sinceTimestamp?)) {
+  // op: { ts, op: "set"|"delete", id, data?, prev }
+}
 ```
 
 ### Batch
@@ -134,6 +149,7 @@ await store.open(dir, {
   version: 1,                     // Schema version
   migrate: (record, fromVersion) => record, // Migration function
   readOnly: false,                // Open in read-only mode (default: false)
+  skipLoad: false,                // Skip loading snapshot/WAL into memory (default: false)
   writeMode: "immediate",         // "immediate" (default), "group" (~12x faster), or "async" (~50x faster, lossy on crash)
   groupCommitSize: 50,            // Group: flush after N ops (default: 50)
   groupCommitMs: 100,             // Group: flush after N ms (default: 100)

package/dist/index.d.ts

@@ -3,6 +3,7 @@ export { FsBackend } from "./backend.js";
 export { LamportClock } from "./clock.js";
 export { createDelta, applyDelta, isDeltaSmaller } from "./delta.js";
 export type { DeltaPatch } from "./delta.js";
+export { streamSnapshotFile } from "./snapshot.js";
 export { acquireLock, releaseLock } from "./lock.js";
 export { validateOp, validateManifest, validateSnapshot, validateArchiveSegment, } from "./validate.js";
-export type { Operation, Snapshot, Manifest, ManifestStats, ArchiveSegment, StoreOptions, StoreStats, StorageBackend, LockHandle, } from "./types.js";
+export type { Operation, Snapshot, Manifest, ManifestInfo, ManifestStats, ArchiveSegment, StoreOptions, StoreStats, StorageBackend, LockHandle, } from "./types.js";

package/dist/index.js

@@ -2,5 +2,6 @@ export { Store } from "./store.js";
 export { FsBackend } from "./backend.js";
 export { LamportClock } from "./clock.js";
 export { createDelta, applyDelta, isDeltaSmaller } from "./delta.js";
+export { streamSnapshotFile } from "./snapshot.js";
 export { acquireLock, releaseLock } from "./lock.js";
 export { validateOp, validateManifest, validateSnapshot, validateArchiveSegment, } from "./validate.js";

package/dist/snapshot.d.ts

@@ -3,3 +3,11 @@ export declare function loadSnapshot<T>(dir: string, relativePath: string): Prom
     records: Map<string, T>;
     version: number;
 }>;
+/**
+ * Stream snapshot records line by line using readline.
+ * True streaming — only one record in memory at a time.
+ * Supports both JSONL and legacy JSON formats.
+ */
+export declare function streamSnapshotFile<T>(dir: string, relativePath: string): AsyncGenerator<[string, T], {
+    version: number;
+}>;

package/dist/snapshot.js

@@ -1,24 +1,89 @@
-import { readFile, rename, writeFile } from "node:fs/promises";
+import { createReadStream, createWriteStream } from "node:fs";
+import { readFile, rename } from "node:fs/promises";
+import { createInterface } from "node:readline";
 import { join } from "node:path";
 import { validateSnapshot } from "./validate.js";
 export async function writeSnapshot(dir, records, version) {
     const timestamp = new Date().toISOString();
-    const filename = `snap-${Date.now()}.json`;
+    const filename = `snap-${Date.now()}.jsonl`;
     const path = join(dir, "snapshots", filename);
-    const snapshot = {
-        version,
-        timestamp,
-        records: Object.fromEntries(records),
-    };
+    // Stream line-by-line to avoid V8 string limit at large record counts
     const tmpPath = path + ".tmp";
-    await writeFile(tmpPath, JSON.stringify(snapshot, null, 2), "utf-8");
+    await new Promise((resolve, reject) => {
+        const ws = createWriteStream(tmpPath, "utf-8");
+        ws.on("error", reject);
+        ws.write(JSON.stringify({ version, timestamp }) + "\n");
+        for (const [id, data] of records) {
+            ws.write(JSON.stringify({ id, data }) + "\n");
+        }
+        ws.end(() => resolve());
+    });
     await rename(tmpPath, path);
     return `snapshots/${filename}`;
 }
 export async function loadSnapshot(dir, relativePath) {
     const path = join(dir, relativePath);
     const content = await readFile(path, "utf-8");
-    const snapshot = validateSnapshot(JSON.parse(content));
-    const records = new Map(Object.entries(snapshot.records));
-    return { records, version: snapshot.version };
+    // Detect format: JSONL (first line is header without "records" key) vs legacy JSON
+    const firstNewline = content.indexOf("\n");
+    const firstLine = firstNewline === -1 ? content : content.slice(0, firstNewline);
+    const parsed = JSON.parse(firstLine);
+    if ("records" in parsed) {
+        // Legacy monolithic JSON format
+        const snapshot = validateSnapshot(parsed);
+        const records = new Map(Object.entries(snapshot.records));
+        return { records, version: snapshot.version };
+    }
+    // JSONL format: first line is header, remaining lines are records
+    const header = parsed;
+    const records = new Map();
+    const lines = content.split("\n");
+    for (let i = 1; i < lines.length; i++) {
+        const line = lines[i].trim();
+        if (!line)
+            continue;
+        const entry = JSON.parse(line);
+        records.set(entry.id, entry.data);
+    }
+    return { records, version: header.version };
+}
+/**
+ * Stream snapshot records line by line using readline.
+ * True streaming — only one record in memory at a time.
+ * Supports both JSONL and legacy JSON formats.
+ */
+export async function* streamSnapshotFile(dir, relativePath) {
+    const path = join(dir, relativePath);
+    // Peek at first line to detect format
+    const content = await readFile(path, "utf-8");
+    const firstNewline = content.indexOf("\n");
+    const firstLine = firstNewline === -1 ? content : content.slice(0, firstNewline);
+    const parsed = JSON.parse(firstLine);
+    if ("records" in parsed) {
+        // Legacy JSON: must load all, then yield
+        const snapshot = validateSnapshot(parsed);
+        for (const [id, record] of Object.entries(snapshot.records)) {
+            yield [id, record];
+        }
+        return { version: snapshot.version };
+    }
+    // JSONL: stream line by line via readline
+    const header = parsed;
+    const rl = createInterface({
+        input: createReadStream(path, "utf-8"),
+        crlfDelay: Infinity,
+    });
+    let isFirst = true;
+    for await (const line of rl) {
+        if (isFirst) {
+            isFirst = false;
+            continue;
+        } // skip header
+        const trimmed = line.trim();
+        if (!trimmed)
+            continue;
+        const entry = JSON.parse(trimmed);
+        yield [entry.id, entry.data];
+    }
+    return { version: header.version };
 }

package/dist/store.d.ts

@@ -1,4 +1,4 @@
-import type { Operation, StoreOptions, StoreStats } from "./types.js";
+import type { ManifestInfo, Operation, StoreOptions, StoreStats } from "./types.js";
 export declare class Store<T = Record<string, unknown>> {
     private dir;
     private records;
@@ -24,6 +24,7 @@ export declare class Store<T = Record<string, unknown>> {
     private groupMs;
     private groupTimer;
     private manifestVersion;
+    private manifest;
     /**
      * Serialize all state-mutating operations through a promise chain.
      * Prevents interleaving of async mutations. Reads remain synchronous and lock-free.
@@ -42,6 +43,20 @@ export declare class Store<T = Record<string, unknown>> {
     all(): T[];
     entries(): [string, T][];
     filter(predicate: (value: T, id: string) => boolean): T[];
+    /** Get read-only manifest info. Returns null if store is not open or no manifest exists. */
+    getManifest(): ManifestInfo | null;
+    /**
+     * Stream snapshot records without loading all into memory.
+     * Yields [id, record] pairs from the current snapshot.
+     * Requires store to be open (manifest must be read).
+     */
+    streamSnapshot(): AsyncGenerator<[string, T]>;
+    /**
+     * Read WAL operations, optionally filtered to those after a given timestamp.
+     * Returns ops in chronological order (by Lamport clock for multi-writer).
+     * Does not modify the in-memory Map — consumer handles replay.
+     */
+    getWalOps(sinceTimestamp?: string): AsyncGenerator<Operation<T>>;
     count(predicate?: (value: T, id: string) => boolean): number;
     batch(fn: () => void): Promise<void>;
     undo(): Promise<boolean>;

package/dist/store.js

@@ -2,6 +2,7 @@ import { createDefaultManifest } from "./manifest.js";
 import { FsBackend } from "./backend.js";
 import { LamportClock } from "./clock.js";
 import { createDelta, applyDelta, isDeltaSmaller } from "./delta.js";
+import { streamSnapshotFile } from "./snapshot.js";
 export class Store {
     dir = "";
     records = new Map();
@@ -17,6 +18,7 @@ export class Store {
         version: 1,
         migrate: (r) => r,
         readOnly: false,
+        skipLoad: false,
     };
     archivedRecordCount = 0;
     batching = false;
@@ -35,6 +37,7 @@ export class Store {
     groupMs = 100;
     groupTimer = null;
     manifestVersion = null;
+    manifest = null;
     /**
      * Serialize all state-mutating operations through a promise chain.
      * Prevents interleaving of async mutations. Reads remain synchronous and lock-free.
@@ -53,7 +56,12 @@ export class Store {
     async open(dir, options) {
         this.dir = dir;
         if (options) {
-            const { backend, agentId, writeMode, groupCommitSize, groupCommitMs, ...rest } = options;
+            const { backend, agentId, writeMode, groupCommitSize, groupCommitMs, skipLoad, ...rest } = options;
+            if (skipLoad) {
+                this.coreOpts.skipLoad = true;
+                // Never checkpoint when skipLoad — Map is empty, would overwrite real data
+                this.coreOpts.checkpointOnClose = false;
+            }
             this.coreOpts = { ...this.coreOpts, ...rest };
             if (backend)
                 this.backend = backend;
@@ -107,6 +115,7 @@ export class Store {
             newManifest.activeAgentOps = { [this.agentId]: opsPath };
         }
         await this.backend.writeManifest(newManifest);
+        this.manifest = newManifest;
         this.version = this.coreOpts.version;
         this.activeOpsPath = opsPath;
         this.created = newManifest.stats.created;
@@ -116,6 +125,34 @@ export class Store {
         }
     }
     async loadExistingStore(manifest) {
+        this.created = manifest.stats.created;
+        this.archiveSegments = manifest.archiveSegments;
+        this.archivedRecordCount = manifest.stats.archivedRecords;
+        this.manifest = manifest;
+        // skipLoad: acquire ops path for writes but don't load records or replay WAL
+        if (this.coreOpts.skipLoad) {
+            this.version = this.coreOpts.version;
+            if (this.isMultiWriter()) {
+                const maxClock = 0;
+                this.clock = new LamportClock(maxClock);
+                if (manifest.activeAgentOps?.[this.agentId]) {
+                    this.activeOpsPath = manifest.activeAgentOps[this.agentId];
+                }
+                else {
+                    this.activeOpsPath = await this.backend.createAgentOpsFile(this.agentId);
+                    const updatedManifest = {
+                        ...manifest,
+                        activeAgentOps: { ...(manifest.activeAgentOps ?? {}), [this.agentId]: this.activeOpsPath },
+                    };
+                    await this.backend.writeManifest(updatedManifest);
+                    this.manifest = updatedManifest;
+                }
+            }
+            else {
+                this.activeOpsPath = manifest.activeOps;
+            }
+            return;
+        }
         // Load snapshot
         let snapshotData;
         try {
@@ -133,9 +170,6 @@ export class Store {
         const { records, version: storedVersion } = snapshotData;
         this.records = records;
         this.version = storedVersion;
-        this.created = manifest.stats.created;
-        this.archiveSegments = manifest.archiveSegments;
-        this.archivedRecordCount = manifest.stats.archivedRecords;
         // Migrate if needed
         if (storedVersion < this.coreOpts.version) {
             for (const [id, record] of this.records) {
@@ -263,6 +297,71 @@ export class Store {
         }
         return results;
     }
+    /** Get read-only manifest info. Returns null if store is not open or no manifest exists. */
+    getManifest() {
+        const m = this.manifest;
+        if (!m)
+            return null;
+        return {
+            currentSnapshot: m.currentSnapshot,
+            activeOps: m.activeOps,
+            archiveSegments: m.archiveSegments,
+            stats: m.stats,
+        };
+    }
+    /**
+     * Stream snapshot records without loading all into memory.
+     * Yields [id, record] pairs from the current snapshot.
+     * Requires store to be open (manifest must be read).
+     */
+    async *streamSnapshot() {
+        this.ensureOpen();
+        const manifest = this.manifest;
+        if (!manifest)
+            return;
+        // True streaming via readline — one record in memory at a time (JSONL format).
+        // Falls back to parse-then-yield for legacy JSON snapshots.
+        yield* streamSnapshotFile(this.dir, manifest.currentSnapshot);
+    }
+    /**
+     * Read WAL operations, optionally filtered to those after a given timestamp.
+     * Returns ops in chronological order (by Lamport clock for multi-writer).
+     * Does not modify the in-memory Map — consumer handles replay.
+     */
+    async *getWalOps(sinceTimestamp) {
+        this.ensureOpen();
+        const manifest = this.manifest;
+        if (!manifest)
+            return;
+        if (manifest.activeAgentOps) {
+            // Multi-writer: must load all ops for merge-sort by (clock, agent)
+            const allOps = [];
+            for (const opsPath of Object.values(manifest.activeAgentOps)) {
+                const ops = (await this.backend.readOps(opsPath));
+                allOps.push(...ops);
+            }
+            allOps.sort((a, b) => {
+                const clockDiff = (a.clock ?? 0) - (b.clock ?? 0);
+                if (clockDiff !== 0)
+                    return clockDiff;
+                return (a.agent ?? "").localeCompare(b.agent ?? "");
+            });
+            for (const op of allOps) {
+                if (sinceTimestamp && op.ts <= sinceTimestamp)
+                    continue;
+                yield op;
+            }
+        }
+        else {
+            // Single-writer: yield directly from ops array (no extra accumulation)
+            const ops = (await this.backend.readOps(manifest.activeOps));
+            for (const op of ops) {
+                if (sinceTimestamp && op.ts <= sinceTimestamp)
+                    continue;
+                yield op;
+            }
+        }
+    }
     count(predicate) {
         this.ensureOpen();
         if (!predicate)
@@ -297,6 +396,9 @@ export class Store {
     async compact() {
         this.ensureOpen();
         this.ensureWritable();
+        if (this.coreOpts.skipLoad) {
+            throw new Error("Cannot compact in skipLoad mode — in-memory Map is incomplete");
+        }
         return this.serialize(() => this._compact());
     }
     async archive(predicate, segment) {

package/dist/types.d.ts

@@ -43,6 +43,13 @@ export interface ArchiveSegment<T = Record<string, unknown>> {
     timestamp: string;
     records: Record<string, T>;
 }
+/** Read-only manifest info exposed to consumers via store.getManifest(). */
+export interface ManifestInfo {
+    readonly currentSnapshot: string;
+    readonly activeOps: string;
+    readonly archiveSegments: readonly string[];
+    readonly stats: Readonly<ManifestStats>;
+}
 export interface StoreOptions {
     /** Auto-checkpoint after this many operations (default: 100) */
     checkpointThreshold?: number;
@@ -54,6 +61,8 @@ export interface StoreOptions {
     migrate?: (record: unknown, fromVersion: number) => unknown;
     /** Open in read-only mode: skips directory lock, rejects all mutations. */
     readOnly?: boolean;
+    /** Skip loading snapshot and replaying WAL into memory. Store opens for writes only — reads return empty. For consumers that manage their own read path (e.g. Parquet-backed storage). */
+    skipLoad?: boolean;
     /** Storage backend implementation (default: FsBackend). */
     backend?: StorageBackend;
     /** Agent ID for multi-writer mode. Enables per-agent WAL streams and LWW conflict resolution. */

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@backloghq/opslog",
-  "version": "0.6.0",
+  "version": "0.7.1",
   "description": "Embedded event-sourced document store. Append-only operation log with immutable snapshots, zero native dependencies.",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",