@typeberry/lib 0.11.0-829b7ea → 0.11.0

package/packages/workers/api-node/config.js

@@ -2,11 +2,79 @@ import { Decoder, Encoder } from "#@typeberry/codec";
 import { ChainSpec } from "#@typeberry/config";
 import { InMemoryBlocks, InMemorySerializedStates, } from "#@typeberry/database";
 import { FjallBlocks, HybridSerializedStates as FjallHybridSerializedStates, FjallRoot, FjallStates, FjallValuesSession, } from "#@typeberry/database-fjall";
+import { LmdbBlocks, HybridSerializedStates as LmdbHybridSerializedStates, LmdbRoot, LmdbStates, } from "#@typeberry/database-lmdb";
 import { Blake2b } from "#@typeberry/hash";
 import { ThreadPort } from "./port.js";
 // Re-exported so the fuzz target can open one values session per run and reuse
 // it across resets (see `HybridWorkerConfig` / `mainFuzz`).
 export { FjallValuesSession };
+/**
+ * Worker config for node.js, backed by the LMDB database.
+ *
+ * @deprecated lmdb is retained as an explicit fallback. Use `FjallWorkerConfig` for regular nodes.
+ */
+export class LmdbWorkerConfig {
+    nodeName;
+    chainSpec;
+    workerParams;
+    dbPath;
+    blake2b;
+    ports;
+    ephemeral;
+    static new({ nodeName, chainSpec, workerParams, dbPath, blake2b, ports = new Map(), ephemeral = false, }) {
+        return new LmdbWorkerConfig(nodeName, chainSpec, workerParams, dbPath, blake2b, ports, ephemeral);
+    }
+    /** Restore node config from a transferable config object. */
+    static async fromTransferable(decodeParams, config) {
+        if (config.databaseBackend !== "lmdb") {
+            throw new Error(`Expected lmdb worker config, got ${config.databaseBackend}.`);
+        }
+        const { blake2b, chainSpec, workerParams, ports } = await decodeTransferableConfig(decodeParams, config);
+        return LmdbWorkerConfig.new({
+            nodeName: config.nodeName,
+            chainSpec,
+            workerParams,
+            dbPath: config.dbPath,
+            blake2b,
+            ports,
+        });
+    }
+    constructor(nodeName, chainSpec, workerParams, dbPath, blake2b, ports, 
+    // When set, the underlying database skips fsync. Only safe for throwaway
+    // databases (the fuzz target wipes on reset). Not transferred to worker
+    // threads, so the durable main node path always gets the default.
+    ephemeral = false) {
+        this.nodeName = nodeName;
+        this.chainSpec = chainSpec;
+        this.workerParams = workerParams;
+        this.dbPath = dbPath;
+        this.blake2b = blake2b;
+        this.ports = ports;
+        this.ephemeral = ephemeral;
+    }
+    async openDatabase(options = { readonly: true }) {
+        const lmdb = LmdbRoot.new(this.dbPath, {
+            readOnly: options.readonly,
+            ephemeral: this.ephemeral,
+        });
+        return {
+            getBlocksDb: () => LmdbBlocks.new(this.chainSpec, lmdb),
+            getStatesDb: () => LmdbStates.new(this.chainSpec, this.blake2b, lmdb),
+            close: async () => await lmdb.close(),
+        };
+    }
+    /** Convert this config into a thread-transferable object. */
+    intoTransferable(paramsCodec) {
+        return {
+            databaseBackend: "lmdb",
+            nodeName: this.nodeName,
+            chainSpec: this.chainSpec,
+            workerParams: Encoder.encodeObject(paramsCodec, this.workerParams, this.chainSpec).raw,
+            dbPath: this.dbPath,
+            workerPorts: Array.from(this.ports.entries()).map(([name, port]) => [name, port.intoTransferable()]),
+        };
+    }
+}
 /** Worker config for node.js, backed by a shared fjall engine. */
 export class FjallWorkerConfig {
     nodeName;
@@ -113,7 +181,12 @@ async function decodeTransferableConfig(decodeParams, config) {
 }
 /** Restore a persistent worker config from its transferable form. */
 export async function persistentConfigFromTransferable(decodeParams, config) {
-    return FjallWorkerConfig.fromTransferable(decodeParams, config);
+    switch (config.databaseBackend) {
+        case "lmdb":
+            return LmdbWorkerConfig.fromTransferable(decodeParams, config);
+        case "fjall":
+            return FjallWorkerConfig.fromTransferable(decodeParams, config);
+    }
 }
 /**
  * Collect the transferable objects (communication ports) embedded in a config.
@@ -159,7 +232,8 @@ export class InMemWorkerConfig {
 }
 /**
  * Hybrid worker config for the fuzz target: in-memory blocks and leaf sets,
- * but large values persisted to disk via fjall.
+ * but large values persisted to disk. The `backend` picks where the values go
+ * (lmdb or fjall).
  *
  * fjall opens its keyspace asynchronously, that is why `new` here is async.
  *
@@ -181,12 +255,14 @@ export class HybridWorkerConfig {
     ephemeral;
     compression;
     states;
-    static async new({ nodeName, chainSpec, workerParams, blake2b, dbPath, ephemeral = false, compression = true, sharedFjallSession, }) {
+    static async new({ nodeName, chainSpec, workerParams, blake2b, dbPath, ephemeral = false, compression = true, backend = "lmdb", sharedFjallSession, }) {
         // The values store is created once here and shared across reopen. When a
         // session is given (fuzz reset reuse) we wrap it instead of opening a new one.
-        const states = sharedFjallSession !== undefined
-            ? FjallHybridSerializedStates.fromSession(chainSpec, blake2b, sharedFjallSession)
-            : await FjallHybridSerializedStates.new({ spec: chainSpec, blake2b, dbPath, ephemeral });
+        const states = backend === "fjall"
+            ? sharedFjallSession !== undefined
+                ? FjallHybridSerializedStates.fromSession(chainSpec, blake2b, sharedFjallSession)
+                : await FjallHybridSerializedStates.new({ spec: chainSpec, blake2b, dbPath, ephemeral })
+            : LmdbHybridSerializedStates.new({ spec: chainSpec, blake2b, dbPath, ephemeral, compression, readOnly: false });
         return new HybridWorkerConfig(nodeName, chainSpec, workerParams, blake2b, dbPath, ephemeral, compression, states);
     }
     blocks;

package/packages/workers/api-node/config.test.js

@@ -1,14 +1,46 @@
 import assert from "node:assert";
 import * as fs from "node:fs";
 import { describe, it } from "node:test";
+import { MessageChannel } from "node:worker_threads";
 import { Bytes } from "#@typeberry/bytes";
 import { codec } from "#@typeberry/codec";
 import { tinyChainSpec } from "#@typeberry/config";
 import { Blake2b, HASH_SIZE } from "#@typeberry/hash";
 import { tryAsU32 } from "#@typeberry/numbers";
-import { configTransferList, FjallWorkerConfig, HybridWorkerConfig } from "./config.js";
+import { configTransferList, FjallWorkerConfig, HybridWorkerConfig, LmdbWorkerConfig } from "./config.js";
 import { ThreadPort } from "./port.js";
 const spec = tinyChainSpec;
+describe("LmdbWorkerConfig transfer list", () => {
+    it("surfaces embedded worker ports so they can be transferred", async () => {
+        const blake2b = await Blake2b.createHasher();
+        const [portA, portB] = ThreadPort.pair(spec);
+        const config = LmdbWorkerConfig.new({
+            nodeName: "node",
+            chainSpec: spec,
+            workerParams: tryAsU32(7),
+            dbPath: "db",
+            blake2b,
+            ports: new Map([["authorship-network", portA]]),
+        });
+        const transferable = config.intoTransferable(codec.varU32);
+        const transferList = configTransferList(transferable);
+        // the single embedded comms port must be reported for transfer
+        assert.strictEqual(transferList.length, 1);
+        const sink = new MessageChannel();
+        try {
+            // reproduces the bug: a config carrying a port cannot be cloned without
+            // listing that port in the transfer list.
+            assert.throws(() => sink.port1.postMessage(transferable, []), /transfer/i);
+            // with the ports surfaced, posting succeeds.
+            assert.doesNotThrow(() => sink.port1.postMessage(transferable, transferList));
+        }
+        finally {
+            sink.port1.close();
+            sink.port2.close();
+            portB.close();
+        }
+    });
+});
 describe("FjallWorkerConfig transfer list", () => {
     it("surfaces embedded worker ports and marks the backend", async () => {
         const blake2b = await Blake2b.createHasher();
@@ -58,33 +90,38 @@ describe("FjallWorkerConfig transfer list", () => {
     });
 });
 describe("HybridWorkerConfig", () => {
-    it("constructs and opens a fjall-backed hybrid db", async () => {
-        const blake2b = await Blake2b.createHasher();
-        const dbPath = fs.mkdtempSync("typeberry-hybrid-fjall-");
-        try {
-            const config = await HybridWorkerConfig.new({
-                nodeName: "node",
-                chainSpec: spec,
-                workerParams: undefined,
-                blake2b,
-                dbPath,
-                ephemeral: true,
-            });
-            const db = await config.openDatabase({ readonly: false });
-            const states = db.getStatesDb();
+    // Both persistent backends must construct asynchronously and hand out a
+    // working db. fjall is the experimental backend we want to benchmark.
+    for (const backend of ["lmdb", "fjall"]) {
+        it(`constructs and opens a ${backend}-backed hybrid db`, async () => {
+            const blake2b = await Blake2b.createHasher();
+            const dbPath = fs.mkdtempSync(`typeberry-hybrid-${backend}-`);
             try {
-                assert.notStrictEqual(db.getBlocksDb(), undefined);
-                assert.notStrictEqual(states, undefined);
+                const config = await HybridWorkerConfig.new({
+                    nodeName: "node",
+                    chainSpec: spec,
+                    workerParams: undefined,
+                    blake2b,
+                    dbPath,
+                    ephemeral: true,
+                    backend,
+                });
+                const db = await config.openDatabase({ readonly: false });
+                const states = db.getStatesDb();
+                try {
+                    assert.notStrictEqual(db.getBlocksDb(), undefined);
+                    assert.notStrictEqual(states, undefined);
+                }
+                finally {
+                    // The values store owns the on-disk resources (the no-op db.close()
+                    // does not), so close it explicitly to release the fjall keyspace.
+                    await states.close();
+                    await db.close();
+                }
             }
             finally {
-                // The values store owns the on-disk resources (the no-op db.close()
-                // does not), so close it explicitly to release the fjall keyspace.
-                await states.close();
-                await db.close();
+                fs.rmSync(dbPath, { recursive: true, force: true });
             }
-        }
-        finally {
-            fs.rmSync(dbPath, { recursive: true, force: true });
-        }
-    });
+        });
+    }
 });