@typeberry/lib 0.8.4 → 0.9.0-4a9f363

package/packages/workers/api-node/config.js

@@ -1,10 +1,18 @@
 import { Decoder, Encoder } from "#@typeberry/codec";
 import { ChainSpec } from "#@typeberry/config";
 import { InMemoryBlocks, InMemorySerializedStates, } from "#@typeberry/database";
-import { HybridSerializedStates, LmdbBlocks, LmdbRoot, LmdbStates } from "#@typeberry/database-lmdb";
+import { FjallBlocks, HybridSerializedStates as FjallHybridSerializedStates, FjallRoot, FjallStates, FjallValuesSession, } from "#@typeberry/database-fjall";
+import { LmdbBlocks, HybridSerializedStates as LmdbHybridSerializedStates, LmdbRoot, LmdbStates, } from "#@typeberry/database-lmdb";
 import { Blake2b } from "#@typeberry/hash";
 import { ThreadPort } from "./port.js";
-/** A worker config that's usable in node.js and uses LMDB database backend. */
+// Re-exported so the fuzz target can open one values session per run and reuse
+// it across resets (see `HybridWorkerConfig` / `mainFuzz`).
+export { FjallValuesSession };
+/**
+ * Worker config for node.js, backed by the LMDB database.
+ *
+ * @deprecated lmdb is retained as an explicit fallback. Use `FjallWorkerConfig` for regular nodes.
+ */
 export class LmdbWorkerConfig {
     nodeName;
     chainSpec;
@@ -18,10 +26,10 @@ export class LmdbWorkerConfig {
     }
     /** Restore node config from a transferable config object. */
     static async fromTransferable(decodeParams, config) {
-        const blake2b = await Blake2b.createHasher();
-        const chainSpec = ChainSpec.new(config.chainSpec);
-        const workerParams = Decoder.decodeObject(decodeParams, config.workerParams, chainSpec);
-        const ports = new Map(config.workerPorts.map(([name, port]) => [name, ThreadPort.fromTransferable(chainSpec, port)]));
+        if (config.databaseBackend !== "lmdb") {
+            throw new Error(`Expected lmdb worker config, got ${config.databaseBackend}.`);
+        }
+        const { blake2b, chainSpec, workerParams, ports } = await decodeTransferableConfig(decodeParams, config);
         return LmdbWorkerConfig.new({
             nodeName: config.nodeName,
             chainSpec,
@@ -32,7 +40,7 @@ export class LmdbWorkerConfig {
         });
     }
     constructor(nodeName, chainSpec, workerParams, dbPath, blake2b, ports, 
-    // When set, the underlying LMDB skips fsync. Only safe for throwaway
+    // When set, the underlying database skips fsync. Only safe for throwaway
     // databases (the fuzz target wipes on reset). Not transferred to worker
     // threads, so the durable main node path always gets the default.
     ephemeral = false) {
@@ -44,7 +52,7 @@ export class LmdbWorkerConfig {
         this.ports = ports;
         this.ephemeral = ephemeral;
     }
-    openDatabase(options = { readonly: true }) {
+    async openDatabase(options = { readonly: true }) {
         const lmdb = LmdbRoot.new(this.dbPath, {
             readOnly: options.readonly,
             ephemeral: this.ephemeral,
@@ -58,14 +66,128 @@ export class LmdbWorkerConfig {
     /** Convert this config into a thread-transferable object. */
     intoTransferable(paramsCodec) {
         return {
+            databaseBackend: "lmdb",
+            nodeName: this.nodeName,
+            chainSpec: this.chainSpec,
+            workerParams: Encoder.encodeObject(paramsCodec, this.workerParams, this.chainSpec).raw,
+            dbPath: this.dbPath,
+            workerPorts: Array.from(this.ports.entries()).map(([name, port]) => [name, port.intoTransferable()]),
+        };
+    }
+}
+/** Worker config for node.js, backed by a shared fjall engine. */
+export class FjallWorkerConfig {
+    nodeName;
+    chainSpec;
+    workerParams;
+    dbPath;
+    blake2b;
+    ports;
+    ephemeral;
+    cacheSizeBytes;
+    static new({ nodeName, chainSpec, workerParams, dbPath, blake2b, ports = new Map(), ephemeral = false, cacheSizeBytes, }) {
+        return new FjallWorkerConfig(nodeName, chainSpec, workerParams, dbPath, blake2b, ports, ephemeral, cacheSizeBytes);
+    }
+    /** Restore node config from a transferable config object. */
+    static async fromTransferable(decodeParams, config) {
+        if (config.databaseBackend !== "fjall") {
+            throw new Error(`Expected fjall worker config, got ${config.databaseBackend}.`);
+        }
+        const { blake2b, chainSpec, workerParams, ports } = await decodeTransferableConfig(decodeParams, config);
+        return FjallWorkerConfig.new({
+            nodeName: config.nodeName,
+            chainSpec,
+            workerParams,
+            dbPath: config.dbPath,
+            blake2b,
+            ports,
+            cacheSizeBytes: config.cacheSizeBytes,
+        });
+    }
+    constructor(nodeName, chainSpec, workerParams, dbPath, blake2b, ports, 
+    // Kept for the fuzz/importer path. When set, persist() is skipped.
+    ephemeral = false, cacheSizeBytes = undefined) {
+        this.nodeName = nodeName;
+        this.chainSpec = chainSpec;
+        this.workerParams = workerParams;
+        this.dbPath = dbPath;
+        this.blake2b = blake2b;
+        this.ports = ports;
+        this.ephemeral = ephemeral;
+        this.cacheSizeBytes = cacheSizeBytes;
+    }
+    async openDatabase(options = { readonly: true }) {
+        const fjall = await FjallRoot.open(this.dbPath, {
+            readOnly: options.readonly,
+            ephemeral: this.ephemeral,
+            cacheSizeBytes: this.cacheSizeBytes,
+        });
+        let blocks = null;
+        let states = null;
+        try {
+            [blocks, states] = await Promise.all([
+                FjallBlocks.open(this.chainSpec, fjall),
+                FjallStates.open(this.chainSpec, this.blake2b, fjall),
+            ]);
+        }
+        catch (e) {
+            await fjall.close();
+            throw e;
+        }
+        return {
+            getBlocksDb: () => {
+                if (blocks === null) {
+                    throw new Error("Fjall database is closed.");
+                }
+                return blocks;
+            },
+            getStatesDb: () => {
+                if (states === null) {
+                    throw new Error("Fjall database is closed.");
+                }
+                return states;
+            },
+            close: async () => {
+                blocks = null;
+                states = null;
+                await fjall.close();
+            },
+        };
+    }
+    /** Convert this config into a thread-transferable object. */
+    intoTransferable(paramsCodec) {
+        return {
+            databaseBackend: "fjall",
             nodeName: this.nodeName,
             chainSpec: this.chainSpec,
             workerParams: Encoder.encodeObject(paramsCodec, this.workerParams, this.chainSpec).raw,
             dbPath: this.dbPath,
             workerPorts: Array.from(this.ports.entries()).map(([name, port]) => [name, port.intoTransferable()]),
+            cacheSizeBytes: this.cacheSizeBytes,
         };
     }
 }
+async function decodeTransferableConfig(decodeParams, config) {
+    const blake2b = await Blake2b.createHasher();
+    const chainSpec = ChainSpec.new(config.chainSpec);
+    const workerParams = Decoder.decodeObject(decodeParams, config.workerParams, chainSpec);
+    const ports = new Map(config.workerPorts.map(([name, port]) => [name, ThreadPort.fromTransferable(chainSpec, port)]));
+    return {
+        blake2b,
+        chainSpec,
+        workerParams,
+        ports,
+    };
+}
+/** Restore a persistent worker config from its transferable form. */
+export async function persistentConfigFromTransferable(decodeParams, config) {
+    switch (config.databaseBackend) {
+        case "lmdb":
+            return LmdbWorkerConfig.fromTransferable(decodeParams, config);
+        case "fjall":
+            return FjallWorkerConfig.fromTransferable(decodeParams, config);
+    }
+}
 /**
  * Collect the transferable objects (communication ports) embedded in a config.
  *
@@ -99,7 +221,7 @@ export class InMemWorkerConfig {
         this.blocks = InMemoryBlocks.new();
         this.states = InMemorySerializedStates.withHasher({ chainSpec, blake2b });
     }
-    openDatabase(_options = { readonly: true }) {
+    async openDatabase(_options = { readonly: true }) {
         // opening/closing db doesn't do anything, we persist the state.
         return {
             getBlocksDb: () => this.blocks,
@@ -110,11 +232,14 @@ export class InMemWorkerConfig {
 }
 /**
  * Hybrid worker config for the fuzz target: in-memory blocks and leaf sets,
- * but large values persisted to LMDB.
+ * but large values persisted to disk. The `backend` picks where the values go
+ * (lmdb or fjall).
+ *
+ * fjall opens its keyspace asynchronously, that is why `new` here is async.
  *
  * Like `InMemWorkerConfig`, the blocks and leaf sets are shared across the
  * open/close/reopen dance that genesis init performs, so `openDatabase`
- * returns the same instances and a no-op close. The LMDB root is opened once
+ * returns the same instances and a no-op close. The values store is opened once
  * here and closed by `HybridSerializedStates.close()` at importer teardown.
  *
  * In-process only: it holds shared mutable state (the in-memory leaf
@@ -129,12 +254,19 @@ export class HybridWorkerConfig {
     dbPath;
     ephemeral;
     compression;
-    static new({ nodeName, chainSpec, workerParams, blake2b, dbPath, ephemeral = false, compression = true, }) {
-        return new HybridWorkerConfig(nodeName, chainSpec, workerParams, blake2b, dbPath, ephemeral, compression);
+    states;
+    static async new({ nodeName, chainSpec, workerParams, blake2b, dbPath, ephemeral = false, compression = true, backend = "lmdb", sharedFjallSession, }) {
+        // The values store is created once here and shared across reopen. When a
+        // session is given (fuzz reset reuse) we wrap it instead of opening a new one.
+        const states = backend === "fjall"
+            ? sharedFjallSession !== undefined
+                ? FjallHybridSerializedStates.fromSession(chainSpec, blake2b, sharedFjallSession)
+                : await FjallHybridSerializedStates.new({ spec: chainSpec, blake2b, dbPath, ephemeral })
+            : LmdbHybridSerializedStates.new({ spec: chainSpec, blake2b, dbPath, ephemeral, compression, readOnly: false });
+        return new HybridWorkerConfig(nodeName, chainSpec, workerParams, blake2b, dbPath, ephemeral, compression, states);
     }
     blocks;
-    states;
-    constructor(nodeName, chainSpec, workerParams, blake2b, dbPath, ephemeral, compression = true) {
+    constructor(nodeName, chainSpec, workerParams, blake2b, dbPath, ephemeral, compression, states) {
         this.nodeName = nodeName;
         this.chainSpec = chainSpec;
         this.workerParams = workerParams;
@@ -142,22 +274,15 @@ export class HybridWorkerConfig {
         this.dbPath = dbPath;
         this.ephemeral = ephemeral;
         this.compression = compression;
+        this.states = states;
         this.blocks = InMemoryBlocks.new();
-        this.states = HybridSerializedStates.new({
-            spec: this.chainSpec,
-            blake2b: this.blake2b,
-            dbPath: this.dbPath,
-            ephemeral: this.ephemeral,
-            compression: this.compression,
-            readOnly: false,
-        });
     }
-    openDatabase(_options = { readonly: true }) {
+    async openDatabase(_options = { readonly: true }) {
         return {
             getBlocksDb: () => this.blocks,
             getStatesDb: () => this.states,
-            // Leaf sets and blocks live in memory; the LMDB values store is closed
-            // via states.close() at importer teardown, so this is a no-op.
+            // Leaf sets and blocks live in memory; the values store is closed via
+            // states.close() at importer teardown, so this is a no-op.
             close: async () => { },
         };
     }

package/packages/workers/api-node/config.test.js

@@ -1,11 +1,13 @@
 import assert from "node:assert";
+import * as fs from "node:fs";
 import { describe, it } from "node:test";
 import { MessageChannel } from "node:worker_threads";
+import { Bytes } from "#@typeberry/bytes";
 import { codec } from "#@typeberry/codec";
 import { tinyChainSpec } from "#@typeberry/config";
-import { Blake2b } from "#@typeberry/hash";
+import { Blake2b, HASH_SIZE } from "#@typeberry/hash";
 import { tryAsU32 } from "#@typeberry/numbers";
-import { configTransferList, LmdbWorkerConfig } from "./config.js";
+import { configTransferList, FjallWorkerConfig, HybridWorkerConfig, LmdbWorkerConfig } from "./config.js";
 import { ThreadPort } from "./port.js";
 const spec = tinyChainSpec;
 describe("LmdbWorkerConfig transfer list", () => {
@@ -39,3 +41,87 @@ describe("LmdbWorkerConfig transfer list", () => {
         }
     });
 });
+describe("FjallWorkerConfig transfer list", () => {
+    it("surfaces embedded worker ports and marks the backend", async () => {
+        const blake2b = await Blake2b.createHasher();
+        const [portA, portB] = ThreadPort.pair(spec);
+        const config = FjallWorkerConfig.new({
+            nodeName: "node",
+            chainSpec: spec,
+            workerParams: tryAsU32(7),
+            dbPath: "db",
+            blake2b,
+            ports: new Map([["authorship-network", portA]]),
+        });
+        try {
+            const transferable = config.intoTransferable(codec.varU32);
+            assert.strictEqual(transferable.databaseBackend, "fjall");
+            assert.strictEqual(configTransferList(transferable).length, 1);
+        }
+        finally {
+            portA.close();
+            portB.close();
+        }
+    });
+    it("opens writable and read-only handles over one shared fjall path", async () => {
+        const blake2b = await Blake2b.createHasher();
+        const dbPath = fs.mkdtempSync("typeberry-fjall-worker-");
+        const config = FjallWorkerConfig.new({
+            nodeName: "node",
+            chainSpec: spec,
+            workerParams: undefined,
+            dbPath,
+            blake2b,
+        });
+        let writer = null;
+        let reader = null;
+        try {
+            writer = await config.openDatabase({ readonly: false });
+            reader = await config.openDatabase({ readonly: true });
+            const best = Bytes.fill(HASH_SIZE, 9).asOpaque();
+            await writer.getBlocksDb().setBestHeaderHash(best);
+            assert.strictEqual(reader.getBlocksDb().getBestHeaderHash().toString(), best.toString());
+        }
+        finally {
+            await reader?.close();
+            await writer?.close();
+            fs.rmSync(dbPath, { recursive: true, force: true });
+        }
+    });
+});
+describe("HybridWorkerConfig", () => {
+    // Both persistent backends must construct asynchronously and hand out a
+    // working db. fjall is the experimental backend we want to benchmark.
+    for (const backend of ["lmdb", "fjall"]) {
+        it(`constructs and opens a ${backend}-backed hybrid db`, async () => {
+            const blake2b = await Blake2b.createHasher();
+            const dbPath = fs.mkdtempSync(`typeberry-hybrid-${backend}-`);
+            try {
+                const config = await HybridWorkerConfig.new({
+                    nodeName: "node",
+                    chainSpec: spec,
+                    workerParams: undefined,
+                    blake2b,
+                    dbPath,
+                    ephemeral: true,
+                    backend,
+                });
+                const db = await config.openDatabase({ readonly: false });
+                const states = db.getStatesDb();
+                try {
+                    assert.notStrictEqual(db.getBlocksDb(), undefined);
+                    assert.notStrictEqual(states, undefined);
+                }
+                finally {
+                    // The values store owns the on-disk resources (the no-op db.close()
+                    // does not), so close it explicitly to release the fjall keyspace.
+                    await states.close();
+                    await db.close();
+                }
+            }
+            finally {
+                fs.rmSync(dbPath, { recursive: true, force: true });
+            }
+        });
+    }
+});

package/packages/workers/api-node/port.d.ts

@@ -18,6 +18,7 @@ export declare class ThreadPort implements Port {
     static pair(spec: ChainSpec): [ThreadPort, ThreadPort];
     readonly threadId: number;
     private readonly events;
+    private readonly pendingMessages;
     static new(spec: ChainSpec, port: MessagePort): ThreadPort;
     private constructor();
     static fromTransferable(spec: ChainSpec, { port }: TransferablePort): ThreadPort;
@@ -28,5 +29,7 @@ export declare class ThreadPort implements Port {
     on<T>(event: string, codec: Codec<T>, callback: (msg: Envelope<T>) => void): () => void;
     once<T>(event: string, codec: Codec<T>, callback: (msg: Envelope<T>) => void): () => void;
     postMessage<T>(event: string, codec: Codec<T>, msg: Envelope<T>): void;
+    private queuePending;
+    private flushPending;
 }
 //# sourceMappingURL=port.d.ts.map

package/packages/workers/api-node/port.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"port.d.ts","sourceRoot":"","sources":["../../../../../packages/workers/api-node/port.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,KAAK,WAAW,EAA+B,MAAM,qBAAqB,CAAC;AACpF,OAAO,EAAE,KAAK,KAAK,EAAoB,MAAM,kBAAkB,CAAC;AAChE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AAGnD,OAAO,KAAK,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,gCAAgC,CAAC;AAErE,MAAM,MAAM,OAAO,GAAG;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,IAAI,EAAE,UAAU,CAAC;CAClB,CAAC;AAIF,qDAAqD;AACrD,MAAM,MAAM,gBAAgB,GAAG;IAC7B,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,WAAW,CAAC;CACnB,CAAC;AACF,qBAAa,UAAW,YAAW,IAAI;IAenC,OAAO,CAAC,QAAQ,CAAC,IAAI;IACrB,OAAO,CAAC,QAAQ,CAAC,IAAI;IAfvB,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,SAAS,GAAG,CAAC,UAAU,EAAE,UAAU,CAAC;IAMtD,SAAgB,QAAQ,SAAY;IACpC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAsB;IAE7C,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,WAAW;IAI7C,OAAO;IAgBP,MAAM,CAAC,gBAAgB,CAAC,IAAI,EAAE,SAAS,EAAE,EAAE,IAAI,EAAE,EAAE,gBAAgB;IAInE,gBAAgB,IAAI,gBAAgB;IAIpC,KAAK,IAAI,IAAI;IAIb,OAAO,CAAC,cAAc;IAkBtB,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC,EAAE,KAAK,KAAK,IAAI,GAAG,IAAI;IAI3C,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC,CAAC,KAAK,IAAI,GAAG,MAAM,IAAI;IASvF,IAAI,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC,CAAC,KAAK,IAAI,GAAG,MAAM,IAAI;IASzF,WAAW,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,QAAQ,CAAC,CAAC,CAAC,GAAG,IAAI;CAWvE"}
+{"version":3,"file":"port.d.ts","sourceRoot":"","sources":["../../../../../packages/workers/api-node/port.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,KAAK,WAAW,EAA+B,MAAM,qBAAqB,CAAC;AACpF,OAAO,EAAE,KAAK,KAAK,EAAoB,MAAM,kBAAkB,CAAC;AAChE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AAEnD,OAAO,KAAK,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,gCAAgC,CAAC;AAErE,MAAM,MAAM,OAAO,GAAG;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,IAAI,EAAE,UAAU,CAAC;CAClB,CAAC;AAIF,qDAAqD;AACrD,MAAM,MAAM,gBAAgB,GAAG;IAC7B,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,WAAW,CAAC;CACnB,CAAC;AACF,qBAAa,UAAW,YAAW,IAAI;IAgBnC,OAAO,CAAC,QAAQ,CAAC,IAAI;IACrB,OAAO,CAAC,QAAQ,CAAC,IAAI;IAhBvB,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,SAAS,GAAG,CAAC,UAAU,EAAE,UAAU,CAAC;IAMtD,SAAgB,QAAQ,SAAY;IACpC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAsB;IAC7C,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAgC;IAEhE,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,WAAW;IAI7C,OAAO;IAmBP,MAAM,CAAC,gBAAgB,CAAC,IAAI,EAAE,SAAS,EAAE,EAAE,IAAI,EAAE,EAAE,gBAAgB;IAInE,gBAAgB,IAAI,gBAAgB;IAIpC,KAAK,IAAI,IAAI;IAKb,OAAO,CAAC,cAAc;IAkBtB,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC,EAAE,KAAK,KAAK,IAAI,GAAG,IAAI;IAI3C,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC,CAAC,KAAK,IAAI,GAAG,MAAM,IAAI;IAUvF,IAAI,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC,CAAC,KAAK,IAAI,GAAG,MAAM,IAAI;IAoBzF,WAAW,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,QAAQ,CAAC,CAAC,CAAC,GAAG,IAAI;IAYtE,OAAO,CAAC,YAAY;IAMpB,OAAO,CAAC,YAAY;CAUrB"}

package/packages/workers/api-node/port.js

@@ -2,7 +2,6 @@ import EventEmitter from "node:events";
 import { threadId } from "node:worker_threads";
 import { Decoder, Encoder } from "#@typeberry/codec";
 import { Logger } from "#@typeberry/logger";
-import { check } from "#@typeberry/utils";
 const MESSAGE_KEYS = ["eventName", "responseId", "data"];
 const logger = Logger.new(import.meta.filename, "workers/api");
 export class ThreadPort {
@@ -14,6 +13,7 @@ export class ThreadPort {
     }
     threadId = threadId;
     events = new EventEmitter();
+    pendingMessages = new Map();
     static new(spec, port) {
         return new ThreadPort(spec, port);
     }
@@ -26,7 +26,10 @@ export class ThreadPort {
                 return;
             }
             const { eventName, responseId, data } = input;
-            check `${this.events.listeners(eventName).length > 0} No listeners for received event ${eventName}!`;
+            if (this.events.listeners(eventName).length === 0) {
+                this.queuePending(input);
+                return;
+            }
             this.events.emit(eventName, responseId, data);
         });
     }
@@ -38,6 +41,7 @@ export class ThreadPort {
     }
     close() {
         this.port.close();
+        this.pendingMessages.clear();
     }
     createListener(codec, callback) {
         return (responseId, data) => {
@@ -64,12 +68,22 @@ export class ThreadPort {
     on(event, codec, callback) {
         const listener = this.createListener(codec, callback);
         this.events.on(event, listener);
+        this.flushPending(event, listener);
         return () => {
             this.events.off(event, listener);
         };
     }
     once(event, codec, callback) {
         const listener = this.createListener(codec, callback);
+        const pending = this.pendingMessages.get(event);
+        const pendingMessage = pending?.shift();
+        if (pending?.length === 0) {
+            this.pendingMessages.delete(event);
+        }
+        if (pendingMessage !== undefined) {
+            listener(pendingMessage.responseId, pendingMessage.data);
+            return () => { };
+        }
         this.events.once(event, listener);
         return () => {
             this.events.off(event, listener);
@@ -86,6 +100,21 @@ export class ThreadPort {
         // always returns owned uint8arrays.
         this.port.postMessage(message, [encoded.raw.buffer]);
     }
+    queuePending(msg) {
+        const pending = this.pendingMessages.get(msg.eventName) ?? [];
+        pending.push(msg);
+        this.pendingMessages.set(msg.eventName, pending);
+    }
+    flushPending(event, listener) {
+        const pending = this.pendingMessages.get(event);
+        if (pending === undefined) {
+            return;
+        }
+        this.pendingMessages.delete(event);
+        for (const msg of pending) {
+            listener(msg.responseId, msg.data);
+        }
+    }
 }
 function isMessage(data) {
     const isObject = data !== null && typeof data === "object";

package/packages/workers/api-node/port.test.js

@@ -36,4 +36,21 @@ describe("ThreadPort", () => {
             data: tryAsU32(42),
         });
     });
+    it("should deliver messages sent before the listener is attached", async () => {
+        const tx = ThreadPort.new(spec, channel.port1);
+        const rx = ThreadPort.new(spec, channel.port2);
+        tx.postMessage("hello", codec.varU32, {
+            responseId: "10",
+            data: tryAsU32(42),
+        });
+        await setTimeout(10);
+        let received = null;
+        rx.on("hello", codec.varU32, (msg) => {
+            received = msg;
+        });
+        assert.deepStrictEqual(received, {
+            responseId: "10",
+            data: tryAsU32(42),
+        });
+    });
 });

package/packages/workers/api-node/protocol.d.ts

@@ -2,7 +2,7 @@ import { type MessagePort, Worker } from "node:worker_threads";
 import type { Decode, Encode } from "#@typeberry/codec";
 import { Listener } from "#@typeberry/listener";
 import type { Api, Internal, LousyProtocol } from "#@typeberry/workers-api/types.js";
-import { LmdbWorkerConfig, type TransferableConfig } from "./config.js";
+import { type PersistentWorkerConfig, type TransferableConfig } from "./config.js";
 /** Type of the control plane message. */
 export declare enum WorkerControlPlaneMsg {
     /** Transfers a communication port to some other thread. */
@@ -31,7 +31,7 @@ export type WorkerControlPlane = ({
 /**
  * Invoked by the main thread, to spawn a worker thread and initiate communication channel.
  */
-export declare function spawnWorker<To, From, Params>(protocol: LousyProtocol<To, From>, bootstrapPath: URL, config: LmdbWorkerConfig<Params>, paramsEncoder: Encode<Params>): {
+export declare function spawnWorker<To, From, Params>(protocol: LousyProtocol<To, From>, bootstrapPath: URL, config: PersistentWorkerConfig<Params>, paramsEncoder: Encode<Params>): {
     api: Api<typeof protocol>;
     worker: Worker;
     workerFinished: Promise<void>;
@@ -40,7 +40,7 @@ export declare function spawnWorker<To, From, Params>(protocol: LousyProtocol<To
  * Initialize worker thread by awaiting the config message.
  */
 export declare function initWorker<To, From, Params>(protocol: LousyProtocol<To, From>, paramsDecoder: Decode<Params>): Promise<{
-    config: LmdbWorkerConfig<Params>;
+    config: PersistentWorkerConfig<Params>;
     comms: Internal<typeof protocol>;
     threadComms: Listener<ThreadComms>;
 }>;

package/packages/workers/api-node/protocol.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"protocol.d.ts","sourceRoot":"","sources":["../../../../../packages/workers/api-node/protocol.ts"],"names":[],"mappings":"AAAA,OAAO,EAAkB,KAAK,WAAW,EAAc,MAAM,EAAE,MAAM,qBAAqB,CAAC;AAC3F,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AACvD,OAAO,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AAI/C,OAAO,KAAK,EAAE,GAAG,EAAE,QAAQ,EAAE,aAAa,EAAE,MAAM,iCAAiC,CAAC;AACpF,OAAO,EAAsB,gBAAgB,EAAE,KAAK,kBAAkB,EAAE,MAAM,aAAa,CAAC;AAM5F,yCAAyC;AACzC,oBAAY,qBAAqB;IAC/B,2DAA2D;IAC3D,iBAAiB,IAAI;IACrB,sCAAsC;IACtC,MAAM,IAAI;CACX;AAED,MAAM,MAAM,WAAW,GAAG;IACxB,kDAAkD;IAClD,UAAU,EAAE,MAAM,CAAC;IACnB,0BAA0B;IAC1B,IAAI,EAAE,WAAW,CAAC;CACnB,CAAC;AACF,yFAAyF;AACzF,MAAM,MAAM,kBAAkB,GAC1B,CAAC;IACC,wDAAwD;IACxD,IAAI,EAAE,qBAAqB,CAAC,iBAAiB,CAAC;CAC/C,GAAG,WAAW,CAAC,GAChB;IACE,8EAA8E;IAC9E,IAAI,EAAE,qBAAqB,CAAC,MAAM,CAAC;IACnC,sCAAsC;IACtC,UAAU,EAAE,WAAW,CAAC;IACxB,4BAA4B;IAC5B,MAAM,EAAE,kBAAkB,CAAC;CAC5B,CAAC;AAeN;;GAEG;AACH,wBAAgB,WAAW,CAAC,EAAE,EAAE,IAAI,EAAE,MAAM,EAC1C,QAAQ,EAAE,aAAa,CAAC,EAAE,EAAE,IAAI,CAAC,EACjC,aAAa,EAAE,GAAG,EAClB,MAAM,EAAE,gBAAgB,CAAC,MAAM,CAAC,EAChC,aAAa,EAAE,MAAM,CAAC,MAAM,CAAC,GAC5B;IACD,GAAG,EAAE,GAAG,CAAC,OAAO,QAAQ,CAAC,CAAC;IAC1B,MAAM,EAAE,MAAM,CAAC;IACf,cAAc,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC;CAC/B,CAoCA;AAED;;GAEG;AACH,wBAAsB,UAAU,CAAC,EAAE,EAAE,IAAI,EAAE,MAAM,EAC/C,QAAQ,EAAE,aAAa,CAAC,EAAE,EAAE,IAAI,CAAC,EACjC,aAAa,EAAE,MAAM,CAAC,MAAM,CAAC,GAC5B,OAAO,CAAC;IACT,MAAM,EAAE,gBAAgB,CAAC,MAAM,CAAC,CAAC;IACjC,KAAK,EAAE,QAAQ,CAAC,OAAO,QAAQ,CAAC,CAAC;IACjC,WAAW,EAAE,QAAQ,CAAC,WAAW,CAAC,CAAC;CACpC,CAAC,CAmDD"}
+{"version":3,"file":"protocol.d.ts","sourceRoot":"","sources":["../../../../../packages/workers/api-node/protocol.ts"],"names":[],"mappings":"AAAA,OAAO,EAAkB,KAAK,WAAW,EAAc,MAAM,EAAE,MAAM,qBAAqB,CAAC;AAC3F,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AACvD,OAAO,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AAI/C,OAAO,KAAK,EAAE,GAAG,EAAE,QAAQ,EAAE,aAAa,EAAE,MAAM,iCAAiC,CAAC;AACpF,OAAO,EAEL,KAAK,sBAAsB,EAE3B,KAAK,kBAAkB,EACxB,MAAM,aAAa,CAAC;AAMrB,yCAAyC;AACzC,oBAAY,qBAAqB;IAC/B,2DAA2D;IAC3D,iBAAiB,IAAI;IACrB,sCAAsC;IACtC,MAAM,IAAI;CACX;AAED,MAAM,MAAM,WAAW,GAAG;IACxB,kDAAkD;IAClD,UAAU,EAAE,MAAM,CAAC;IACnB,0BAA0B;IAC1B,IAAI,EAAE,WAAW,CAAC;CACnB,CAAC;AACF,yFAAyF;AACzF,MAAM,MAAM,kBAAkB,GAC1B,CAAC;IACC,wDAAwD;IACxD,IAAI,EAAE,qBAAqB,CAAC,iBAAiB,CAAC;CAC/C,GAAG,WAAW,CAAC,GAChB;IACE,8EAA8E;IAC9E,IAAI,EAAE,qBAAqB,CAAC,MAAM,CAAC;IACnC,sCAAsC;IACtC,UAAU,EAAE,WAAW,CAAC;IACxB,4BAA4B;IAC5B,MAAM,EAAE,kBAAkB,CAAC;CAC5B,CAAC;AAeN;;GAEG;AACH,wBAAgB,WAAW,CAAC,EAAE,EAAE,IAAI,EAAE,MAAM,EAC1C,QAAQ,EAAE,aAAa,CAAC,EAAE,EAAE,IAAI,CAAC,EACjC,aAAa,EAAE,GAAG,EAClB,MAAM,EAAE,sBAAsB,CAAC,MAAM,CAAC,EACtC,aAAa,EAAE,MAAM,CAAC,MAAM,CAAC,GAC5B;IACD,GAAG,EAAE,GAAG,CAAC,OAAO,QAAQ,CAAC,CAAC;IAC1B,MAAM,EAAE,MAAM,CAAC;IACf,cAAc,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC;CAC/B,CAoCA;AAED;;GAEG;AACH,wBAAsB,UAAU,CAAC,EAAE,EAAE,IAAI,EAAE,MAAM,EAC/C,QAAQ,EAAE,aAAa,CAAC,EAAE,EAAE,IAAI,CAAC,EACjC,aAAa,EAAE,MAAM,CAAC,MAAM,CAAC,GAC5B,OAAO,CAAC;IACT,MAAM,EAAE,sBAAsB,CAAC,MAAM,CAAC,CAAC;IACvC,KAAK,EAAE,QAAQ,CAAC,OAAO,QAAQ,CAAC,CAAC;IACjC,WAAW,EAAE,QAAQ,CAAC,WAAW,CAAC,CAAC;CACpC,CAAC,CAwDD"}

package/packages/workers/api-node/protocol.js

@@ -3,7 +3,7 @@ import { Listener } from "#@typeberry/listener";
 import { Level, Logger } from "#@typeberry/logger";
 import { assertNever } from "#@typeberry/utils";
 import { Channel } from "#@typeberry/workers-api";
-import { configTransferList, LmdbWorkerConfig } from "./config.js";
+import { configTransferList, persistentConfigFromTransferable, } from "./config.js";
 import { logHeapLimit, workerResourceLimits } from "./host-environment.js";
 import { ThreadPort } from "./port.js";
 const logger = Logger.new(import.meta.filename, "workers");
@@ -70,13 +70,14 @@ export async function initWorker(protocol, paramsDecoder) {
     logger.trace `Worker ${protocol.name} starting.`;
     logHeapLimit(logger, protocol.name);
     return new Promise((resolve, reject) => {
-        if (parentPort === null) {
+        const workerParentPort = parentPort;
+        if (workerParentPort === null) {
             throw new Error(`Unable to start ${protocol.name} worker. Not running in a worker thread!`);
         }
-        parentPort.once("close", () => reject(new Error(`(${protocol.name}) parent port closed too early`)));
+        workerParentPort.once("close", () => reject(new Error(`(${protocol.name}) parent port closed too early`)));
         let isResolved = false;
         const threadComms = new Listener();
-        parentPort.on("message", async (msg) => {
+        workerParentPort.on("message", async (msg) => {
             if (!isControlPlane(msg)) {
                 logger.error `--> (${protocol.name}) received unexpected message: ${msg}`;
                 return;
@@ -93,14 +94,19 @@ export async function initWorker(protocol, paramsDecoder) {
                 }
                 logger.trace `--> (${protocol.name}) received configuration.`;
                 isResolved = true;
-                const config = await LmdbWorkerConfig.fromTransferable(paramsDecoder, msg.config);
-                const rxPort = ThreadPort.new(config.chainSpec, msg.parentPort);
-                const comms = Channel.rx(protocol, rxPort);
-                resolve({
-                    config,
-                    comms,
-                    threadComms,
-                });
+                try {
+                    const config = await persistentConfigFromTransferable(paramsDecoder, msg.config);
+                    const rxPort = ThreadPort.new(config.chainSpec, msg.parentPort);
+                    const comms = Channel.rx(protocol, rxPort);
+                    resolve({
+                        config,
+                        comms,
+                        threadComms,
+                    });
+                }
+                catch (e) {
+                    reject(e);
+                }
                 return;
             }
             assertNever(msg.kind);

package/packages/workers/block-authorship/{generator.d.ts → block-generator.d.ts}

@@ -18,7 +18,7 @@ import { type Opaque } from "#@typeberry/utils";
  *
  */
 export type BlockSealInput = Opaque<BytesBlob, "Seal">;
-/** Construction arguments for {@link Generator}. */
+/** Construction arguments for `BlockGenerator`. */
 export type GeneratorArgs = {
     chainSpec: ChainSpec;
     bandersnatch: BandernsatchWasm;
@@ -27,7 +27,7 @@ export type GeneratorArgs = {
     blocks: BlocksDb;
     states: StatesDb;
 };
-export declare class Generator {
+export declare class BlockGenerator {
     private readonly metrics;
     readonly chainSpec: ChainSpec;
     readonly bandersnatch: BandernsatchWasm;
@@ -35,8 +35,8 @@ export declare class Generator {
     readonly blake2b: Blake2b;
     private readonly blocks;
     private readonly states;
-    /** Build a {@link Generator} from its collaborators. */
-    static new(args: GeneratorArgs): Generator;
+    /** Build a block generator from its collaborators. */
+    static new(args: GeneratorArgs): BlockGenerator;
     private constructor();
     private getLastHeaderAndState;
     nextBlockView(validatorIndex: ValidatorIndex, bandersnatchSecret: BandersnatchSecretSeed, sealPayload: BlockSealInput, timeSlot: TimeSlot, pendingTickets?: {
@@ -71,4 +71,4 @@ export declare class Generator {
         id: EntropyHash;
     }[]): Promise<Block>;
 }
-//# sourceMappingURL=generator.d.ts.map
+//# sourceMappingURL=block-generator.d.ts.map

package/packages/workers/block-authorship/block-generator.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"block-generator.d.ts","sourceRoot":"","sources":["../../../../../packages/workers/block-authorship/block-generator.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,KAAK,EACL,KAAK,WAAW,EAIhB,KAAK,QAAQ,EACb,KAAK,cAAc,EACpB,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAAE,KAAK,SAAS,EAAa,MAAM,2BAA2B,CAAC;AAEtE,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,6BAA6B,CAAC;AAChE,OAAO,EAAS,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAEpD,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AACnD,OAAO,EAAoC,KAAK,sBAAsB,EAAE,MAAM,mBAAmB,CAAC;AAClG,OAAO,KAAK,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AAC9D,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,iBAAiB,CAAC;AAIvD,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,yCAAyC,CAAC;AAGhF,OAAO,EAAqB,KAAK,MAAM,EAAU,MAAM,kBAAkB,CAAC;AAM1E;;;;;;;;GAQG;AACH,MAAM,MAAM,cAAc,GAAG,MAAM,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;AAEvD,mDAAmD;AACnD,MAAM,MAAM,aAAa,GAAG;IAC1B,SAAS,EAAE,SAAS,CAAC;IACrB,YAAY,EAAE,gBAAgB,CAAC;IAC/B,YAAY,EAAE,MAAM,CAAC,YAAY,CAAC;IAClC,OAAO,EAAE,OAAO,CAAC;IACjB,MAAM,EAAE,QAAQ,CAAC;IACjB,MAAM,EAAE,QAAQ,CAAC;CAClB,CAAC;AAEF,qBAAa,cAAc;IACzB,OAAO,CAAC,QAAQ,CAAC,OAAO,CAA2C;IAEnE,SAAgB,SAAS,EAAE,SAAS,CAAC;IACrC,SAAgB,YAAY,EAAE,gBAAgB,CAAC;IAC/C,SAAgB,YAAY,EAAE,MAAM,CAAC,YAAY,CAAC;IAClD,SAAgB,OAAO,EAAE,OAAO,CAAC;IACjC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAW;IAClC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAW;IAElC,sDAAsD;IACtD,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,aAAa;IAI9B,OAAO;IAUP,OAAO,CAAC,qBAAqB;IAYvB,aAAa,CACjB,cAAc,EAAE,cAAc,EAC9B,kBAAkB,EAAE,sBAAsB,EAC1C,WAAW,EAAE,cAAc,EAC3B,QAAQ,EAAE,QAAQ,EAClB,cAAc,GAAE;QAAE,MAAM,EAAE,YAAY,CAAC;QAAC,EAAE,EAAE,WAAW,CAAA;KAAE,EAAO,GAC/D,OAAO,CAAC,SAAS,CAAC;IAKrB;;;;;;;;;OASG;YACW,cAAc;IAiB5B;;;;;;;;;;OAUG;IACH,OAAO,CAAC,uBAAuB;IA4BzB,SAAS,CACb,cAAc,EAAE,cAAc,EAC9B,kBAAkB,EAAE,sBAAsB,EAC1C,WAAW,EAAE,cAAc,EAC3B,QAAQ,EAAE,QAAQ,EAClB,cAAc,GAAE;QAAE,MAAM,EAAE,YAAY,CAAC;QAAC,EAAE,EAAE,WAAW,CAAA;KAAE,EAAO;CAqGnE"}

package/packages/workers/block-authorship/{generator.js → block-generator.js}

@@ -13,7 +13,7 @@ import { asOpaqueType, now, Result } from "#@typeberry/utils";
 import * as metrics from "./metrics.js";
 const EMPTY_AUX_DATA = BytesBlob.empty();
 const logger = Logger.new(import.meta.filename, "author");
-export class Generator {
+export class BlockGenerator {
     metrics;
     chainSpec;
     bandersnatch;
@@ -21,9 +21,9 @@ export class Generator {
     blake2b;
     blocks;
     states;
-    /** Build a {@link Generator} from its collaborators. */
+    /** Build a block generator from its collaborators. */
     static new(args) {
-        return new Generator(args);
+        return new BlockGenerator(args);
     }
     constructor(args) {
         this.chainSpec = args.chainSpec;

package/packages/workers/block-authorship/block-generator.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=block-generator.test.d.ts.map

package/packages/workers/block-authorship/block-generator.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"block-generator.test.d.ts","sourceRoot":"","sources":["../../../../../packages/workers/block-authorship/block-generator.test.ts"],"names":[],"mappings":""}