@typeberry/lib 0.7.3-ffe481c → 0.7.4-2289aab

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@typeberry/lib",
-  "version": "0.7.3-ffe481c",
+  "version": "0.7.4-2289aab",
   "description": "Typeberry Library",
   "main": "./bin/lib/index.js",
   "types": "./bin/lib/index.d.ts",

package/packages/jam/database-lmdb/hybrid-states.d.ts

@@ -0,0 +1,42 @@
+import type { HeaderHash, StateRootHash } from "#@typeberry/block";
+import type { ChainSpec } from "#@typeberry/config";
+import { type InitStatesDb, LeafDb, type StatesDb, StateUpdateError } from "#@typeberry/database";
+import type { Blake2b } from "#@typeberry/hash";
+import type { ServicesUpdate, State } from "#@typeberry/state";
+import { SerializedState, type StateEntries } from "#@typeberry/state-merkleization";
+import { OK, Result } from "#@typeberry/utils";
+/**
+ * Hybrid serialized-states db.
+ *
+ * States (leafs) are kept in-memory, but large values are persisted to lmdb.
+ * Reads go straight to lmdb, which keeps its own page cache.
+ * NOTE: this DB is designed for long fuzzing and to be used with pruning to
+ * keep the heap usage bounded.
+ */
+export declare class HybridSerializedStates implements StatesDb<SerializedState<LeafDb>>, InitStatesDb<StateEntries> {
+    private readonly spec;
+    private readonly blake2b;
+    private readonly root;
+    private readonly inMemStates;
+    private readonly lmdbValues;
+    private readonly valuesDb;
+    static new({ spec, blake2b, dbPath, readOnly, ephemeral, }: {
+        spec: ChainSpec;
+        blake2b: Blake2b;
+        dbPath: string;
+        readOnly?: boolean;
+        ephemeral?: boolean;
+    }): HybridSerializedStates;
+    private constructor();
+    insertInitialState(headerHash: HeaderHash, entries: StateEntries): Promise<Result<OK, StateUpdateError>>;
+    updateAndSetState(header: HeaderHash, state: SerializedState<LeafDb>, update: Partial<State & ServicesUpdate>): Promise<Result<OK, StateUpdateError>>;
+    getStateRoot(state: SerializedState<LeafDb>): Promise<StateRootHash>;
+    getState(header: HeaderHash): SerializedState<LeafDb> | null;
+    markUnused(header: HeaderHash): void;
+    close(): Promise<void>;
+    /** Write new large values to LMDB in one transaction. */
+    private writeValues;
+    /** Read a value from LMDB. */
+    private readValue;
+}
+//# sourceMappingURL=hybrid-states.d.ts.map

package/packages/jam/database-lmdb/hybrid-states.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"hybrid-states.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/database-lmdb/hybrid-states.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAGlE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AACnD,OAAO,EACL,KAAK,YAAY,EACjB,MAAM,EACN,KAAK,QAAQ,EACb,gBAAgB,EAGjB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAC;AAC/C,OAAO,KAAK,EAAE,cAAc,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAC9D,OAAO,EACL,eAAe,EACf,KAAK,YAAY,EAGlB,MAAM,gCAAgC,CAAC;AAExC,OAAO,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAG9C;;;;;;;GAOG;AACH,qBAAa,sBAAuB,YAAW,QAAQ,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC,EAAE,YAAY,CAAC,YAAY,CAAC;IAwBxG,OAAO,CAAC,QAAQ,CAAC,IAAI;IACrB,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,IAAI;IAzBvB,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAyE;IACrG,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAQ;IAEnC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAW;IAEpC,MAAM,CAAC,GAAG,CAAC,EACT,IAAI,EACJ,OAAO,EACP,MAAM,EACN,QAAQ,EACR,SAAS,GACV,EAAE;QACD,IAAI,EAAE,SAAS,CAAC;QAChB,OAAO,EAAE,OAAO,CAAC;QACjB,MAAM,EAAE,MAAM,CAAC;QACf,QAAQ,CAAC,EAAE,OAAO,CAAC;QACnB,SAAS,CAAC,EAAE,OAAO,CAAC;KACrB;IAKD,OAAO;IASD,kBAAkB,CAAC,UAAU,EAAE,UAAU,EAAE,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC;IAcxG,iBAAiB,CACrB,MAAM,EAAE,UAAU,EAClB,KAAK,EAAE,eAAe,CAAC,MAAM,CAAC,EAC9B,MAAM,EAAE,OAAO,CAAC,KAAK,GAAG,cAAc,CAAC,GACtC,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC;IAkBlC,YAAY,CAAC,KAAK,EAAE,eAAe,CAAC,MAAM,CAAC,GAAG,OAAO,CAAC,aAAa,CAAC;IAI1E,QAAQ,CAAC,MAAM,EAAE,UAAU,GAAG,eAAe,CAAC,MAAM,CAAC,GAAG,IAAI;IAS5D,UAAU,CAAC,MAAM,EAAE,UAAU,GAAG,IAAI;IAM9B,KAAK;IAKX,yDAAyD;YAC3C,WAAW;IAgBzB,8BAA8B;IAC9B,OAAO,CAAC,SAAS;CAOlB"}

package/packages/jam/database-lmdb/hybrid-states.js

@@ -0,0 +1,105 @@
+import { HashDictionary, SortedSet } from "#@typeberry/collections";
+import { LeafDb, StateUpdateError, updateLeafs, } from "#@typeberry/database";
+import { SerializedState, StateEntryUpdateAction, serializeStateUpdate, } from "#@typeberry/state-merkleization";
+import { leafComparator } from "#@typeberry/trie";
+import { OK, Result } from "#@typeberry/utils";
+import { LmdbRoot } from "./root.js";
+/**
+ * Hybrid serialized-states db.
+ *
+ * States (leafs) are kept in-memory, but large values are persisted to lmdb.
+ * Reads go straight to lmdb, which keeps its own page cache.
+ * NOTE: this DB is designed for long fuzzing and to be used with pruning to
+ * keep the heap usage bounded.
+ */
+export class HybridSerializedStates {
+    spec;
+    blake2b;
+    root;
+    inMemStates = HashDictionary.new();
+    lmdbValues;
+    // A single shared values accessor reused by every `LeafDb` we hand out.
+    valuesDb;
+    static new({ spec, blake2b, dbPath, readOnly, ephemeral, }) {
+        const root = LmdbRoot.new(dbPath, readOnly, ephemeral);
+        return new HybridSerializedStates(spec, blake2b, root);
+    }
+    constructor(spec, blake2b, root) {
+        this.spec = spec;
+        this.blake2b = blake2b;
+        this.root = root;
+        this.lmdbValues = this.root.subDb("values");
+        this.valuesDb = { get: (key) => this.readValue(key) };
+    }
+    async insertInitialState(headerHash, entries) {
+        const { values, leafs } = updateLeafs(SortedSet.fromArray(leafComparator, []), this.blake2b, Array.from(entries, (x) => [StateEntryUpdateAction.Insert, x[0], x[1]]));
+        const res = await this.writeValues(values);
+        if (res.isError) {
+            return res;
+        }
+        this.inMemStates.set(headerHash, leafs);
+        return Result.ok(OK);
+    }
+    async updateAndSetState(header, state, update) {
+        const updatedValues = serializeStateUpdate(this.spec, this.blake2b, update);
+        // Clone the leaf set before mutating: the previous state keeps using its own.
+        const newLeafs = SortedSet.fromSortedArray(leafComparator, state.backend.leafs.array);
+        const { values, leafs } = updateLeafs(newLeafs, this.blake2b, updatedValues);
+        const res = await this.writeValues(values);
+        if (res.isError) {
+            // Leave the caller's state untouched: its new leaves would reference
+            // values that never reached disk.
+            return res;
+        }
+        // Re-create the lookup with the shared values accessor only once the new
+        // values are durably written.
+        state.updateBackend(LeafDb.fromLeaves(leafs, this.valuesDb));
+        this.inMemStates.set(header, leafs);
+        return Result.ok(OK);
+    }
+    async getStateRoot(state) {
+        return state.backend.getStateRoot(this.blake2b);
+    }
+    getState(header) {
+        const leafs = this.inMemStates.get(header);
+        if (leafs === undefined) {
+            return null;
+        }
+        const leafDb = LeafDb.fromLeaves(leafs, this.valuesDb);
+        return SerializedState.new(this.spec, this.blake2b, leafDb);
+    }
+    markUnused(header) {
+        // We only remove the state from memory - values are not pruned at all,
+        // but since they are stored on disk we should be safe.
+        this.inMemStates.delete(header);
+    }
+    async close() {
+        await this.lmdbValues.close();
+        await this.root.close();
+    }
+    /** Write new large values to LMDB in one transaction. */
+    async writeValues(values) {
+        if (values.length === 0) {
+            return Result.ok(OK);
+        }
+        try {
+            await this.lmdbValues.transaction(() => {
+                for (const [hash, val] of values) {
+                    this.lmdbValues.put(hash.raw, val.raw);
+                }
+            });
+        }
+        catch (e) {
+            return Result.error(StateUpdateError.Commit, () => `Failed to commit values: ${e}`);
+        }
+        return Result.ok(OK);
+    }
+    /** Read a value from LMDB. */
+    readValue(key) {
+        const val = this.lmdbValues.get(key.raw);
+        if (val === undefined) {
+            throw new Error(`Missing value at key: ${key}`);
+        }
+        return val;
+    }
+}

package/packages/jam/database-lmdb/hybrid-states.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=hybrid-states.test.d.ts.map

package/packages/jam/database-lmdb/hybrid-states.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"hybrid-states.test.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/database-lmdb/hybrid-states.test.ts"],"names":[],"mappings":""}

package/packages/jam/database-lmdb/hybrid-states.test.js

@@ -0,0 +1,88 @@
+// packages/jam/database-lmdb/hybrid-states.test.ts
+import assert from "node:assert";
+import * as fs from "node:fs";
+import { afterEach, before, beforeEach, describe, it } from "node:test";
+import { Bytes, BytesBlob } from "#@typeberry/bytes";
+import { tinyChainSpec } from "#@typeberry/config";
+import { Blake2b, HASH_SIZE } from "#@typeberry/hash";
+import { InMemoryState } from "#@typeberry/state";
+import { StateEntries } from "#@typeberry/state-merkleization";
+import { deepEqual, OK, Result } from "#@typeberry/utils";
+import { HybridSerializedStates } from "./hybrid-states.js";
+let blake2b;
+before(async () => {
+    blake2b = await Blake2b.createHasher();
+});
+function createTempDir(suffix = "hybrid") {
+    return fs.mkdtempSync(`typeberry-${suffix}`);
+}
+describe("Hybrid serialized states", () => {
+    const spec = tinyChainSpec;
+    const headerHash = Bytes.zero(HASH_SIZE).asOpaque();
+    let dbPath = "";
+    beforeEach(() => {
+        dbPath = createTempDir();
+    });
+    afterEach(() => {
+        fs.rmSync(dbPath, { recursive: true });
+    });
+    it("round-trips an initial state through the on-disk values store", async () => {
+        const states = HybridSerializedStates.new({
+            spec,
+            blake2b,
+            dbPath,
+        });
+        try {
+            const empty = InMemoryState.empty(spec);
+            const serialized = StateEntries.serializeInMemory(spec, blake2b, empty);
+            const expectedRoot = serialized.getRootHash(blake2b);
+            const res = await states.insertInitialState(headerHash, serialized);
+            deepEqual(res, Result.ok(OK));
+            const state = states.getState(headerHash);
+            assert.ok(state !== null);
+            const stateRoot = await states.getStateRoot(state);
+            assert.strictEqual(`${stateRoot}`, `${expectedRoot}`);
+            deepEqual(InMemoryState.copyFrom(spec, state, new Map()), empty);
+        }
+        finally {
+            await states.close();
+        }
+    });
+    it("reads large values back from disk", async () => {
+        const states = HybridSerializedStates.new({ spec, blake2b, dbPath });
+        try {
+            // > 32 bytes => stored in the values db (not embedded in the leaf).
+            const big1 = BytesBlob.blobFromString("x".repeat(100));
+            const big2 = BytesBlob.blobFromString("y".repeat(100));
+            const key1 = Bytes.fill(HASH_SIZE, 1).asOpaque();
+            const key2 = Bytes.fill(HASH_SIZE, 2).asOpaque();
+            const entries = StateEntries.fromEntriesUnsafe([
+                [key1, big1],
+                [key2, big2],
+            ]);
+            const res = await states.insertInitialState(headerHash, entries);
+            deepEqual(res, Result.ok(OK));
+            const state = states.getState(headerHash);
+            assert.ok(state !== null);
+            assert.strictEqual(`${state.backend.get(key2)}`, `${big2}`);
+            assert.strictEqual(`${state.backend.get(key1)}`, `${big1}`);
+        }
+        finally {
+            await states.close();
+        }
+    });
+    it("drops the leaf set on markUnused while values stay on disk", async () => {
+        const states = HybridSerializedStates.new({ spec, blake2b, dbPath });
+        try {
+            const empty = InMemoryState.empty(spec);
+            const serialized = StateEntries.serializeInMemory(spec, blake2b, empty);
+            await states.insertInitialState(headerHash, serialized);
+            assert.ok(states.getState(headerHash) !== null);
+            states.markUnused(headerHash);
+            assert.strictEqual(states.getState(headerHash), null);
+        }
+        finally {
+            await states.close();
+        }
+    });
+});

package/packages/jam/database-lmdb/index.d.ts

@@ -1,4 +1,5 @@
 export * from "./blocks.js";
+export * from "./hybrid-states.js";
 export * from "./root.js";
 export * from "./states.js";
 //# sourceMappingURL=index.d.ts.map

package/packages/jam/database-lmdb/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/database-lmdb/index.ts"],"names":[],"mappings":"AAAA,cAAc,aAAa,CAAC;AAC5B,cAAc,WAAW,CAAC;AAC1B,cAAc,aAAa,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/database-lmdb/index.ts"],"names":[],"mappings":"AAAA,cAAc,aAAa,CAAC;AAC5B,cAAc,oBAAoB,CAAC;AACnC,cAAc,WAAW,CAAC;AAC1B,cAAc,aAAa,CAAC"}

package/packages/jam/database-lmdb/index.js

@@ -1,3 +1,4 @@
 export * from "./blocks.js";
+export * from "./hybrid-states.js";
 export * from "./root.js";
 export * from "./states.js";

package/packages/jam/node/main-fuzz.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"main-fuzz.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/node/main-fuzz.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,KAAK,WAAW,EAAmB,MAAM,oBAAoB,CAAC;AACvE,OAAO,EAAE,EAAE,IAAI,MAAM,EAAE,MAAM,uBAAuB,CAAC;AAOrD,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAIjD,MAAM,MAAM,UAAU,GAAG;IACvB,OAAO,EAAE,WAAW,CAAC;IACrB,aAAa,EAAE,SAAS,CAAC;IACzB,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,uBAAuB,EAAE,OAAO,CAAC;CAClC,CAAC;AAOF;;;;;;;;GAQG;AACH,wBAAgB,iBAAiB,CAAC,UAAU,EAAE,MAAM,GAAG,SAAS,GAAG,MAAM,GAAG,SAAS,CASpF;AAED,iFAAiF;AACjF,wBAAsB,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAE5D;AAED,wBAAgB,cAAc;;;;EAM7B;AAED,wBAAsB,QAAQ,CAAC,UAAU,EAAE,UAAU,EAAE,WAAW,EAAE,CAAC,CAAC,EAAE,MAAM,KAAK,MAAM,uBAwGxF"}
+{"version":3,"file":"main-fuzz.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/node/main-fuzz.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,KAAK,WAAW,EAAmB,MAAM,oBAAoB,CAAC;AACvE,OAAO,EAAE,EAAE,IAAI,MAAM,EAAE,MAAM,uBAAuB,CAAC;AAOrD,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAIjD,MAAM,MAAM,UAAU,GAAG;IACvB,OAAO,EAAE,WAAW,CAAC;IACrB,aAAa,EAAE,SAAS,CAAC;IACzB,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,uBAAuB,EAAE,OAAO,CAAC;CAClC,CAAC;AAOF;;;;;;;;GAQG;AACH,wBAAgB,iBAAiB,CAAC,UAAU,EAAE,MAAM,GAAG,SAAS,GAAG,MAAM,GAAG,SAAS,CASpF;AAED,iFAAiF;AACjF,wBAAsB,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAE5D;AAED,wBAAgB,cAAc;;;;EAM7B;AAED,wBAAsB,QAAQ,CAAC,UAAU,EAAE,UAAU,EAAE,WAAW,EAAE,CAAC,CAAC,EAAE,MAAM,KAAK,MAAM,uBAyGxF"}

package/packages/jam/node/main-fuzz.js

@@ -81,8 +81,6 @@ export async function mainFuzz(fuzzConfig, withRelPath) {
                 await finish;
             }
             const buildNode = (databaseBasePath) => {
-                // Enable state/blocks pruning only when running in memory.
-                // For disk backend, we store everything.
                 const isPersistent = databaseBasePath !== undefined;
                 return mainImporter({
                     ...config,
@@ -99,11 +97,14 @@ export async function mainFuzz(fuzzConfig, withRelPath) {
                     network: null,
                 }, withRelPath, {
                     initGenesisFromAncestry: fuzzConfig.initGenesisFromAncestry,
-                    dummyFinalityDepth: isPersistent ? 0 : 10_000,
-                    pruneBlocks: !isPersistent,
+                    // Hybrid keeps leaf sets in RAM, so they must be windowed exactly
+                    // like the in-memory backend; only the large values live on disk.
+                    dummyFinalityDepth: 10_000,
+                    pruneBlocks: true,
                     // The fuzz db is wiped on every reset, so durability is pointless:
-                    // skip fsync + compression to cut the per-block leaf write cost.
+                    // skip fsync + compression to cut the per-block value write cost.
                     ephemeralDb: isPersistent,
+                    stateBackend: isPersistent ? "hybrid" : "lmdb",
                 });
             };
             if (fuzzDbBase !== undefined) {

package/packages/jam/node/main-importer.d.ts

@@ -6,6 +6,8 @@ export type ImporterOptions = {
     pruneBlocks?: boolean;
     /** Open the LMDB database without fsync/compression. Only safe for throwaway dbs (e.g. fuzzing). */
     ephemeralDb?: boolean;
+    /** Persistent backend to use when `databaseBasePath` is set. Defaults to full LMDB. */
+    stateBackend?: "lmdb" | "hybrid";
 };
 export declare function mainImporter(config: JamConfig, withRelPath: (v: string) => string, options?: ImporterOptions): Promise<NodeApi>;
 //# sourceMappingURL=main-importer.d.ts.map

package/packages/jam/node/main-importer.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"main-importer.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/node/main-importer.ts"],"names":[],"mappings":"AAUA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AACjD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAIzC,MAAM,MAAM,eAAe,GAAG;IAC5B,uBAAuB,CAAC,EAAE,OAAO,CAAC;IAClC,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,oGAAoG;IACpG,WAAW,CAAC,EAAE,OAAO,CAAC;CACvB,CAAC;AAEF,wBAAsB,YAAY,CAChC,MAAM,EAAE,SAAS,EACjB,WAAW,EAAE,CAAC,CAAC,EAAE,MAAM,KAAK,MAAM,EAClC,OAAO,GAAE,eAAoB,GAC5B,OAAO,CAAC,OAAO,CAAC,CAiFlB"}
+{"version":3,"file":"main-importer.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/node/main-importer.ts"],"names":[],"mappings":"AAUA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AACjD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAIzC,MAAM,MAAM,eAAe,GAAG;IAC5B,uBAAuB,CAAC,EAAE,OAAO,CAAC;IAClC,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,oGAAoG;IACpG,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,uFAAuF;IACvF,YAAY,CAAC,EAAE,MAAM,GAAG,QAAQ,CAAC;CAClC,CAAC;AAEF,wBAAsB,YAAY,CAChC,MAAM,EAAE,SAAS,EACjB,WAAW,EAAE,CAAC,CAAC,EAAE,MAAM,KAAK,MAAM,EAClC,OAAO,GAAE,eAAoB,GAC5B,OAAO,CAAC,OAAO,CAAC,CA0FlB"}

package/packages/jam/node/main-importer.js

@@ -5,7 +5,7 @@ import { Blake2b, HASH_SIZE } from "#@typeberry/hash";
 import { createImporter, ImporterConfig } from "#@typeberry/importer";
 import { tryAsU16 } from "#@typeberry/numbers";
 import { CURRENT_SUITE, CURRENT_VERSION, Result, resultToString, version } from "#@typeberry/utils";
-import { InMemWorkerConfig, LmdbWorkerConfig } from "#@typeberry/workers-api-node";
+import { HybridWorkerConfig, InMemWorkerConfig, LmdbWorkerConfig } from "#@typeberry/workers-api-node";
 import { getChainSpec, getDatabasePath, initializeDatabase, logger } from "./common.js";
 const zeroHash = Bytes.zero(HASH_SIZE).asOpaque();
 export async function mainImporter(config, withRelPath, options = {}) {
@@ -31,14 +31,23 @@ export async function mainImporter(config, withRelPath, options = {}) {
             blake2b,
             workerParams,
         })
-        : LmdbWorkerConfig.new({
-            nodeName,
-            chainSpec,
-            blake2b,
-            dbPath,
-            workerParams,
-            ephemeral: options.ephemeralDb ?? false,
-        });
+        : options.stateBackend === "hybrid"
+            ? HybridWorkerConfig.new({
+                nodeName,
+                chainSpec,
+                blake2b,
+                dbPath,
+                workerParams,
+                ephemeral: options.ephemeralDb ?? false,
+            })
+            : LmdbWorkerConfig.new({
+                nodeName,
+                chainSpec,
+                blake2b,
+                dbPath,
+                workerParams,
+                ephemeral: options.ephemeralDb ?? false,
+            });
     // Initialize the database with genesis state and block if there isn't one.
     logger.info `🛢️ Opening database at ${dbPath}`;
     const rootDb = workerConfig.openDatabase({ readonly: false });

package/packages/workers/api-node/config.d.ts

@@ -71,4 +71,39 @@ export declare class InMemWorkerConfig<T = undefined> implements WorkerConfig<T,
         readonly: boolean;
     }): RootDb<BlocksDb, SerializedStatesDb>;
 }
+/**
+ * Hybrid worker config for the fuzz target: in-memory blocks and leaf sets,
+ * but large values persisted to LMDB.
+ *
+ * Like `InMemWorkerConfig`, the blocks and leaf sets are shared across the
+ * open/close/reopen dance that genesis init performs, so `openDatabase`
+ * returns the same instances and a no-op close. The LMDB root is opened once
+ * here and closed by `HybridSerializedStates.close()` at importer teardown.
+ *
+ * In-process only: it holds shared mutable state (the in-memory leaf
+ * dictionary) and so is not thread-transferable. The fuzz target runs the
+ * importer in-process via `createImporter`, not in a spawned worker.
+ */
+export declare class HybridWorkerConfig<T = undefined> implements WorkerConfig<T, BlocksDb, SerializedStatesDb> {
+    readonly nodeName: string;
+    readonly chainSpec: ChainSpec;
+    readonly workerParams: T;
+    readonly blake2b: Blake2b;
+    readonly dbPath: string;
+    readonly ephemeral: boolean;
+    static new<T>({ nodeName, chainSpec, workerParams, blake2b, dbPath, ephemeral, }: {
+        nodeName: string;
+        chainSpec: ChainSpec;
+        workerParams: T;
+        blake2b: Blake2b;
+        dbPath: string;
+        ephemeral?: boolean;
+    }): HybridWorkerConfig<T>;
+    private readonly blocks;
+    private readonly states;
+    private constructor();
+    openDatabase(_options?: {
+        readonly: boolean;
+    }): RootDb<BlocksDb, SerializedStatesDb>;
+}
 //# sourceMappingURL=config.d.ts.map

package/packages/workers/api-node/config.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../../../../../packages/workers/api-node/config.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AACvD,OAAO,EAAE,KAAK,MAAM,EAAW,KAAK,MAAM,EAAW,MAAM,kBAAkB,CAAC;AAC9E,OAAO,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AAC9C,OAAO,EACL,KAAK,QAAQ,EAGb,KAAK,MAAM,EACX,KAAK,kBAAkB,EACxB,MAAM,qBAAqB,CAAC;AAE7B,OAAO,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAC;AAC1C,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAC3D,OAAO,EAAE,UAAU,EAAE,KAAK,gBAAgB,EAAE,MAAM,WAAW,CAAC;AAE9D,+EAA+E;AAC/E,qBAAa,gBAAgB,CAAC,CAAC,GAAG,IAAI,CAAE,YAAW,YAAY,CAAC,CAAC,EAAE,QAAQ,EAAE,kBAAkB,CAAC;aAyC5E,QAAQ,EAAE,MAAM;aAChB,SAAS,EAAE,SAAS;aACpB,YAAY,EAAE,CAAC;aACf,MAAM,EAAE,MAAM;aACd,OAAO,EAAE,OAAO;aAChB,KAAK,EAAE,GAAG,CAAC,MAAM,EAAE,UAAU,CAAC;aAI9B,SAAS,EAAE,OAAO;IAjDpC,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,EACZ,QAAQ,EACR,SAAS,EACT,YAAY,EACZ,MAAM,EACN,OAAO,EACP,KAAiB,EACjB,SAAiB,GAClB,EAAE;QACD,QAAQ,EAAE,MAAM,CAAC;QACjB,SAAS,EAAE,SAAS,CAAC;QACrB,YAAY,EAAE,CAAC,CAAC;QAChB,MAAM,EAAE,MAAM,CAAC;QACf,OAAO,EAAE,OAAO,CAAC;QACjB,KAAK,CAAC,EAAE,GAAG,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;QAChC,SAAS,CAAC,EAAE,OAAO,CAAC;KACrB;IAID,6DAA6D;WAChD,gBAAgB,CAAC,CAAC,EAAE,YAAY,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,kBAAkB;IAkBpF,OAAO;IAaP,YAAY,CAAC,OAAO,GAAE;QAAE,QAAQ,EAAE,OAAO,CAAA;KAAuB,GAAG,MAAM,CAAC,QAAQ,EAAE,kBAAkB,CAAC;IAUvG,6DAA6D;IAC7D,gBAAgB,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC,CAAC,GAAG,kBAAkB;CAS7D;AAED,6DAA6D;AAC7D,MAAM,MAAM,kBAAkB,GAAG;IAC/B,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,SAAS,CAAC;IACrB,YAAY,EAAE,UAAU,CAAC;IACzB,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,EAAE,CAAC,MAAM,EAAE,gBAAgB,CAAC,EAAE,CAAC;CAC3C,CAAC;AAEF;;;;;;GAMG;AACH,wBAAgB,kBAAkB,CAAC,MAAM,EAAE,kBAAkB,GAAG,WAAW,EAAE,CAE5E;AAED;;;;GAIG;AACH,qBAAa,iBAAiB,CAAC,CAAC,GAAG,SAAS,CAAE,YAAW,YAAY,CAAC,CAAC,EAAE,QAAQ,EAAE,kBAAkB,CAAC;aAmBlF,QAAQ,EAAE,MAAM;aAChB,SAAS,EAAE,SAAS;aACpB,YAAY,EAAE,CAAC;aACf,OAAO,EAAE,OAAO;IArBlC,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,EACZ,QAAQ,EACR,SAAS,EACT,YAAY,EACZ,OAAO,GACR,EAAE;QACD,QAAQ,EAAE,MAAM,CAAC;QACjB,SAAS,EAAE,SAAS,CAAC;QACrB,YAAY,EAAE,CAAC,CAAC;QAChB,OAAO,EAAE,OAAO,CAAC;KAClB;IAID,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAiB;IACxC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAA2B;IAElD,OAAO;IAUP,YAAY,CAAC,QAAQ,GAAE;QAAE,QAAQ,EAAE,OAAO,CAAA;KAAuB,GAAG,MAAM,CAAC,QAAQ,EAAE,kBAAkB,CAAC;CAQzG"}
+{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../../../../../packages/workers/api-node/config.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AACvD,OAAO,EAAE,KAAK,MAAM,EAAW,KAAK,MAAM,EAAW,MAAM,kBAAkB,CAAC;AAC9E,OAAO,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AAC9C,OAAO,EACL,KAAK,QAAQ,EAGb,KAAK,MAAM,EACX,KAAK,kBAAkB,EACxB,MAAM,qBAAqB,CAAC;AAE7B,OAAO,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAC;AAC1C,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAC3D,OAAO,EAAE,UAAU,EAAE,KAAK,gBAAgB,EAAE,MAAM,WAAW,CAAC;AAE9D,+EAA+E;AAC/E,qBAAa,gBAAgB,CAAC,CAAC,GAAG,IAAI,CAAE,YAAW,YAAY,CAAC,CAAC,EAAE,QAAQ,EAAE,kBAAkB,CAAC;aAyC5E,QAAQ,EAAE,MAAM;aAChB,SAAS,EAAE,SAAS;aACpB,YAAY,EAAE,CAAC;aACf,MAAM,EAAE,MAAM;aACd,OAAO,EAAE,OAAO;aAChB,KAAK,EAAE,GAAG,CAAC,MAAM,EAAE,UAAU,CAAC;aAI9B,SAAS,EAAE,OAAO;IAjDpC,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,EACZ,QAAQ,EACR,SAAS,EACT,YAAY,EACZ,MAAM,EACN,OAAO,EACP,KAAiB,EACjB,SAAiB,GAClB,EAAE;QACD,QAAQ,EAAE,MAAM,CAAC;QACjB,SAAS,EAAE,SAAS,CAAC;QACrB,YAAY,EAAE,CAAC,CAAC;QAChB,MAAM,EAAE,MAAM,CAAC;QACf,OAAO,EAAE,OAAO,CAAC;QACjB,KAAK,CAAC,EAAE,GAAG,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;QAChC,SAAS,CAAC,EAAE,OAAO,CAAC;KACrB;IAID,6DAA6D;WAChD,gBAAgB,CAAC,CAAC,EAAE,YAAY,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,kBAAkB;IAkBpF,OAAO;IAaP,YAAY,CAAC,OAAO,GAAE;QAAE,QAAQ,EAAE,OAAO,CAAA;KAAuB,GAAG,MAAM,CAAC,QAAQ,EAAE,kBAAkB,CAAC;IAUvG,6DAA6D;IAC7D,gBAAgB,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC,CAAC,GAAG,kBAAkB;CAS7D;AAED,6DAA6D;AAC7D,MAAM,MAAM,kBAAkB,GAAG;IAC/B,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,SAAS,CAAC;IACrB,YAAY,EAAE,UAAU,CAAC;IACzB,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,EAAE,CAAC,MAAM,EAAE,gBAAgB,CAAC,EAAE,CAAC;CAC3C,CAAC;AAEF;;;;;;GAMG;AACH,wBAAgB,kBAAkB,CAAC,MAAM,EAAE,kBAAkB,GAAG,WAAW,EAAE,CAE5E;AAED;;;;GAIG;AACH,qBAAa,iBAAiB,CAAC,CAAC,GAAG,SAAS,CAAE,YAAW,YAAY,CAAC,CAAC,EAAE,QAAQ,EAAE,kBAAkB,CAAC;aAmBlF,QAAQ,EAAE,MAAM;aAChB,SAAS,EAAE,SAAS;aACpB,YAAY,EAAE,CAAC;aACf,OAAO,EAAE,OAAO;IArBlC,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,EACZ,QAAQ,EACR,SAAS,EACT,YAAY,EACZ,OAAO,GACR,EAAE;QACD,QAAQ,EAAE,MAAM,CAAC;QACjB,SAAS,EAAE,SAAS,CAAC;QACrB,YAAY,EAAE,CAAC,CAAC;QAChB,OAAO,EAAE,OAAO,CAAC;KAClB;IAID,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAiB;IACxC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAA2B;IAElD,OAAO;IAUP,YAAY,CAAC,QAAQ,GAAE;QAAE,QAAQ,EAAE,OAAO,CAAA;KAAuB,GAAG,MAAM,CAAC,QAAQ,EAAE,kBAAkB,CAAC;CAQzG;AAED;;;;;;;;;;;;GAYG;AACH,qBAAa,kBAAkB,CAAC,CAAC,GAAG,SAAS,CAAE,YAAW,YAAY,CAAC,CAAC,EAAE,QAAQ,EAAE,kBAAkB,CAAC;aAuBnF,QAAQ,EAAE,MAAM;aAChB,SAAS,EAAE,SAAS;aACpB,YAAY,EAAE,CAAC;aACf,OAAO,EAAE,OAAO;aAChB,MAAM,EAAE,MAAM;aACd,SAAS,EAAE,OAAO;IA3BpC,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,EACZ,QAAQ,EACR,SAAS,EACT,YAAY,EACZ,OAAO,EACP,MAAM,EACN,SAAiB,GAClB,EAAE;QACD,QAAQ,EAAE,MAAM,CAAC;QACjB,SAAS,EAAE,SAAS,CAAC;QACrB,YAAY,EAAE,CAAC,CAAC;QAChB,OAAO,EAAE,OAAO,CAAC;QACjB,MAAM,EAAE,MAAM,CAAC;QACf,SAAS,CAAC,EAAE,OAAO,CAAC;KACrB;IAID,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAiB;IACxC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAyB;IAEhD,OAAO;IAkBP,YAAY,CAAC,QAAQ,GAAE;QAAE,QAAQ,EAAE,OAAO,CAAA;KAAuB,GAAG,MAAM,CAAC,QAAQ,EAAE,kBAAkB,CAAC;CASzG"}

package/packages/workers/api-node/config.js

@@ -1,7 +1,7 @@
 import { Decoder, Encoder } from "#@typeberry/codec";
 import { ChainSpec } from "#@typeberry/config";
 import { InMemoryBlocks, InMemorySerializedStates, } from "#@typeberry/database";
-import { LmdbBlocks, LmdbRoot, LmdbStates } from "#@typeberry/database-lmdb";
+import { HybridSerializedStates, LmdbBlocks, LmdbRoot, LmdbStates } from "#@typeberry/database-lmdb";
 import { Blake2b } from "#@typeberry/hash";
 import { ThreadPort } from "./port.js";
 /** A worker config that's usable in node.js and uses LMDB database backend. */
@@ -105,3 +105,54 @@ export class InMemWorkerConfig {
         };
     }
 }
+/**
+ * Hybrid worker config for the fuzz target: in-memory blocks and leaf sets,
+ * but large values persisted to LMDB.
+ *
+ * Like `InMemWorkerConfig`, the blocks and leaf sets are shared across the
+ * open/close/reopen dance that genesis init performs, so `openDatabase`
+ * returns the same instances and a no-op close. The LMDB root is opened once
+ * here and closed by `HybridSerializedStates.close()` at importer teardown.
+ *
+ * In-process only: it holds shared mutable state (the in-memory leaf
+ * dictionary) and so is not thread-transferable. The fuzz target runs the
+ * importer in-process via `createImporter`, not in a spawned worker.
+ */
+export class HybridWorkerConfig {
+    nodeName;
+    chainSpec;
+    workerParams;
+    blake2b;
+    dbPath;
+    ephemeral;
+    static new({ nodeName, chainSpec, workerParams, blake2b, dbPath, ephemeral = false, }) {
+        return new HybridWorkerConfig(nodeName, chainSpec, workerParams, blake2b, dbPath, ephemeral);
+    }
+    blocks;
+    states;
+    constructor(nodeName, chainSpec, workerParams, blake2b, dbPath, ephemeral) {
+        this.nodeName = nodeName;
+        this.chainSpec = chainSpec;
+        this.workerParams = workerParams;
+        this.blake2b = blake2b;
+        this.dbPath = dbPath;
+        this.ephemeral = ephemeral;
+        this.blocks = InMemoryBlocks.new();
+        this.states = HybridSerializedStates.new({
+            spec: this.chainSpec,
+            blake2b: this.blake2b,
+            dbPath: this.dbPath,
+            ephemeral: this.ephemeral,
+            readOnly: false,
+        });
+    }
+    openDatabase(_options = { readonly: true }) {
+        return {
+            getBlocksDb: () => this.blocks,
+            getStatesDb: () => this.states,
+            // Leaf sets and blocks live in memory; the LMDB values store is closed
+            // via states.close() at importer teardown, so this is a no-op.
+            close: async () => { },
+        };
+    }
+}