@typeberry/lib 0.9.0-959889e → 0.9.0-a723180

package/packages/jam/database-lmdb/hybrid-states.test.js

@@ -2,12 +2,14 @@
 import assert from "node:assert";
 import * as fs from "node:fs";
 import { afterEach, before, beforeEach, describe, it } from "node:test";
+import { tryAsServiceId } from "#@typeberry/block";
 import { Bytes, BytesBlob } from "#@typeberry/bytes";
 import { tinyChainSpec } from "#@typeberry/config";
 import { Blake2b, HASH_SIZE } from "#@typeberry/hash";
-import { InMemoryState } from "#@typeberry/state";
+import { InMemoryState, StorageItem, UpdateStorage, } from "#@typeberry/state";
+import { testState } from "#@typeberry/state/test.utils.js";
 import { StateEntries } from "#@typeberry/state-merkleization";
-import { deepEqual, OK, Result } from "#@typeberry/utils";
+import { asOpaqueType, deepEqual, OK, Result } from "#@typeberry/utils";
 import { HybridSerializedStates } from "./hybrid-states.js";
 let blake2b;
 before(async () => {
@@ -86,3 +88,93 @@ describe("Hybrid serialized states", () => {
         }
     });
 });
+function hh(n) {
+    return Bytes.fill(HASH_SIZE, n).asOpaque();
+}
+const storageKey = asOpaqueType(BytesBlob.blobFromString("test-key"));
+/** A state update writing a single large (non-embedded) value under `storageKey`. */
+function storageUpdate(value) {
+    const item = StorageItem.create({ key: storageKey, value: BytesBlob.blobFromString(value) });
+    return {
+        storage: new Map([[tryAsServiceId(0), [UpdateStorage.set({ storage: item })]]]),
+    };
+}
+// > 32 bytes => stored in the values db (not embedded in the leaf).
+const BIG_1 = "a".repeat(100);
+const BIG_2 = "b".repeat(100);
+/** `true` if every value referenced by the state can still be resolved. */
+function canReadFully(state) {
+    if (state === null) {
+        return false;
+    }
+    try {
+        state.backend.intoStateEntries();
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+/** Value removals are queued, so poll for the expected outcome. */
+async function eventually(check, what, timeoutMs = 5_000) {
+    const start = Date.now();
+    while (!check()) {
+        if (Date.now() - start > timeoutMs) {
+            assert.fail(`Timed out waiting for: ${what}`);
+        }
+        await new Promise((resolve) => setTimeout(resolve, 10));
+    }
+}
+describe("Hybrid serialized states value refcounting", () => {
+    const spec = tinyChainSpec;
+    let dbPath = "";
+    beforeEach(() => {
+        dbPath = createTempDir();
+    });
+    afterEach(() => {
+        fs.rmSync(dbPath, { recursive: true });
+    });
+    it("removes a replaced value from disk once the replacement finalizes", async () => {
+        const states = HybridSerializedStates.new({ spec, blake2b, dbPath });
+        try {
+            await states.insertInitialState(hh(0), StateEntries.serializeInMemory(spec, blake2b, testState()));
+            const s0 = states.getState(hh(0));
+            assert.ok(s0 !== null);
+            await states.updateAndSetState(hh(1), s0, storageUpdate(BIG_1));
+            // a handle to the post-1 state, surviving the pruning below
+            const stale1 = states.getState(hh(1));
+            const s1 = states.getState(hh(1));
+            assert.ok(s1 !== null);
+            await states.updateAndSetState(hh(2), s1, storageUpdate(BIG_2));
+            states.commitFinalized([hh(1)]);
+            assert.ok(canReadFully(stale1), "still referenced by the finalized tip");
+            states.commitFinalized([hh(2)]);
+            await eventually(() => !canReadFully(stale1), "replaced value removed from lmdb");
+            assert.ok(canReadFully(states.getState(hh(2))), "the new finalized tip stays fully readable");
+        }
+        finally {
+            await states.close();
+        }
+    });
+    it("collects values of a pruned dead fork and keeps the surviving chain intact", async () => {
+        const states = HybridSerializedStates.new({ spec, blake2b, dbPath });
+        try {
+            await states.insertInitialState(hh(0), StateEntries.serializeInMemory(spec, blake2b, testState()));
+            const s0 = states.getState(hh(0));
+            assert.ok(s0 !== null);
+            await states.updateAndSetState(hh(1), s0, storageUpdate(BIG_1));
+            // a dead fork on top of genesis, inserting a different value
+            const fork = states.getState(hh(0));
+            assert.ok(fork !== null);
+            await states.updateAndSetState(hh(0xaa), fork, storageUpdate(BIG_2));
+            const staleFork = states.getState(hh(0xaa));
+            states.markUnused(hh(0xaa));
+            assert.strictEqual(states.getState(hh(0xaa)), null);
+            await eventually(() => !canReadFully(staleFork), "fork-only value removed from lmdb");
+            assert.ok(canReadFully(states.getState(hh(1))), "surviving chain is unaffected");
+        }
+        finally {
+            await states.close();
+        }
+    });
+});

package/packages/jam/database-lmdb/states.d.ts

@@ -68,6 +68,7 @@ export declare class LmdbStates implements StatesDb<SerializedState<LeafDb>>, In
     updateAndSetState(headerHash: HeaderHash, state: SerializedState<LeafDb>, update: Partial<State & ServicesUpdate>): Promise<Result<OK, StateUpdateError>>;
     getStateRoot(state: SerializedState<LeafDb>): Promise<StateRootHash>;
     getState(root: HeaderHash): SerializedState<LeafDb> | null;
+    commitFinalized(_headers: HeaderHash[]): void;
     markUnused(header: HeaderHash): void;
     diskSizeInBytes(): number | null;
     close(): Promise<void>;

package/packages/jam/database-lmdb/states.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"states.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/database-lmdb/states.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAGlE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AACnD,OAAO,EAAE,KAAK,YAAY,EAAE,MAAM,EAAE,KAAK,QAAQ,EAAE,gBAAgB,EAAe,MAAM,qBAAqB,CAAC;AAC9G,OAAO,KAAK,EAAE,OAAO,EAAiB,MAAM,iBAAiB,CAAC;AAE9D,OAAO,KAAK,EAAE,cAAc,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAC9D,OAAO,KAAK,EAAE,YAAY,EAAY,MAAM,gCAAgC,CAAC;AAC7E,OAAO,EAAE,eAAe,EAAgD,MAAM,gCAAgC,CAAC;AAG/G,OAAO,EAAE,EAAE,EAAE,MAAM,EAAkB,MAAM,kBAAkB,CAAC;AAC9D,OAAO,KAAK,EAAE,QAAQ,EAAS,MAAM,WAAW,CAAC;AAGjD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+CG;AAEH,qBAAa,UAAW,YAAW,QAAQ,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC,EAAE,YAAY,CAAC,YAAY,CAAC;IAS5F,OAAO,CAAC,QAAQ,CAAC,IAAI;IACrB,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,IAAI;IAVvB,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAQ;IAC/B,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAQ;IAE/B,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ;IAI5D,OAAO;IASD,kBAAkB,CACtB,UAAU,EAAE,UAAU,EACtB,eAAe,EAAE,YAAY,GAC5B,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC;YAQ1B,eAAe;IA0BvB,iBAAiB,CACrB,UAAU,EAAE,UAAU,EACtB,KAAK,EAAE,eAAe,CAAC,MAAM,CAAC,EAC9B,MAAM,EAAE,OAAO,CAAC,KAAK,GAAG,cAAc,CAAC,GACtC,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC;IAWlC,YAAY,CAAC,KAAK,EAAE,eAAe,CAAC,MAAM,CAAC,GAAG,OAAO,CAAC,aAAa,CAAC;IAI1E,QAAQ,CAAC,IAAI,EAAE,UAAU,GAAG,eAAe,CAAC,MAAM,CAAC,GAAG,IAAI;IAsB1D,UAAU,CAAC,MAAM,EAAE,UAAU,GAAG,IAAI;IAIpC,eAAe,IAAI,MAAM,GAAG,IAAI;IAI1B,KAAK;CAGZ"}
+{"version":3,"file":"states.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/database-lmdb/states.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAGlE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AACnD,OAAO,EAAE,KAAK,YAAY,EAAE,MAAM,EAAE,KAAK,QAAQ,EAAE,gBAAgB,EAAe,MAAM,qBAAqB,CAAC;AAC9G,OAAO,KAAK,EAAE,OAAO,EAAiB,MAAM,iBAAiB,CAAC;AAE9D,OAAO,KAAK,EAAE,cAAc,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAC9D,OAAO,KAAK,EAAE,YAAY,EAAY,MAAM,gCAAgC,CAAC;AAC7E,OAAO,EAAE,eAAe,EAAgD,MAAM,gCAAgC,CAAC;AAG/G,OAAO,EAAE,EAAE,EAAE,MAAM,EAAkB,MAAM,kBAAkB,CAAC;AAC9D,OAAO,KAAK,EAAE,QAAQ,EAAS,MAAM,WAAW,CAAC;AAGjD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+CG;AAEH,qBAAa,UAAW,YAAW,QAAQ,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC,EAAE,YAAY,CAAC,YAAY,CAAC;IAS5F,OAAO,CAAC,QAAQ,CAAC,IAAI;IACrB,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,IAAI;IAVvB,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAQ;IAC/B,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAQ;IAE/B,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ;IAI5D,OAAO;IASD,kBAAkB,CACtB,UAAU,EAAE,UAAU,EACtB,eAAe,EAAE,YAAY,GAC5B,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC;YAQ1B,eAAe;IA0BvB,iBAAiB,CACrB,UAAU,EAAE,UAAU,EACtB,KAAK,EAAE,eAAe,CAAC,MAAM,CAAC,EAC9B,MAAM,EAAE,OAAO,CAAC,KAAK,GAAG,cAAc,CAAC,GACtC,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC;IAWlC,YAAY,CAAC,KAAK,EAAE,eAAe,CAAC,MAAM,CAAC,GAAG,OAAO,CAAC,aAAa,CAAC;IAI1E,QAAQ,CAAC,IAAI,EAAE,UAAU,GAAG,eAAe,CAAC,MAAM,CAAC,GAAG,IAAI;IAsB1D,eAAe,CAAC,QAAQ,EAAE,UAAU,EAAE,GAAG,IAAI;IAM7C,UAAU,CAAC,MAAM,EAAE,UAAU,GAAG,IAAI;IAIpC,eAAe,IAAI,MAAM,GAAG,IAAI;IAI1B,KAAK;CAGZ"}

package/packages/jam/database-lmdb/states.js

@@ -128,6 +128,11 @@ export class LmdbStates {
         }
         return SerializedState.new(this.spec, this.blake2b, leafDbResult.ok);
     }
+    commitFinalized(_headers) {
+        // Values are never pruned here. This db survives restarts, so refcounting
+        // would need counts persisted (and crash-consistent) alongside the values.
+        // See the TODO above - not implemented until actually needed.
+    }
     markUnused(header) {
         this.states.removeSync(header.raw);
     }

package/packages/jam/node/export.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"export.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/node/export.ts"],"names":[],"mappings":"AAUA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAEjD,wBAAsB,YAAY,CAAC,aAAa,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,WAAW,EAAE,CAAC,CAAC,EAAE,MAAM,KAAK,MAAM,iBA2F9G"}
+{"version":3,"file":"export.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/node/export.ts"],"names":[],"mappings":"AAWA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAEjD,wBAAsB,YAAY,CAAC,aAAa,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,WAAW,EAAE,CAAC,CAAC,EAAE,MAAM,KAAK,MAAM,iBAoG9G"}

package/packages/jam/node/export.js

@@ -3,9 +3,10 @@ import path from "node:path";
 import { Block as BlockCodec } from "#@typeberry/block";
 import { Bytes } from "#@typeberry/bytes";
 import { Encoder } from "#@typeberry/codec";
+import { RegularStateBackend } from "#@typeberry/config-node";
 import { Blake2b, HASH_SIZE } from "#@typeberry/hash";
 import { Logger } from "#@typeberry/logger";
-import { LmdbWorkerConfig } from "#@typeberry/workers-api-node";
+import { FjallWorkerConfig, LmdbWorkerConfig } from "#@typeberry/workers-api-node";
 import { getChainSpec, getDatabasePath } from "./common.js";
 export async function exportBlocks(jamNodeConfig, output, withRelPath) {
     const logger = Logger.new(import.meta.filename, "export");
@@ -29,14 +30,22 @@ export async function exportBlocks(jamNodeConfig, output, withRelPath) {
     const blake2b = await Blake2b.createHasher();
     const chainSpec = getChainSpec(jamNodeConfig.node.flavor);
     const { dbPath } = getDatabasePath(blake2b, jamNodeConfig.nodeName, jamNodeConfig.node.chainSpec.genesisHeader, withRelPath(jamNodeConfig.node.databaseBasePath));
-    const config = LmdbWorkerConfig.new({
-        nodeName: jamNodeConfig.nodeName,
-        chainSpec,
-        blake2b,
-        dbPath,
-        workerParams: undefined,
-    });
-    const rootDb = config.openDatabase();
+    const config = jamNodeConfig.node.stateBackend === RegularStateBackend.Fjall
+        ? FjallWorkerConfig.new({
+            nodeName: jamNodeConfig.nodeName,
+            chainSpec,
+            blake2b,
+            dbPath,
+            workerParams: undefined,
+        })
+        : LmdbWorkerConfig.new({
+            nodeName: jamNodeConfig.nodeName,
+            chainSpec,
+            blake2b,
+            dbPath,
+            workerParams: undefined,
+        });
+    const rootDb = await config.openDatabase();
     const blocks = rootDb.getBlocksDb();
     logger.info `📖 Gathering blocks...`;
     const hashes = [];

package/packages/jam/node/main-fuzz.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"main-fuzz.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/node/main-fuzz.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,KAAK,WAAW,EAAmB,MAAM,oBAAoB,CAAC;AACvE,OAAO,EAAE,EAAE,IAAI,MAAM,EAAE,MAAM,uBAAuB,CAAC;AAIrD,OAAO,EAAE,KAAK,MAAM,EAAoC,MAAM,kBAAkB,CAAC;AAGjF,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAIjD,MAAM,MAAM,UAAU,GAAG;IACvB,OAAO,EAAE,WAAW,CAAC;IACrB,aAAa,EAAE,SAAS,CAAC;IACzB,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,uBAAuB,EAAE,OAAO,CAAC;CAClC,CAAC;AAoBF;;;;;;;;GAQG;AACH,wBAAgB,iBAAiB,CAAC,UAAU,EAAE,MAAM,GAAG,SAAS,GAAG,MAAM,GAAG,SAAS,CASpF;AAED,iFAAiF;AACjF,wBAAsB,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAE5D;AAED,wBAAgB,cAAc;;;;EAM7B;AAED,wBAAsB,QAAQ,CAAC,UAAU,EAAE,UAAU,EAAE,WAAW,EAAE,CAAC,CAAC,EAAE,MAAM,KAAK,MAAM,GAAG,OAAO,CAAC;IAAE,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC,CAsKrH"}
+{"version":3,"file":"main-fuzz.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/node/main-fuzz.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,KAAK,WAAW,EAAmB,MAAM,oBAAoB,CAAC;AACvE,OAAO,EAAE,EAAE,IAAI,MAAM,EAAE,MAAM,uBAAuB,CAAC;AAIrD,OAAO,EAAE,KAAK,MAAM,EAAoC,MAAM,kBAAkB,CAAC;AAGjF,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAIjD,MAAM,MAAM,UAAU,GAAG;IACvB,OAAO,EAAE,WAAW,CAAC;IACrB,aAAa,EAAE,SAAS,CAAC;IACzB,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,uBAAuB,EAAE,OAAO,CAAC;CAClC,CAAC;AAoBF;;;;;;;;GAQG;AACH,wBAAgB,iBAAiB,CAAC,UAAU,EAAE,MAAM,GAAG,SAAS,GAAG,MAAM,GAAG,SAAS,CASpF;AAED,iFAAiF;AACjF,wBAAsB,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAE5D;AAED,wBAAgB,cAAc;;;;EAM7B;AAED,wBAAsB,QAAQ,CAAC,UAAU,EAAE,UAAU,EAAE,WAAW,EAAE,CAAC,CAAC,EAAE,MAAM,KAAK,MAAM,GAAG,OAAO,CAAC;IAAE,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC,CAgLrH"}

package/packages/jam/node/main-fuzz.js

@@ -78,6 +78,7 @@ export async function mainFuzz(fuzzConfig, withRelPath) {
     // Set when close() starts. Guards resetState so a fuzz command arriving
     // mid-shutdown can't build a fresh node that close() then orphans.
     let isClosing = false;
+    let activeReset = null;
     const chainSpec = getChainSpec(config.node.flavor);
     const closeFuzzTarget = startFuzzTarget(fuzzConfig.version, fuzzConfig.socket, {
         ...getFuzzDetails(),
@@ -101,84 +102,94 @@ export async function mainFuzz(fuzzConfig, withRelPath) {
             }
             return runningNode.getStateEntries(hash);
         },
-        resetState: async (header, state, ancestry) => {
-            if (isClosing) {
-                return Bytes.zero(HASH_SIZE).asOpaque();
-            }
-            if (runningNode !== null) {
-                const finish = runningNode.close();
-                runningNode = null;
-                await finish;
-            }
-            const buildNode = (databaseBasePath) => {
-                const isPersistent = databaseBasePath !== undefined;
-                return mainImporter({
-                    ...config,
-                    node: {
-                        ...config.node,
-                        databaseBasePath,
-                        chainSpec: {
-                            ...config.node.chainSpec,
-                            genesisHeader: Encoder.encodeObject(Header.Codec, header, chainSpec),
-                            genesisState: new Map(state),
+        resetState: (header, state, ancestry) => {
+            const reset = (async () => {
+                if (isClosing) {
+                    return Bytes.zero(HASH_SIZE).asOpaque();
+                }
+                if (runningNode !== null) {
+                    const finish = runningNode.close();
+                    runningNode = null;
+                    await finish;
+                }
+                const buildNode = (databaseBasePath) => {
+                    const isPersistent = databaseBasePath !== undefined;
+                    return mainImporter({
+                        ...config,
+                        node: {
+                            ...config.node,
+                            databaseBasePath,
+                            chainSpec: {
+                                ...config.node.chainSpec,
+                                genesisHeader: Encoder.encodeObject(Header.Codec, header, chainSpec),
+                                genesisState: new Map(state),
+                            },
                         },
-                    },
-                    ancestry,
-                    network: null,
-                }, withRelPath, {
-                    initGenesisFromAncestry: fuzzConfig.initGenesisFromAncestry,
-                    // Hybrid keeps leaf sets in RAM, so they must be windowed exactly
-                    // like the in-memory backend; only the large values live on disk.
-                    dummyFinalityDepth: 20,
-                    pruneBlocks: true,
-                    // The on-disk fuzz db is throwaway (we wipe it), so open it ephemeral and
-                    // skip the fsync, we do not need durability here. On full spec ephemeral
-                    // also turns on compression further down, so the big values do not grow the
-                    // db too much. Tiny stays uncompressed, its db is small and speed matters more.
-                    ephemeral: isPersistent,
-                    stateBackend: isPersistent ? hybridStateBackend : "lmdb",
-                    // Reuse the session keyspace (fjall-hybrid only, other backends
-                    // ignore it). Nothing to pass for the in-memory fallback.
-                    sharedFjallSession: isPersistent ? (fjallSession ?? undefined) : undefined,
-                });
-            };
-            if (fuzzDbBase !== undefined) {
-                try {
-                    if (hybridStateBackend === FUZZ_DB_FJALL) {
-                        // fjall-hybrid: open the values keyspace once and reuse it on every
-                        // reset. The values partition is content-addressed and immutable, so
-                        // it is fine that values pile up across resets, the unreferenced ones
-                        // just sit there. `initializeDatabase` decides whether the db is
-                        // already initialized from the in-memory blocks, which we rebuild on
-                        // every reset, not from the values store, so reusing it does not
-                        // resume the previous run.
-                        if (fjallSession === null) {
-                            // Start from a clean slate once, then keep the keyspace open.
+                        ancestry,
+                        network: null,
+                    }, withRelPath, {
+                        initGenesisFromAncestry: fuzzConfig.initGenesisFromAncestry,
+                        // Hybrid keeps leaf sets in RAM, so they must be windowed exactly
+                        // like the in-memory backend; only the large values live on disk.
+                        dummyFinalityDepth: 20,
+                        pruneBlocks: true,
+                        // The on-disk fuzz db is throwaway (we wipe it), so open it ephemeral and
+                        // skip the fsync, we do not need durability here. On full spec ephemeral
+                        // also turns on compression further down, so the big values do not grow the
+                        // db too much. Tiny stays uncompressed, its db is small and speed matters more.
+                        ephemeral: isPersistent,
+                        stateBackend: isPersistent ? hybridStateBackend : "lmdb",
+                        // Reuse the session keyspace (fjall-hybrid only, other backends
+                        // ignore it). Nothing to pass for the in-memory fallback.
+                        sharedFjallSession: isPersistent ? (fjallSession ?? undefined) : undefined,
+                    });
+                };
+                if (fuzzDbBase !== undefined) {
+                    try {
+                        if (hybridStateBackend === FUZZ_DB_FJALL) {
+                            // fjall-hybrid: open the values keyspace once and reuse it on every
+                            // reset. The values partition is content-addressed and immutable, so
+                            // it is fine that values pile up across resets, the unreferenced ones
+                            // just sit there. `initializeDatabase` decides whether the db is
+                            // already initialized from the in-memory blocks, which we rebuild on
+                            // every reset, not from the values store, so reusing it does not
+                            // resume the previous run.
+                            if (fjallSession === null) {
+                                // Start from a clean slate once, then keep the keyspace open.
+                                await wipeFuzzDb(fuzzDbBase);
+                                fjallSession = await FjallValuesSession.open(`${withRelPath(fuzzDbBase)}/${FUZZ_FJALL_VALUES_SUBDIR}`, {
+                                    ephemeral: true,
+                                    cacheSizeBytes: FUZZ_FJALL_CACHE_BYTES,
+                                });
+                                logger.info `🗄️ Opened reusable fjall values session at ${withRelPath(fuzzDbBase)}/${FUZZ_FJALL_VALUES_SUBDIR}`;
+                            }
+                        }
+                        else {
+                            // lmdb-hybrid: keep the old behaviour, wipe and reopen on every
+                            // reset. A fresh db each reset makes `initializeDatabase` set up
+                            // genesis again instead of resuming the previous run.
                             await wipeFuzzDb(fuzzDbBase);
-                            fjallSession = await FjallValuesSession.open(`${withRelPath(fuzzDbBase)}/${FUZZ_FJALL_VALUES_SUBDIR}`, {
-                                ephemeral: true,
-                                cacheSizeBytes: FUZZ_FJALL_CACHE_BYTES,
-                            });
-                            logger.info `🗄️ Opened reusable fjall values session at ${withRelPath(fuzzDbBase)}/${FUZZ_FJALL_VALUES_SUBDIR}`;
                         }
+                        runningNode = await buildNode(fuzzDbBase);
+                        return await runningNode.getBestStateRootHash();
                     }
-                    else {
-                        // lmdb-hybrid: keep the old behaviour, wipe and reopen on every
-                        // reset. A fresh db each reset makes `initializeDatabase` set up
-                        // genesis again instead of resuming the previous run.
-                        await wipeFuzzDb(fuzzDbBase);
+                    catch (e) {
+                        // A partially-opened db may leak on failure; acceptable for this degraded fallback (proper cleanup belongs in mainImporter).
+                        logger.warn `Failed to open persistent fuzz db at ${fuzzDbBase}, falling back to in-memory: ${e}`;
+                        runningNode = null;
                     }
-                    runningNode = await buildNode(fuzzDbBase);
-                    return await runningNode.getBestStateRootHash();
                 }
-                catch (e) {
-                    // A partially-opened db may leak on failure; acceptable for this degraded fallback (proper cleanup belongs in mainImporter).
-                    logger.warn `Failed to open persistent fuzz db at ${fuzzDbBase}, falling back to in-memory: ${e}`;
-                    runningNode = null;
+                runningNode = await buildNode(undefined);
+                return await runningNode.getBestStateRootHash();
+            })();
+            activeReset = reset;
+            const clearActiveReset = () => {
+                if (activeReset === reset) {
+                    activeReset = null;
                 }
-            }
-            runningNode = await buildNode(undefined);
-            return await runningNode.getBestStateRootHash();
+            };
+            reset.then(clearActiveReset, clearActiveReset);
+            return reset;
         },
     });
     return {
@@ -189,6 +200,9 @@ export async function mainFuzz(fuzzConfig, withRelPath) {
             // Drain the active session (flush + close DB). Swallow errors so a
             // failing close still lets the process exit 0; the db is wiped next.
             // The node references the shared fjall session, so it must close first.
+            if (activeReset !== null) {
+                await activeReset.catch((e) => logger.error `Error waiting for fuzz reset: ${e}`);
+            }
             if (runningNode !== null) {
                 const node = runningNode;
                 runningNode = null;

package/packages/jam/node/main-importer.d.ts

@@ -1,7 +1,7 @@
 import { type FjallValuesSession } from "#@typeberry/workers-api-node";
 import type { JamConfig } from "./jam-config.js";
 import type { NodeApi } from "./main.js";
-export type StateBackend = "lmdb" | "lmdb-hybrid" | "fjall-hybrid";
+export type StateBackend = "lmdb" | "fjall" | "lmdb-hybrid" | "fjall-hybrid";
 export type ImporterOptions = {
     initGenesisFromAncestry?: boolean;
     dummyFinalityDepth?: number;
@@ -9,7 +9,9 @@ export type ImporterOptions = {
     /** Open the database without fsync/compression. Only safe for throwaway dbs (e.g. fuzzing). */
     ephemeral?: boolean;
     /**
-     * Persistent backend used when `databaseBasePath` is set. Defaults to full LMDB.
+     * Persistent backend used when `databaseBasePath` is set. Defaults to config's backend (fjall unless overridden).
+     *
+     * lmdb and lmdb-hybrid are deprecated and retained as explicit fallbacks.
      */
     stateBackend?: StateBackend;
     /**

package/packages/jam/node/main-importer.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"main-importer.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/node/main-importer.ts"],"names":[],"mappings":"AASA,OAAO,EACL,KAAK,kBAAkB,EAIxB,MAAM,6BAA6B,CAAC;AAErC,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AACjD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAIzC,MAAM,MAAM,YAAY,GAAG,MAAM,GAAG,aAAa,GAAG,cAAc,CAAC;AAEnE,MAAM,MAAM,eAAe,GAAG;IAC5B,uBAAuB,CAAC,EAAE,OAAO,CAAC;IAClC,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,+FAA+F;IAC/F,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB;;OAEG;IACH,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B;;;;OAIG;IACH,kBAAkB,CAAC,EAAE,kBAAkB,CAAC;CACzC,CAAC;AAEF,wBAAsB,YAAY,CAChC,MAAM,EAAE,SAAS,EACjB,WAAW,EAAE,CAAC,CAAC,EAAE,MAAM,KAAK,MAAM,EAClC,OAAO,GAAE,eAAoB,GAC5B,OAAO,CAAC,OAAO,CAAC,CAsGlB"}
+{"version":3,"file":"main-importer.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/node/main-importer.ts"],"names":[],"mappings":"AASA,OAAO,EACL,KAAK,kBAAkB,EAKxB,MAAM,6BAA6B,CAAC;AAErC,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AACjD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAIzC,MAAM,MAAM,YAAY,GAAG,MAAM,GAAG,OAAO,GAAG,aAAa,GAAG,cAAc,CAAC;AAE7E,MAAM,MAAM,eAAe,GAAG;IAC5B,uBAAuB,CAAC,EAAE,OAAO,CAAC;IAClC,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,+FAA+F;IAC/F,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB;;;;OAIG;IACH,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B;;;;OAIG;IACH,kBAAkB,CAAC,EAAE,kBAAkB,CAAC;CACzC,CAAC;AAEF,wBAAsB,YAAY,CAChC,MAAM,EAAE,SAAS,EACjB,WAAW,EAAE,CAAC,CAAC,EAAE,MAAM,KAAK,MAAM,EAClC,OAAO,GAAE,eAAoB,GAC5B,OAAO,CAAC,OAAO,CAAC,CAqHlB"}

package/packages/jam/node/main-importer.js

@@ -1,12 +1,12 @@
 import { Bytes } from "#@typeberry/bytes";
 import { PvmBackend } from "#@typeberry/config";
-import { KnownChainSpec } from "#@typeberry/config-node";
+import { KnownChainSpec, RegularStateBackend } from "#@typeberry/config-node";
 import { bandersnatch, initWasm } from "#@typeberry/crypto";
 import { Blake2b, HASH_SIZE } from "#@typeberry/hash";
 import { createImporter, ImporterConfig } from "#@typeberry/importer";
 import { tryAsU16 } from "#@typeberry/numbers";
 import { CURRENT_SUITE, CURRENT_VERSION, Result, resultToString, version } from "#@typeberry/utils";
-import { HybridWorkerConfig, InMemWorkerConfig, LmdbWorkerConfig, } from "#@typeberry/workers-api-node";
+import { FjallWorkerConfig, HybridWorkerConfig, InMemWorkerConfig, LmdbWorkerConfig, } from "#@typeberry/workers-api-node";
 import { getChainSpec, getDatabasePath, initializeDatabase, logger } from "./common.js";
 const zeroHash = Bytes.zero(HASH_SIZE).asOpaque();
 export async function mainImporter(config, withRelPath, options = {}) {
@@ -18,8 +18,13 @@ export async function mainImporter(config, withRelPath, options = {}) {
     logger.info `🐇 Bandersnatch ${bandesnatchNative.isOk ? "native 🚀" : `using wasm: ${bandesnatchNative.error}`}`;
     // Single source of truth for the states db backend: drives both the log line
     // below and the worker config picked further down.
-    const dbBackend = config.node.databaseBasePath === undefined ? "in-memory" : (options.stateBackend ?? "lmdb");
+    const dbBackend = config.node.databaseBasePath === undefined
+        ? "in-memory"
+        : (options.stateBackend ?? config.node.stateBackend ?? RegularStateBackend.Fjall);
     logger.info `🗄️ States DB: ${dbBackend}.`;
+    if (dbBackend === "lmdb" || dbBackend === "lmdb-hybrid") {
+        logger.warn `🗄️ The ${dbBackend} state backend is deprecated. Use fjall unless you need a temporary fallback.`;
+    }
     const chainSpec = getChainSpec(config.node.flavor);
     const blake2b = await Blake2b.createHasher();
     const nodeName = config.nodeName;
@@ -51,17 +56,26 @@ export async function mainImporter(config, withRelPath, options = {}) {
                 backend: dbBackend === "lmdb-hybrid" ? "lmdb" : "fjall",
                 sharedFjallSession: options.sharedFjallSession,
             })
-            : LmdbWorkerConfig.new({
-                nodeName,
-                chainSpec,
-                blake2b,
-                dbPath,
-                workerParams,
-                ephemeral,
-            });
+            : dbBackend === "fjall"
+                ? FjallWorkerConfig.new({
+                    nodeName,
+                    chainSpec,
+                    blake2b,
+                    dbPath,
+                    workerParams,
+                    ephemeral,
+                })
+                : LmdbWorkerConfig.new({
+                    nodeName,
+                    chainSpec,
+                    blake2b,
+                    dbPath,
+                    workerParams,
+                    ephemeral,
+                });
     // Initialize the database with genesis state and block if there isn't one.
     logger.info `🛢️ Opening database at ${dbPath}`;
-    const rootDb = workerConfig.openDatabase({ readonly: false });
+    const rootDb = await workerConfig.openDatabase({ readonly: false });
     await initializeDatabase(chainSpec, blake2b, genesisHeaderHash, rootDb, config.node.chainSpec, config.ancestry, {
         initGenesisFromAncestry: options.initGenesisFromAncestry,
     });

package/packages/jam/node/main.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"main.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/node/main.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,SAAS,EAAE,UAAU,EAAc,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAEzF,OAAO,EAAE,KAAK,SAAS,EAAc,MAAM,mBAAmB,CAAC;AAe/D,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,gCAAgC,CAAC;AACnE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,sBAAsB,CAAC;AACtD,OAAO,EAAkC,MAAM,EAAW,MAAM,kBAAkB,CAAC;AAKnF,OAAO,KAAK,EAAE,SAAS,EAAiB,MAAM,iBAAiB,CAAC;AAWhE,MAAM,MAAM,OAAO,GAAG;IACpB,SAAS,EAAE,SAAS,CAAC;IACrB,eAAe,CAAC,IAAI,EAAE,UAAU,GAAG,OAAO,CAAC,YAAY,GAAG,IAAI,CAAC,CAAC;IAChE,WAAW,CAAC,KAAK,EAAE,SAAS,GAAG,OAAO,CAAC,MAAM,CAAC,aAAa,EAAE,MAAM,CAAC,CAAC,CAAC;IACtE,oBAAoB,IAAI,OAAO,CAAC,aAAa,CAAC,CAAC;IAC/C,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CACxB,CAAC;AAEF,wBAAsB,IAAI,CACxB,MAAM,EAAE,SAAS,EACjB,WAAW,EAAE,CAAC,CAAC,EAAE,MAAM,KAAK,MAAM,EAClC,SAAS,EAAE,SAAS,GAAG,IAAI,GAC1B,OAAO,CAAC,OAAO,CAAC,CA2KlB"}
+{"version":3,"file":"main.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/node/main.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,SAAS,EAAE,UAAU,EAAc,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAEzF,OAAO,EAAE,KAAK,SAAS,EAAc,MAAM,mBAAmB,CAAC;AAgB/D,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,gCAAgC,CAAC;AACnE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,sBAAsB,CAAC;AACtD,OAAO,EAAkC,MAAM,EAAW,MAAM,kBAAkB,CAAC;AAYnF,OAAO,KAAK,EAAE,SAAS,EAAiB,MAAM,iBAAiB,CAAC;AAWhE,MAAM,MAAM,OAAO,GAAG;IACpB,SAAS,EAAE,SAAS,CAAC;IACrB,eAAe,CAAC,IAAI,EAAE,UAAU,GAAG,OAAO,CAAC,YAAY,GAAG,IAAI,CAAC,CAAC;IAChE,WAAW,CAAC,KAAK,EAAE,SAAS,GAAG,OAAO,CAAC,MAAM,CAAC,aAAa,EAAE,MAAM,CAAC,CAAC,CAAC;IACtE,oBAAoB,IAAI,OAAO,CAAC,aAAa,CAAC,CAAC;IAC/C,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CACxB,CAAC;AAEF,wBAAsB,IAAI,CACxB,MAAM,EAAE,SAAS,EACjB,WAAW,EAAE,CAAC,CAAC,EAAE,MAAM,KAAK,MAAM,EAClC,SAAS,EAAE,SAAS,GAAG,IAAI,GAC1B,OAAO,CAAC,OAAO,CAAC,CA2LlB"}

package/packages/jam/node/main.js

@@ -1,6 +1,7 @@
 import { isMainThread } from "node:worker_threads";
 import { AUTHORSHIP_NETWORK_PORT } from "#@typeberry/comms-authorship-network";
 import { PvmBackend } from "#@typeberry/config";
+import { RegularStateBackend } from "#@typeberry/config-node";
 import { initWasm } from "#@typeberry/crypto";
 import { deriveBandersnatchSecretKey, deriveEd25519SecretKey, trivialSeed, } from "#@typeberry/crypto/key-derivation.js";
 import { Blake2b } from "#@typeberry/hash";
@@ -10,7 +11,7 @@ import { Listener } from "#@typeberry/listener";
 import { tryAsU16, tryAsU32 } from "#@typeberry/numbers";
 import { CURRENT_SUITE, CURRENT_VERSION, Result, version } from "#@typeberry/utils";
 import { DirectPort, DirectWorkerConfig } from "#@typeberry/workers-api";
-import { InMemWorkerConfig, LmdbWorkerConfig, logHostEnvironment, ThreadPort } from "#@typeberry/workers-api-node";
+import { FjallWorkerConfig, InMemWorkerConfig, LmdbWorkerConfig, logHostEnvironment, ThreadPort, } from "#@typeberry/workers-api-node";
 import { getChainSpec, getDatabasePath, initializeDatabase, logger } from "./common.js";
 import { initializeExtensions } from "./extensions.js";
 import * as metrics from "./metrics.js";
@@ -29,9 +30,12 @@ export async function main(config, withRelPath, telemetry) {
     const blake2b = await Blake2b.createHasher();
     const nodeName = config.nodeName;
     const isInMemory = config.node.databaseBasePath === undefined;
-    logger.info `🗄️ States DB: ${isInMemory ? "in-memory" : "lmdb"}.`;
+    logger.info `🗄️ States DB: ${isInMemory ? "in-memory" : config.node.stateBackend}.`;
+    if (!isInMemory && config.node.stateBackend === RegularStateBackend.Lmdb) {
+        logger.warn `🗄️ The lmdb state backend is deprecated. Use state_backend="fjall" unless you need a temporary fallback.`;
+    }
     const { dbPath, genesisHeaderHash } = getDatabasePath(blake2b, nodeName, config.node.chainSpec.genesisHeader, withRelPath(config.node.databaseBasePath ?? "<in-memory>"));
-    const baseConfig = { nodeName, chainSpec, blake2b, dbPath };
+    const baseConfig = { nodeName, chainSpec, blake2b, dbPath, stateBackend: config.node.stateBackend };
     const importerParams = {
         ...baseConfig,
         workerParams: ImporterConfig.create({
@@ -42,16 +46,29 @@ export async function main(config, withRelPath, telemetry) {
     };
     const importerConfig = isInMemory
         ? { isInMemory, config: InMemWorkerConfig.new(importerParams) }
-        : { isInMemory, config: LmdbWorkerConfig.new(importerParams) };
+        : { isInMemory, config: createPersistentWorkerConfig(importerParams) };
     // Initialize the database with genesis state and block if there isn't one.
     logger.info `🛢️ Opening database at ${dbPath}`;
-    const rootDb = importerConfig.config.openDatabase({ readonly: false });
-    await initializeDatabase(chainSpec, blake2b, genesisHeaderHash, rootDb, config.node.chainSpec, config.ancestry);
-    // NOTE [ToDr] even though, we should be closing the database here,
-    // it seems that opening it in the main thread for writing, and later
-    // in the importer thread, causes issues. Everything works fine though,
-    // if we DO NOT close the database (I guess it's process-shared?)
-    // await rootDb.close();
+    const rootDb = await importerConfig.config.openDatabase({ readonly: false });
+    try {
+        await initializeDatabase(chainSpec, blake2b, genesisHeaderHash, rootDb, config.node.chainSpec, config.ancestry);
+    }
+    catch (e) {
+        try {
+            await rootDb.close();
+        }
+        catch (closeError) {
+            logger.warn `Failed to close database after initialization error: ${closeError}`;
+        }
+        throw e;
+    }
+    // fjall-js shares the engine explicitly and requires every handle to close.
+    // Keep lmdb's historical main-thread handle open until shutdown.
+    let mainRootDb = rootDb;
+    if (!importerConfig.isInMemory && config.node.stateBackend === RegularStateBackend.Fjall) {
+        await rootDb.close();
+        mainRootDb = null;
+    }
     // Start block importer
     let importer;
     let closeImporter;
@@ -138,8 +155,10 @@ export async function main(config, withRelPath, telemetry) {
             await closeImporter();
             logger.log `[main] ☠️  Closing the extensions`;
             closeExtensions();
-            logger.log `[main] 🛢️ Closing the database`;
-            await rootDb.close();
+            if (mainRootDb !== null) {
+                logger.log `[main] 🛢️ Closing the database`;
+                await mainRootDb.close();
+            }
             logger.log `[main] 📳 Closing telemetry`;
             await telemetry?.close();
             logger.info `[main] ✅ Done.`;
@@ -167,7 +186,7 @@ const initAuthorship = async (importer, isAuthoring, isFastForward, params, base
             statesDb: params.rootDb.getStatesDb(),
             workerParams,
         }), params.networkingPort)
-        : await spawnBlockGeneratorWorker(LmdbWorkerConfig.new({
+        : await spawnBlockGeneratorWorker(createPersistentWorkerConfig({
             ...baseConfig,
             workerParams,
             ports: new Map([[AUTHORSHIP_NETWORK_PORT, params.networkingPort]]),
@@ -205,7 +224,7 @@ const initNetwork = async (importer, params, baseConfig, genesisHeaderHash, netw
             statesDb: params.rootDb.getStatesDb(),
             workerParams: networkingConfig,
         }), params.authorshipPort)
-        : await spawnNetworkWorker(LmdbWorkerConfig.new({
+        : await spawnNetworkWorker(createPersistentWorkerConfig({
             ...baseConfig,
             workerParams: networkingConfig,
             ports: new Map([[AUTHORSHIP_NETWORK_PORT, params.authorshipPort]]),
@@ -222,3 +241,11 @@ const initNetwork = async (importer, params, baseConfig, genesisHeaderHash, netw
     });
     return { closeNetwork: finish, networkApi: network, networkWorker: worker };
 };
+function createPersistentWorkerConfig({ stateBackend, ...params }) {
+    switch (stateBackend) {
+        case RegularStateBackend.Fjall:
+            return FjallWorkerConfig.new(params);
+        case RegularStateBackend.Lmdb:
+            return LmdbWorkerConfig.new(params);
+    }
+}