@typeberry/lib 0.8.4 → 0.9.0-1c3f157

package/packages/jam/jamnp-s/tasks/ticket-distribution.test.js

@@ -8,7 +8,8 @@ import { tinyChainSpec } from "#@typeberry/config";
 import { BANDERSNATCH_PROOF_BYTES } from "#@typeberry/crypto";
 import { Logger } from "#@typeberry/logger";
 import { createTestPeerPair, MockNetwork } from "#@typeberry/networking/testing.js";
-import { OK } from "#@typeberry/utils";
+import { AcceptTicketsValidator, ValidationError } from "#@typeberry/ticket-pool";
+import { OK, Result } from "#@typeberry/utils";
 import { Connections } from "../peers.js";
 import { StreamManager } from "../stream-manager.js";
 import { TicketDistributionTask } from "./ticket-distribution.js";
@@ -34,6 +35,9 @@ describe("TicketDistributionTask", () => {
         const receivedTickets = [];
         // Use real TicketDistributionTask
         const ticketTask = TicketDistributionTask.start(streamManager, connections, tinyChainSpec);
+        // Default validator accepts every ticket so the test asserts purely on distribution
+        // behaviour. Tests that exercise the rejection path overwrite this.
+        ticketTask.setTicketValidator(new AcceptTicketsValidator());
         // Intercept received tickets by wrapping onTicketReceived behavior
         // The task already adds received tickets to pending queue via addTicket,
         // so we can track them by checking the pending queue growth or by
@@ -217,7 +221,7 @@ describe("TicketDistributionTask", () => {
         assert.strictEqual(peer2.receivedTickets.length, 1);
         assert.deepStrictEqual(peer2.receivedTickets[0].ticket, ticket);
     });
-    it("should NOT redistribute ticket if validation callback returns false", async () => {
+    it("should NOT redistribute ticket if validator rejects", async () => {
         const self = await init("self");
         const peer1 = await init("peer1");
         const peer2 = await init("peer2");
@@ -225,35 +229,53 @@ describe("TicketDistributionTask", () => {
         self.openConnection(peer2);
         await tick();
         // Validation always rejects
-        self.ticketTask.setOnTicketReceived(async () => false);
+        self.ticketTask.setTicketValidator({
+            validate: async () => Result.error(ValidationError.InvalidProof, () => "rejected"),
+        });
         const ticket = createTestTicket(0);
         peer1.ticketTask.addTicket(TEST_EPOCH, ticket);
         peer1.ticketTask.maintainDistribution();
         await tick();
-        // self.addTicket was NOT called (callback returned false), so nothing to redistribute
+        // self.addTicket was NOT called (validator rejected), so nothing to redistribute
         assert.strictEqual(self.receivedTickets.length, 0);
         self.ticketTask.maintainDistribution();
         await tick();
         assert.strictEqual(peer2.receivedTickets.length, 0);
     });
-    it("should redistribute ticket if validation callback returns true", async () => {
+    it("should redistribute ticket if validator accepts", async () => {
         const self = await init("self");
         const peer1 = await init("peer1");
         const peer2 = await init("peer2");
         self.openConnection(peer1);
         self.openConnection(peer2);
         await tick();
-        // Validation always accepts
-        self.ticketTask.setOnTicketReceived(async () => true);
+        // Default init() already wires an AcceptTicketsValidator
         const ticket = createTestTicket(0);
         peer1.ticketTask.addTicket(TEST_EPOCH, ticket);
         peer1.ticketTask.maintainDistribution();
         await tick();
-        // self.addTicket WAS called (callback returned true)
+        // self.addTicket WAS called
         assert.strictEqual(self.receivedTickets.length, 1);
         self.ticketTask.maintainDistribution();
         await tick();
         assert.strictEqual(peer2.receivedTickets.length, 1);
         assert.deepStrictEqual(peer2.receivedTickets[0].ticket, ticket);
     });
+    it("replacePool overwrites the redistribution pool", async () => {
+        const self = await init("self");
+        const peer1 = await init("peer1");
+        self.openConnection(peer1);
+        await tick();
+        // Locally added tickets first
+        self.ticketTask.addTicket(TEST_EPOCH, createTestTicket(0));
+        self.ticketTask.addTicket(TEST_EPOCH, createTestTicket(1));
+        // Pool dump replaces with a different set
+        const dump = [createTestTicket(2), createTestTicket(3)];
+        self.ticketTask.replacePool(TEST_EPOCH, dump);
+        self.ticketTask.maintainDistribution();
+        await tick();
+        assert.strictEqual(peer1.receivedTickets.length, 2);
+        assert.deepStrictEqual(peer1.receivedTickets[0].ticket, dump[0]);
+        assert.deepStrictEqual(peer1.receivedTickets[1].ticket, dump[1]);
+    });
 });

package/packages/jam/node/export.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"export.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/node/export.ts"],"names":[],"mappings":"AAUA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAEjD,wBAAsB,YAAY,CAAC,aAAa,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,WAAW,EAAE,CAAC,CAAC,EAAE,MAAM,KAAK,MAAM,iBA2F9G"}
+{"version":3,"file":"export.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/node/export.ts"],"names":[],"mappings":"AAWA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAEjD,wBAAsB,YAAY,CAAC,aAAa,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,WAAW,EAAE,CAAC,CAAC,EAAE,MAAM,KAAK,MAAM,iBAoG9G"}

package/packages/jam/node/export.js

@@ -3,9 +3,10 @@ import path from "node:path";
 import { Block as BlockCodec } from "#@typeberry/block";
 import { Bytes } from "#@typeberry/bytes";
 import { Encoder } from "#@typeberry/codec";
+import { RegularStateBackend } from "#@typeberry/config-node";
 import { Blake2b, HASH_SIZE } from "#@typeberry/hash";
 import { Logger } from "#@typeberry/logger";
-import { LmdbWorkerConfig } from "#@typeberry/workers-api-node";
+import { FjallWorkerConfig, LmdbWorkerConfig } from "#@typeberry/workers-api-node";
 import { getChainSpec, getDatabasePath } from "./common.js";
 export async function exportBlocks(jamNodeConfig, output, withRelPath) {
     const logger = Logger.new(import.meta.filename, "export");
@@ -29,14 +30,22 @@ export async function exportBlocks(jamNodeConfig, output, withRelPath) {
     const blake2b = await Blake2b.createHasher();
     const chainSpec = getChainSpec(jamNodeConfig.node.flavor);
     const { dbPath } = getDatabasePath(blake2b, jamNodeConfig.nodeName, jamNodeConfig.node.chainSpec.genesisHeader, withRelPath(jamNodeConfig.node.databaseBasePath));
-    const config = LmdbWorkerConfig.new({
-        nodeName: jamNodeConfig.nodeName,
-        chainSpec,
-        blake2b,
-        dbPath,
-        workerParams: undefined,
-    });
-    const rootDb = config.openDatabase();
+    const config = jamNodeConfig.node.stateBackend === RegularStateBackend.Fjall
+        ? FjallWorkerConfig.new({
+            nodeName: jamNodeConfig.nodeName,
+            chainSpec,
+            blake2b,
+            dbPath,
+            workerParams: undefined,
+        })
+        : LmdbWorkerConfig.new({
+            nodeName: jamNodeConfig.nodeName,
+            chainSpec,
+            blake2b,
+            dbPath,
+            workerParams: undefined,
+        });
+    const rootDb = await config.openDatabase();
     const blocks = rootDb.getBlocksDb();
     logger.info `📖 Gathering blocks...`;
     const hashes = [];

package/packages/jam/node/main-fuzz.d.ts

@@ -1,5 +1,6 @@
 import { type FuzzVersion } from "#@typeberry/ext-ipc";
 import { v1 as fuzzV1 } from "#@typeberry/fuzz-proto";
+import { type Closer } from "#@typeberry/utils";
 import type { JamConfig } from "./jam-config.js";
 export type FuzzConfig = {
     version: FuzzVersion;
@@ -24,5 +25,7 @@ export declare function getFuzzDetails(): {
     nodeVersion: fuzzV1.Version;
     gpVersion: fuzzV1.Version;
 };
-export declare function mainFuzz(fuzzConfig: FuzzConfig, withRelPath: (v: string) => string): Promise<() => void>;
+export declare function mainFuzz(fuzzConfig: FuzzConfig, withRelPath: (v: string) => string): Promise<{
+    close: Closer;
+}>;
 //# sourceMappingURL=main-fuzz.d.ts.map

package/packages/jam/node/main-fuzz.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"main-fuzz.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/node/main-fuzz.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,KAAK,WAAW,EAAmB,MAAM,oBAAoB,CAAC;AACvE,OAAO,EAAE,EAAE,IAAI,MAAM,EAAE,MAAM,uBAAuB,CAAC;AAOrD,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAIjD,MAAM,MAAM,UAAU,GAAG;IACvB,OAAO,EAAE,WAAW,CAAC;IACrB,aAAa,EAAE,SAAS,CAAC;IACzB,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,uBAAuB,EAAE,OAAO,CAAC;CAClC,CAAC;AAOF;;;;;;;;GAQG;AACH,wBAAgB,iBAAiB,CAAC,UAAU,EAAE,MAAM,GAAG,SAAS,GAAG,MAAM,GAAG,SAAS,CASpF;AAED,iFAAiF;AACjF,wBAAsB,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAE5D;AAED,wBAAgB,cAAc;;;;EAM7B;AAED,wBAAsB,QAAQ,CAAC,UAAU,EAAE,UAAU,EAAE,WAAW,EAAE,CAAC,CAAC,EAAE,MAAM,KAAK,MAAM,uBA2GxF"}
+{"version":3,"file":"main-fuzz.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/node/main-fuzz.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,KAAK,WAAW,EAAmB,MAAM,oBAAoB,CAAC;AACvE,OAAO,EAAE,EAAE,IAAI,MAAM,EAAE,MAAM,uBAAuB,CAAC;AAIrD,OAAO,EAAE,KAAK,MAAM,EAAoC,MAAM,kBAAkB,CAAC;AAGjF,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAIjD,MAAM,MAAM,UAAU,GAAG;IACvB,OAAO,EAAE,WAAW,CAAC;IACrB,aAAa,EAAE,SAAS,CAAC;IACzB,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,uBAAuB,EAAE,OAAO,CAAC;CAClC,CAAC;AAoBF;;;;;;;;GAQG;AACH,wBAAgB,iBAAiB,CAAC,UAAU,EAAE,MAAM,GAAG,SAAS,GAAG,MAAM,GAAG,SAAS,CASpF;AAED,iFAAiF;AACjF,wBAAsB,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAE5D;AAED,wBAAgB,cAAc;;;;EAM7B;AAED,wBAAsB,QAAQ,CAAC,UAAU,EAAE,UAAU,EAAE,WAAW,EAAE,CAAC,CAAC,EAAE,MAAM,KAAK,MAAM,GAAG,OAAO,CAAC;IAAE,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC,CAgLrH"}

package/packages/jam/node/main-fuzz.js

@@ -8,12 +8,23 @@ import { v1 as fuzzV1 } from "#@typeberry/fuzz-proto";
 import { HASH_SIZE } from "#@typeberry/hash";
 import { Logger } from "#@typeberry/logger";
 import { CURRENT_VERSION, Result, version } from "#@typeberry/utils";
-import { logHostEnvironment } from "#@typeberry/workers-api-node";
+import { FjallValuesSession, logHostEnvironment } from "#@typeberry/workers-api-node";
 import { getChainSpec } from "./common.js";
 import { mainImporter } from "./main-importer.js";
 const logger = Logger.new(import.meta.filename, "fuzztarget");
 /** Dedicated subdirectory under the configured base path that the fuzzer owns and wipes. */
 const FUZZ_DB_SUBDIR = "typeberry-fuzz-db";
+const FUZZ_DB_FJALL = "fjall-hybrid";
+const FUZZ_DB_LMDB = "lmdb-hybrid";
+const FUZZ_DB_OPTIONS = [FUZZ_DB_FJALL, FUZZ_DB_LMDB];
+/** Subdirectory (under the fuzzer's db dir) holding the reused fjall values keyspace. */
+const FUZZ_FJALL_VALUES_SUBDIR = "values-session";
+/**
+ * Size of the fjall block-cache for the fuzz session. Values pile up across
+ * resets (for fjall we do not wipe between them), so this cache is what keeps
+ * the resident memory bounded.
+ */
+const FUZZ_FJALL_CACHE_BYTES = 128 * 1024 * 1024;
 /**
  * Resolve the directory the fuzzer should use for its on-disk database, or
  * `undefined` for an in-memory database. The dedicated `FUZZ_DB_SUBDIR` is
@@ -50,7 +61,24 @@ export async function mainFuzz(fuzzConfig, withRelPath) {
     logHostEnvironment(logger);
     const { jamNodeConfig: config } = fuzzConfig;
     const fuzzDbBase = resolveFuzzDbBase(config.node.databaseBasePath);
+    const rawFuzzDb = process.env.JAM_FUZZ_DB?.trim() ?? "";
+    // Using experimental fjall-hybrid by default, with an option to test lmdb as well.
+    const hybridStateBackend = rawFuzzDb === "" ? FUZZ_DB_FJALL : rawFuzzDb;
+    if (!isValidStateBackend(hybridStateBackend)) {
+        throw new Error(`JAM_FUZZ_DB must be one of: ${FUZZ_DB_OPTIONS} (got: "${rawFuzzDb}").`);
+    }
+    if (fuzzDbBase !== undefined) {
+        logger.info `🗄️ Fuzz persistent backend: ${hybridStateBackend}.`;
+    }
     let runningNode = null;
+    // The fjall values keyspace is opened once per fuzz session and reused on
+    // every reset, because opening it is the slow part. Only the in-memory blocks
+    // and leaf sets are rebuilt for each vector. fjall-hybrid only.
+    let fjallSession = null;
+    // Set when close() starts. Guards resetState so a fuzz command arriving
+    // mid-shutdown can't build a fresh node that close() then orphans.
+    let isClosing = false;
+    let activeReset = null;
     const chainSpec = getChainSpec(config.node.flavor);
     const closeFuzzTarget = startFuzzTarget(fuzzConfig.version, fuzzConfig.socket, {
         ...getFuzzDetails(),
@@ -74,65 +102,124 @@ export async function mainFuzz(fuzzConfig, withRelPath) {
             }
             return runningNode.getStateEntries(hash);
         },
-        resetState: async (header, state, ancestry) => {
+        resetState: (header, state, ancestry) => {
+            const reset = (async () => {
+                if (isClosing) {
+                    return Bytes.zero(HASH_SIZE).asOpaque();
+                }
+                if (runningNode !== null) {
+                    const finish = runningNode.close();
+                    runningNode = null;
+                    await finish;
+                }
+                const buildNode = (databaseBasePath) => {
+                    const isPersistent = databaseBasePath !== undefined;
+                    return mainImporter({
+                        ...config,
+                        node: {
+                            ...config.node,
+                            databaseBasePath,
+                            chainSpec: {
+                                ...config.node.chainSpec,
+                                genesisHeader: Encoder.encodeObject(Header.Codec, header, chainSpec),
+                                genesisState: new Map(state),
+                            },
+                        },
+                        ancestry,
+                        network: null,
+                    }, withRelPath, {
+                        initGenesisFromAncestry: fuzzConfig.initGenesisFromAncestry,
+                        // Hybrid keeps leaf sets in RAM, so they must be windowed exactly
+                        // like the in-memory backend; only the large values live on disk.
+                        dummyFinalityDepth: 20,
+                        pruneBlocks: true,
+                        // The on-disk fuzz db is throwaway (we wipe it), so open it ephemeral and
+                        // skip the fsync, we do not need durability here. On full spec ephemeral
+                        // also turns on compression further down, so the big values do not grow the
+                        // db too much. Tiny stays uncompressed, its db is small and speed matters more.
+                        ephemeral: isPersistent,
+                        stateBackend: isPersistent ? hybridStateBackend : "lmdb",
+                        // Reuse the session keyspace (fjall-hybrid only, other backends
+                        // ignore it). Nothing to pass for the in-memory fallback.
+                        sharedFjallSession: isPersistent ? (fjallSession ?? undefined) : undefined,
+                    });
+                };
+                if (fuzzDbBase !== undefined) {
+                    try {
+                        if (hybridStateBackend === FUZZ_DB_FJALL) {
+                            // fjall-hybrid: open the values keyspace once and reuse it on every
+                            // reset. The values partition is content-addressed and immutable, so
+                            // it is fine that values pile up across resets, the unreferenced ones
+                            // just sit there. `initializeDatabase` decides whether the db is
+                            // already initialized from the in-memory blocks, which we rebuild on
+                            // every reset, not from the values store, so reusing it does not
+                            // resume the previous run.
+                            if (fjallSession === null) {
+                                // Start from a clean slate once, then keep the keyspace open.
+                                await wipeFuzzDb(fuzzDbBase);
+                                fjallSession = await FjallValuesSession.open(`${withRelPath(fuzzDbBase)}/${FUZZ_FJALL_VALUES_SUBDIR}`, {
+                                    ephemeral: true,
+                                    cacheSizeBytes: FUZZ_FJALL_CACHE_BYTES,
+                                });
+                                logger.info `🗄️ Opened reusable fjall values session at ${withRelPath(fuzzDbBase)}/${FUZZ_FJALL_VALUES_SUBDIR}`;
+                            }
+                        }
+                        else {
+                            // lmdb-hybrid: keep the old behaviour, wipe and reopen on every
+                            // reset. A fresh db each reset makes `initializeDatabase` set up
+                            // genesis again instead of resuming the previous run.
+                            await wipeFuzzDb(fuzzDbBase);
+                        }
+                        runningNode = await buildNode(fuzzDbBase);
+                        return await runningNode.getBestStateRootHash();
+                    }
+                    catch (e) {
+                        // A partially-opened db may leak on failure; acceptable for this degraded fallback (proper cleanup belongs in mainImporter).
+                        logger.warn `Failed to open persistent fuzz db at ${fuzzDbBase}, falling back to in-memory: ${e}`;
+                        runningNode = null;
+                    }
+                }
+                runningNode = await buildNode(undefined);
+                return await runningNode.getBestStateRootHash();
+            })();
+            activeReset = reset;
+            const clearActiveReset = () => {
+                if (activeReset === reset) {
+                    activeReset = null;
+                }
+            };
+            reset.then(clearActiveReset, clearActiveReset);
+            return reset;
+        },
+    });
+    return {
+        close: async () => {
+            isClosing = true;
+            // Stop accepting connections + unlink the socket.
+            closeFuzzTarget();
+            // Drain the active session (flush + close DB). Swallow errors so a
+            // failing close still lets the process exit 0; the db is wiped next.
+            // The node references the shared fjall session, so it must close first.
+            if (activeReset !== null) {
+                await activeReset.catch((e) => logger.error `Error waiting for fuzz reset: ${e}`);
+            }
             if (runningNode !== null) {
-                const finish = runningNode.close();
+                const node = runningNode;
                 runningNode = null;
-                await finish;
+                await node.close().catch((e) => logger.error `Error closing fuzz node: ${e}`);
+            }
+            // Release the reused fjall values keyspace before wiping its files.
+            if (fjallSession !== null) {
+                const session = fjallSession;
+                fjallSession = null;
+                await session.close().catch((e) => logger.error `Error closing fjall session: ${e}`);
             }
-            const buildNode = (databaseBasePath) => {
-                const isPersistent = databaseBasePath !== undefined;
-                return mainImporter({
-                    ...config,
-                    node: {
-                        ...config.node,
-                        databaseBasePath,
-                        chainSpec: {
-                            ...config.node.chainSpec,
-                            genesisHeader: Encoder.encodeObject(Header.Codec, header, chainSpec),
-                            genesisState: new Map(state),
-                        },
-                    },
-                    ancestry,
-                    network: null,
-                }, withRelPath, {
-                    initGenesisFromAncestry: fuzzConfig.initGenesisFromAncestry,
-                    // Hybrid keeps leaf sets in RAM, so they must be windowed exactly
-                    // like the in-memory backend; only the large values live on disk.
-                    dummyFinalityDepth: 20,
-                    pruneBlocks: true,
-                    // Long full-spec sessions accumulate a large, never-pruned values db.
-                    // Syncing lets the OS reclaim dirty mmap pages, and compression (full
-                    // spec only, where values are big) bounds its on-disk/page-cache size.
-                    // Tiny stays uncompressed since its db is small and speed matters more.
-                    ephemeral: isPersistent,
-                    stateBackend: isPersistent ? "hybrid" : "lmdb",
-                });
-            };
             if (fuzzDbBase !== undefined) {
-                // Each reset starts a fresh session from the genesis the fuzzer just sent,
-                // so the on-disk db must be empty: otherwise initializeDatabase sees an
-                // already-initialized db and silently resumes the previous run's state.
-                await wipeFuzzDb(fuzzDbBase);
-                try {
-                    runningNode = await buildNode(fuzzDbBase);
-                    return await runningNode.getBestStateRootHash();
-                }
-                catch (e) {
-                    // A partially-opened db may leak on failure; acceptable for this degraded fallback (proper cleanup belongs in mainImporter).
-                    logger.warn `Failed to open persistent fuzz db at ${fuzzDbBase}, falling back to in-memory: ${e}`;
-                    runningNode = null;
-                }
+                await wipeFuzzDb(fuzzDbBase).catch(() => { });
             }
-            runningNode = await buildNode(undefined);
-            return await runningNode.getBestStateRootHash();
         },
-    });
-    return () => {
-        closeFuzzTarget();
-        if (fuzzDbBase !== undefined) {
-            // best-effort cleanup on shutdown; ignore failures (dir may already be gone).
-            wipeFuzzDb(fuzzDbBase).catch(() => { });
-        }
     };
 }
+function isValidStateBackend(val) {
+    return FUZZ_DB_OPTIONS.indexOf(val) !== -1;
+}

package/packages/jam/node/main-importer.d.ts

@@ -1,13 +1,25 @@
+import { type FjallValuesSession } from "#@typeberry/workers-api-node";
 import type { JamConfig } from "./jam-config.js";
 import type { NodeApi } from "./main.js";
+export type StateBackend = "lmdb" | "fjall" | "lmdb-hybrid" | "fjall-hybrid";
 export type ImporterOptions = {
     initGenesisFromAncestry?: boolean;
     dummyFinalityDepth?: number;
     pruneBlocks?: boolean;
-    /** Open the LMDB database without fsync/compression. Only safe for throwaway dbs (e.g. fuzzing). */
+    /** Open the database without fsync/compression. Only safe for throwaway dbs (e.g. fuzzing). */
     ephemeral?: boolean;
-    /** Persistent backend to use when `databaseBasePath` is set. Defaults to full LMDB. */
-    stateBackend?: "lmdb" | "hybrid";
+    /**
+     * Persistent backend used when `databaseBasePath` is set. Defaults to config's backend (fjall unless overridden).
+     *
+     * lmdb and lmdb-hybrid are deprecated and retained as explicit fallbacks.
+     */
+    stateBackend?: StateBackend;
+    /**
+     * Reuse an already-open fjall values session instead of opening a fresh
+     * keyspace. Only used when `stateBackend === "fjall-hybrid"`. The fuzz target
+     * opens one per run and reuses it across resets.
+     */
+    sharedFjallSession?: FjallValuesSession;
 };
 export declare function mainImporter(config: JamConfig, withRelPath: (v: string) => string, options?: ImporterOptions): Promise<NodeApi>;
 //# sourceMappingURL=main-importer.d.ts.map

package/packages/jam/node/main-importer.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"main-importer.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/node/main-importer.ts"],"names":[],"mappings":"AAWA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AACjD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAIzC,MAAM,MAAM,eAAe,GAAG;IAC5B,uBAAuB,CAAC,EAAE,OAAO,CAAC;IAClC,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,oGAAoG;IACpG,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,uFAAuF;IACvF,YAAY,CAAC,EAAE,MAAM,GAAG,QAAQ,CAAC;CAClC,CAAC;AAEF,wBAAsB,YAAY,CAChC,MAAM,EAAE,SAAS,EACjB,WAAW,EAAE,CAAC,CAAC,EAAE,MAAM,KAAK,MAAM,EAClC,OAAO,GAAE,eAAoB,GAC5B,OAAO,CAAC,OAAO,CAAC,CAoGlB"}
+{"version":3,"file":"main-importer.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/node/main-importer.ts"],"names":[],"mappings":"AASA,OAAO,EACL,KAAK,kBAAkB,EAKxB,MAAM,6BAA6B,CAAC;AAErC,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AACjD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAIzC,MAAM,MAAM,YAAY,GAAG,MAAM,GAAG,OAAO,GAAG,aAAa,GAAG,cAAc,CAAC;AAE7E,MAAM,MAAM,eAAe,GAAG;IAC5B,uBAAuB,CAAC,EAAE,OAAO,CAAC;IAClC,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,+FAA+F;IAC/F,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB;;;;OAIG;IACH,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B;;;;OAIG;IACH,kBAAkB,CAAC,EAAE,kBAAkB,CAAC;CACzC,CAAC;AAEF,wBAAsB,YAAY,CAChC,MAAM,EAAE,SAAS,EACjB,WAAW,EAAE,CAAC,CAAC,EAAE,MAAM,KAAK,MAAM,EAClC,OAAO,GAAE,eAAoB,GAC5B,OAAO,CAAC,OAAO,CAAC,CAqHlB"}

package/packages/jam/node/main-importer.js

@@ -1,12 +1,12 @@
 import { Bytes } from "#@typeberry/bytes";
 import { PvmBackend } from "#@typeberry/config";
-import { KnownChainSpec } from "#@typeberry/config-node";
+import { KnownChainSpec, RegularStateBackend } from "#@typeberry/config-node";
 import { bandersnatch, initWasm } from "#@typeberry/crypto";
 import { Blake2b, HASH_SIZE } from "#@typeberry/hash";
 import { createImporter, ImporterConfig } from "#@typeberry/importer";
 import { tryAsU16 } from "#@typeberry/numbers";
 import { CURRENT_SUITE, CURRENT_VERSION, Result, resultToString, version } from "#@typeberry/utils";
-import { HybridWorkerConfig, InMemWorkerConfig, LmdbWorkerConfig } from "#@typeberry/workers-api-node";
+import { FjallWorkerConfig, HybridWorkerConfig, InMemWorkerConfig, LmdbWorkerConfig, } from "#@typeberry/workers-api-node";
 import { getChainSpec, getDatabasePath, initializeDatabase, logger } from "./common.js";
 const zeroHash = Bytes.zero(HASH_SIZE).asOpaque();
 export async function mainImporter(config, withRelPath, options = {}) {
@@ -18,8 +18,13 @@ export async function mainImporter(config, withRelPath, options = {}) {
     logger.info `🐇 Bandersnatch ${bandesnatchNative.isOk ? "native 🚀" : `using wasm: ${bandesnatchNative.error}`}`;
     // Single source of truth for the states db backend: drives both the log line
     // below and the worker config picked further down.
-    const dbBackend = config.node.databaseBasePath === undefined ? "in-memory" : (options.stateBackend ?? "lmdb");
+    const dbBackend = config.node.databaseBasePath === undefined
+        ? "in-memory"
+        : (options.stateBackend ?? config.node.stateBackend ?? RegularStateBackend.Fjall);
     logger.info `🗄️ States DB: ${dbBackend}.`;
+    if (dbBackend === "lmdb" || dbBackend === "lmdb-hybrid") {
+        logger.warn `🗄️ The ${dbBackend} state backend is deprecated. Use fjall unless you need a temporary fallback.`;
+    }
     const chainSpec = getChainSpec(config.node.flavor);
     const blake2b = await Blake2b.createHasher();
     const nodeName = config.nodeName;
@@ -39,8 +44,8 @@ export async function mainImporter(config, withRelPath, options = {}) {
             blake2b,
             workerParams,
         })
-        : dbBackend === "hybrid"
-            ? HybridWorkerConfig.new({
+        : dbBackend === "lmdb-hybrid" || dbBackend === "fjall-hybrid"
+            ? await HybridWorkerConfig.new({
                 nodeName,
                 chainSpec,
                 blake2b,
@@ -48,18 +53,29 @@ export async function mainImporter(config, withRelPath, options = {}) {
                 workerParams,
                 ephemeral,
                 compression,
+                backend: dbBackend === "lmdb-hybrid" ? "lmdb" : "fjall",
+                sharedFjallSession: options.sharedFjallSession,
             })
-            : LmdbWorkerConfig.new({
-                nodeName,
-                chainSpec,
-                blake2b,
-                dbPath,
-                workerParams,
-                ephemeral,
-            });
+            : dbBackend === "fjall"
+                ? FjallWorkerConfig.new({
+                    nodeName,
+                    chainSpec,
+                    blake2b,
+                    dbPath,
+                    workerParams,
+                    ephemeral,
+                })
+                : LmdbWorkerConfig.new({
+                    nodeName,
+                    chainSpec,
+                    blake2b,
+                    dbPath,
+                    workerParams,
+                    ephemeral,
+                });
     // Initialize the database with genesis state and block if there isn't one.
     logger.info `🛢️ Opening database at ${dbPath}`;
-    const rootDb = workerConfig.openDatabase({ readonly: false });
+    const rootDb = await workerConfig.openDatabase({ readonly: false });
     await initializeDatabase(chainSpec, blake2b, genesisHeaderHash, rootDb, config.node.chainSpec, config.ancestry, {
         initGenesisFromAncestry: options.initGenesisFromAncestry,
     });

package/packages/jam/node/main.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"main.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/node/main.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,SAAS,EAAE,UAAU,EAAc,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAEzF,OAAO,EAAE,KAAK,SAAS,EAAc,MAAM,mBAAmB,CAAC;AAe/D,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,gCAAgC,CAAC;AACnE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,sBAAsB,CAAC;AACtD,OAAO,EAAkC,MAAM,EAAW,MAAM,kBAAkB,CAAC;AAKnF,OAAO,KAAK,EAAE,SAAS,EAAiB,MAAM,iBAAiB,CAAC;AAWhE,MAAM,MAAM,OAAO,GAAG;IACpB,SAAS,EAAE,SAAS,CAAC;IACrB,eAAe,CAAC,IAAI,EAAE,UAAU,GAAG,OAAO,CAAC,YAAY,GAAG,IAAI,CAAC,CAAC;IAChE,WAAW,CAAC,KAAK,EAAE,SAAS,GAAG,OAAO,CAAC,MAAM,CAAC,aAAa,EAAE,MAAM,CAAC,CAAC,CAAC;IACtE,oBAAoB,IAAI,OAAO,CAAC,aAAa,CAAC,CAAC;IAC/C,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CACxB,CAAC;AAEF,wBAAsB,IAAI,CACxB,MAAM,EAAE,SAAS,EACjB,WAAW,EAAE,CAAC,CAAC,EAAE,MAAM,KAAK,MAAM,EAClC,SAAS,EAAE,SAAS,GAAG,IAAI,GAC1B,OAAO,CAAC,OAAO,CAAC,CA2KlB"}
+{"version":3,"file":"main.d.ts","sourceRoot":"","sources":["../../../../../packages/jam/node/main.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,SAAS,EAAE,UAAU,EAAc,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAEzF,OAAO,EAAE,KAAK,SAAS,EAAc,MAAM,mBAAmB,CAAC;AAgB/D,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,gCAAgC,CAAC;AACnE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,sBAAsB,CAAC;AACtD,OAAO,EAAkC,MAAM,EAAW,MAAM,kBAAkB,CAAC;AAYnF,OAAO,KAAK,EAAE,SAAS,EAAiB,MAAM,iBAAiB,CAAC;AAWhE,MAAM,MAAM,OAAO,GAAG;IACpB,SAAS,EAAE,SAAS,CAAC;IACrB,eAAe,CAAC,IAAI,EAAE,UAAU,GAAG,OAAO,CAAC,YAAY,GAAG,IAAI,CAAC,CAAC;IAChE,WAAW,CAAC,KAAK,EAAE,SAAS,GAAG,OAAO,CAAC,MAAM,CAAC,aAAa,EAAE,MAAM,CAAC,CAAC,CAAC;IACtE,oBAAoB,IAAI,OAAO,CAAC,aAAa,CAAC,CAAC;IAC/C,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CACxB,CAAC;AAEF,wBAAsB,IAAI,CACxB,MAAM,EAAE,SAAS,EACjB,WAAW,EAAE,CAAC,CAAC,EAAE,MAAM,KAAK,MAAM,EAClC,SAAS,EAAE,SAAS,GAAG,IAAI,GAC1B,OAAO,CAAC,OAAO,CAAC,CA2LlB"}

package/packages/jam/node/main.js

@@ -1,6 +1,7 @@
 import { isMainThread } from "node:worker_threads";
 import { AUTHORSHIP_NETWORK_PORT } from "#@typeberry/comms-authorship-network";
 import { PvmBackend } from "#@typeberry/config";
+import { RegularStateBackend } from "#@typeberry/config-node";
 import { initWasm } from "#@typeberry/crypto";
 import { deriveBandersnatchSecretKey, deriveEd25519SecretKey, trivialSeed, } from "#@typeberry/crypto/key-derivation.js";
 import { Blake2b } from "#@typeberry/hash";
@@ -10,7 +11,7 @@ import { Listener } from "#@typeberry/listener";
 import { tryAsU16, tryAsU32 } from "#@typeberry/numbers";
 import { CURRENT_SUITE, CURRENT_VERSION, Result, version } from "#@typeberry/utils";
 import { DirectPort, DirectWorkerConfig } from "#@typeberry/workers-api";
-import { InMemWorkerConfig, LmdbWorkerConfig, logHostEnvironment, ThreadPort } from "#@typeberry/workers-api-node";
+import { FjallWorkerConfig, InMemWorkerConfig, LmdbWorkerConfig, logHostEnvironment, ThreadPort, } from "#@typeberry/workers-api-node";
 import { getChainSpec, getDatabasePath, initializeDatabase, logger } from "./common.js";
 import { initializeExtensions } from "./extensions.js";
 import * as metrics from "./metrics.js";
@@ -29,9 +30,12 @@ export async function main(config, withRelPath, telemetry) {
     const blake2b = await Blake2b.createHasher();
     const nodeName = config.nodeName;
     const isInMemory = config.node.databaseBasePath === undefined;
-    logger.info `🗄️ States DB: ${isInMemory ? "in-memory" : "lmdb"}.`;
+    logger.info `🗄️ States DB: ${isInMemory ? "in-memory" : config.node.stateBackend}.`;
+    if (!isInMemory && config.node.stateBackend === RegularStateBackend.Lmdb) {
+        logger.warn `🗄️ The lmdb state backend is deprecated. Use state_backend="fjall" unless you need a temporary fallback.`;
+    }
     const { dbPath, genesisHeaderHash } = getDatabasePath(blake2b, nodeName, config.node.chainSpec.genesisHeader, withRelPath(config.node.databaseBasePath ?? "<in-memory>"));
-    const baseConfig = { nodeName, chainSpec, blake2b, dbPath };
+    const baseConfig = { nodeName, chainSpec, blake2b, dbPath, stateBackend: config.node.stateBackend };
     const importerParams = {
         ...baseConfig,
         workerParams: ImporterConfig.create({
@@ -42,16 +46,29 @@ export async function main(config, withRelPath, telemetry) {
     };
     const importerConfig = isInMemory
         ? { isInMemory, config: InMemWorkerConfig.new(importerParams) }
-        : { isInMemory, config: LmdbWorkerConfig.new(importerParams) };
+        : { isInMemory, config: createPersistentWorkerConfig(importerParams) };
     // Initialize the database with genesis state and block if there isn't one.
     logger.info `🛢️ Opening database at ${dbPath}`;
-    const rootDb = importerConfig.config.openDatabase({ readonly: false });
-    await initializeDatabase(chainSpec, blake2b, genesisHeaderHash, rootDb, config.node.chainSpec, config.ancestry);
-    // NOTE [ToDr] even though, we should be closing the database here,
-    // it seems that opening it in the main thread for writing, and later
-    // in the importer thread, causes issues. Everything works fine though,
-    // if we DO NOT close the database (I guess it's process-shared?)
-    // await rootDb.close();
+    const rootDb = await importerConfig.config.openDatabase({ readonly: false });
+    try {
+        await initializeDatabase(chainSpec, blake2b, genesisHeaderHash, rootDb, config.node.chainSpec, config.ancestry);
+    }
+    catch (e) {
+        try {
+            await rootDb.close();
+        }
+        catch (closeError) {
+            logger.warn `Failed to close database after initialization error: ${closeError}`;
+        }
+        throw e;
+    }
+    // fjall-js shares the engine explicitly and requires every handle to close.
+    // Keep lmdb's historical main-thread handle open until shutdown.
+    let mainRootDb = rootDb;
+    if (!importerConfig.isInMemory && config.node.stateBackend === RegularStateBackend.Fjall) {
+        await rootDb.close();
+        mainRootDb = null;
+    }
     // Start block importer
     let importer;
     let closeImporter;
@@ -138,8 +155,10 @@ export async function main(config, withRelPath, telemetry) {
             await closeImporter();
             logger.log `[main] ☠️  Closing the extensions`;
             closeExtensions();
-            logger.log `[main] 🛢️ Closing the database`;
-            await rootDb.close();
+            if (mainRootDb !== null) {
+                logger.log `[main] 🛢️ Closing the database`;
+                await mainRootDb.close();
+            }
             logger.log `[main] 📳 Closing telemetry`;
             await telemetry?.close();
             logger.info `[main] ✅ Done.`;
@@ -167,7 +186,7 @@ const initAuthorship = async (importer, isAuthoring, isFastForward, params, base
             statesDb: params.rootDb.getStatesDb(),
             workerParams,
         }), params.networkingPort)
-        : await spawnBlockGeneratorWorker(LmdbWorkerConfig.new({
+        : await spawnBlockGeneratorWorker(createPersistentWorkerConfig({
             ...baseConfig,
             workerParams,
             ports: new Map([[AUTHORSHIP_NETWORK_PORT, params.networkingPort]]),
@@ -205,7 +224,7 @@ const initNetwork = async (importer, params, baseConfig, genesisHeaderHash, netw
             statesDb: params.rootDb.getStatesDb(),
             workerParams: networkingConfig,
         }), params.authorshipPort)
-        : await spawnNetworkWorker(LmdbWorkerConfig.new({
+        : await spawnNetworkWorker(createPersistentWorkerConfig({
             ...baseConfig,
             workerParams: networkingConfig,
             ports: new Map([[AUTHORSHIP_NETWORK_PORT, params.authorshipPort]]),
@@ -222,3 +241,11 @@ const initNetwork = async (importer, params, baseConfig, genesisHeaderHash, netw
     });
     return { closeNetwork: finish, networkApi: network, networkWorker: worker };
 };
+function createPersistentWorkerConfig({ stateBackend, ...params }) {
+    switch (stateBackend) {
+        case RegularStateBackend.Fjall:
+            return FjallWorkerConfig.new(params);
+        case RegularStateBackend.Lmdb:
+            return LmdbWorkerConfig.new(params);
+    }
+}