@typeberry/jam 0.1.0-3c30204 → 0.1.0-eb00e84

package/importer/index.js

@@ -5695,7 +5695,6 @@ async function verifyBatch(input) {
 
 ;// CONCATENATED MODULE: ./packages/core/hash/hash.ts
 
-
 /**
  * Size of the output of the hash functions.
  *
@@ -5705,7 +5704,6 @@ async function verifyBatch(input) {
 const hash_HASH_SIZE = 32;
 /** A hash without last byte (useful for trie representation). */
 const TRUNCATED_HASH_SIZE = 31;
-const ZERO_HASH = bytes_Bytes.zero(hash_HASH_SIZE);
 /**
  * Container for some object with a hash that is related to this object.
  *
@@ -8538,7 +8536,7 @@ const common_tryAsServiceGas = (v) => opaque_asOpaqueType(numbers_tryAsU64(v));
 /** Attempt to convert a number into `CoreIndex`. */
 const common_tryAsCoreIndex = (v) => opaque_asOpaqueType(numbers_tryAsU16(v));
 /** Attempt to convert a number into `Epoch`. */
-const tryAsEpoch = (v) => asOpaqueType(tryAsU32(v));
+const tryAsEpoch = (v) => opaque_asOpaqueType(numbers_tryAsU32(v));
 function tryAsPerValidator(array, spec) {
     debug_check(array.length === spec.validatorsCount, `Invalid per-validator array length. Expected ${spec.validatorsCount}, got: ${array.length}`);
     return sized_array_asKnownSize(array);
@@ -18922,6 +18920,88 @@ class PvmExecutor {
 
 
 
+;// CONCATENATED MODULE: ./workers/importer/import-queue.ts
+
+
+
+
+
+class ImportQueue {
+    spec;
+    importer;
+    toImport = SortedArray.fromSortedArray((a, b) => {
+        const diff = a.timeSlot - b.timeSlot;
+        if (diff < 0) {
+            return Ordering.Greater;
+        }
+        if (diff > 0) {
+            return Ordering.Less;
+        }
+        return Ordering.Equal;
+    });
+    queuedBlocks = HashSet.new();
+    lastEpoch = tryAsEpoch(2 ** 32 - 1);
+    constructor(spec, importer) {
+        this.spec = spec;
+        this.importer = importer;
+    }
+    isCurrentEpoch(timeSlot) {
+        const epoch = Math.floor(timeSlot / this.spec.epochLength);
+        return this.lastEpoch === epoch;
+    }
+    startPreverification() {
+        for (const entry of this.toImport) {
+            if (this.isCurrentEpoch(entry.timeSlot)) {
+                entry.seal = this.importer.preverifySeal(entry.timeSlot, entry.block);
+            }
+        }
+    }
+    static getBlockDetails(block) {
+        let encodedHeader;
+        let timeSlot;
+        try {
+            encodedHeader = block.header.encoded();
+            timeSlot = block.header.view().timeSlotIndex.materialize();
+        }
+        catch {
+            return result_Result.error("invalid");
+        }
+        const headerHash = hashBytes(encodedHeader).asOpaque();
+        return result_Result.ok(new WithHash(headerHash, { block, timeSlot }));
+    }
+    push(details) {
+        const headerHash = details.hash;
+        if (this.queuedBlocks.has(headerHash)) {
+            return result_Result.error("already queued");
+        }
+        const { timeSlot, block } = details.data;
+        const entry = {
+            headerHash,
+            timeSlot,
+            block,
+            seal: this.isCurrentEpoch(timeSlot) ? this.importer.preverifySeal(timeSlot, block) : Promise.resolve(null),
+        };
+        this.toImport.insert(entry);
+        this.queuedBlocks.insert(headerHash);
+        return result_Result.ok(result_OK);
+    }
+    shift() {
+        const entry = this.toImport.pop();
+        if (entry !== undefined) {
+            this.queuedBlocks.delete(entry.headerHash);
+            const blockEpoch = Math.floor(entry.timeSlot / this.spec.epochLength);
+            const hasEpochChanged = this.lastEpoch !== blockEpoch;
+            this.lastEpoch = tryAsEpoch(blockEpoch);
+            // currently removed block is changing the epoch, so fire up
+            // preverifcation for the following blocks.
+            if (hasEpochChanged) {
+                this.startPreverification();
+            }
+        }
+        return entry;
+    }
+}
+
 ;// CONCATENATED MODULE: ./packages/jam/transition/block-verifier.ts
 
 
@@ -18935,7 +19015,7 @@ var BlockVerifierError;
     BlockVerifierError[BlockVerifierError["InvalidStateRoot"] = 4] = "InvalidStateRoot";
     BlockVerifierError[BlockVerifierError["AlreadyImported"] = 5] = "AlreadyImported";
 })(BlockVerifierError || (BlockVerifierError = {}));
-const block_verifier_ZERO_HASH = bytes_Bytes.zero(hash_HASH_SIZE).asOpaque();
+const ZERO_HASH = bytes_Bytes.zero(hash_HASH_SIZE).asOpaque();
 class BlockVerifier {
     hasher;
     blocks;
@@ -18955,7 +19035,7 @@ class BlockVerifier {
         // https://graypaper.fluffylabs.dev/#/cc517d7/0c9d000c9d00?v=0.6.5
         const parentHash = headerView.parentHeaderHash.materialize();
         // importing genesis block
-        if (!parentHash.isEqualTo(block_verifier_ZERO_HASH)) {
+        if (!parentHash.isEqualTo(ZERO_HASH)) {
             const parentBlock = this.blocks.getHeader(parentHash);
             if (parentBlock === null) {
                 return result_Result.error(BlockVerifierError.ParentNotFound, `Parent ${parentHash.toString()} not found`);
@@ -19468,22 +19548,304 @@ async function verifyTickets(bandersnatch, numberOfValidators, epochRoot, ticket
     }));
 }
 
-;// CONCATENATED MODULE: ./packages/jam/safrole/bandersnatch-wasm.ts
+;// CONCATENATED MODULE: external "node:os"
+const external_node_os_namespaceObject = __WEBPACK_EXTERNAL_createRequire(import.meta.url)("node:os");
+var external_node_os_default = /*#__PURE__*/__nccwpck_require__.n(external_node_os_namespaceObject);
+;// CONCATENATED MODULE: ./packages/core/concurrent/parent.ts
+
+
+// Amount of tasks in the queue that will trigger creation of new worker thread.
+// NOTE this might need to be configurable in the future.
+const QUEUE_SIZE_WORKER_THRESHOLD = 5;
+/** Execution pool manager. */
+class Executor {
+    workers;
+    maxWorkers;
+    workerPath;
+    /** Initialize a new concurrent executor given a path to the worker. */
+    static async initialize(workerPath, options) {
+        debug_check(options.maxWorkers > 0, "Max workers has to be positive.");
+        debug_check(options.minWorkers <= options.maxWorkers, "Min workers has to be lower or equal to max workers.");
+        const workers = [];
+        for (let i = 0; i < options.minWorkers; i++) {
+            workers.push(await initWorker(workerPath));
+        }
+        return new Executor(workers, options.maxWorkers, workerPath);
+    }
+    // keeps track of the indices of worker threads that are currently free and available to execute tasks
+    freeWorkerIndices = [];
+    taskQueue = [];
+    isDestroyed = false;
+    isWorkerInitializing = false;
+    constructor(workers, maxWorkers, workerPath) {
+        this.workers = workers;
+        this.maxWorkers = maxWorkers;
+        this.workerPath = workerPath;
+        // intial free workers.
+        for (let i = 0; i < workers.length; i++) {
+            this.freeWorkerIndices.push(i);
+        }
+    }
+    /** Attempt to initialize a new worker. */
+    async initNewWorker(onSuccess = () => { }) {
+        if (this.workers.length >= this.maxWorkers) {
+            // biome-ignore lint/suspicious/noConsole: warning
+            console.warn(`Task queue has ${this.taskQueue.length} pending items and we can't init any more workers.`);
+            return;
+        }
+        if (this.isWorkerInitializing) {
+            return;
+        }
+        this.isWorkerInitializing = true;
+        this.workers.push(await initWorker(this.workerPath));
+        this.freeWorkerIndices.push(this.workers.length - 1);
+        this.isWorkerInitializing = false;
+        onSuccess();
+    }
+    /** Terminate all workers and clear the executor. */
+    async destroy() {
+        for (const worker of this.workers) {
+            worker.port.close();
+            await worker.worker.terminate();
+        }
+        this.workers.length = 0;
+        this.isDestroyed = true;
+    }
+    /** Execute a task with given parameters. */
+    async run(params) {
+        return new Promise((resolve, reject) => {
+            if (this.isDestroyed) {
+                reject("pool destroyed");
+                return;
+            }
+            this.taskQueue.push({
+                params,
+                resolve,
+                reject,
+            });
+            this.processEntryFromTaskQueue();
+        });
+    }
+    /** Process single element from the task queue. */
+    processEntryFromTaskQueue() {
+        const freeWorker = this.freeWorkerIndices.pop();
+        // no free workers available currently,
+        // we will retry when one of the tasks completes.
+        if (freeWorker === undefined) {
+            if (this.taskQueue.length > QUEUE_SIZE_WORKER_THRESHOLD) {
+                this.initNewWorker(() => {
+                    // process an entry in this newly initialized worker.
+                    this.processEntryFromTaskQueue();
+                });
+            }
+            return;
+        }
+        const task = this.taskQueue.pop();
+        // no tasks in the queue
+        if (task === undefined) {
+            this.freeWorkerIndices.push(freeWorker);
+            return;
+        }
+        const worker = this.workers[freeWorker];
+        worker.runTask(task, () => {
+            // mark the worker as available again
+            this.freeWorkerIndices.push(freeWorker);
+            // and continue processing the queue
+            this.processEntryFromTaskQueue();
+        });
+    }
+}
+async function initWorker(workerPath) {
+    // create a worker and initialize communication channel
+    const { port1, port2 } = new MessageChannel();
+    const workerThread = new external_node_worker_threads_namespaceObject.Worker(workerPath, {});
+    workerThread.postMessage(port1, [port1]);
+    // // wait for the worker to start
+    await new Promise((resolve, reject) => {
+        workerThread.once("message", resolve);
+        workerThread.once("error", reject);
+    });
+    // make sure the threads don't prevent the program from stopping.
+    workerThread.unref();
+    return new WorkerChannel(workerThread, port2);
+}
+class WorkerChannel {
+    worker;
+    port;
+    constructor(worker, port) {
+        this.worker = worker;
+        this.port = port;
+    }
+    runTask(task, onFinish) {
+        const message = {
+            params: task.params,
+        };
+        // when we receive a response, make sure to process it
+        this.port.once("message", (e) => {
+            if (e.isOk) {
+                task.resolve(e.ok);
+            }
+            else {
+                task.reject(new Error(e.error));
+            }
+            onFinish();
+        });
+        // send the task to work on.
+        this.port.postMessage(message, message.params.getTransferList());
+    }
+}
+
+;// CONCATENATED MODULE: ./packages/core/concurrent/worker.ts
+
+
+/** A in-worker abstraction. */
+class ConcurrentWorker {
+    runInternal;
+    state;
+    static new(run, state) {
+        return new ConcurrentWorker(run, state);
+    }
+    constructor(runInternal, state) {
+        this.runInternal = runInternal;
+        this.state = state;
+    }
+    listenToParentPort() {
+        if (external_node_worker_threads_namespaceObject.parentPort === null) {
+            throw new Error("This method is meant to be run inside a worker thread!");
+        }
+        external_node_worker_threads_namespaceObject.parentPort.once("close", () => {
+            process.exit(0);
+        });
+        external_node_worker_threads_namespaceObject.parentPort.once("message", (port) => {
+            this.listenTo(port);
+            // send back readiness signal.
+            external_node_worker_threads_namespaceObject.parentPort?.postMessage("ready");
+        });
+    }
+    listenTo(port) {
+        port.once("close", () => {
+            port.removeAllListeners();
+            process.exit(0);
+        });
+        port.on("message", (ev) => {
+            const { params } = ev;
+            this.run(params)
+                .then((result) => {
+                const response = result_Result.ok(result);
+                port.postMessage(response, result.getTransferList());
+            })
+                .catch((e) => {
+                const response = result_Result.error(`${e}`);
+                port.postMessage(response, []);
+            });
+        });
+    }
+    async run(params) {
+        return await this.runInternal(params, this.state);
+    }
+    async destroy() { }
+}
+
+;// CONCATENATED MODULE: ./packages/core/concurrent/index.ts
+
 
+
+;// CONCATENATED MODULE: ./packages/jam/safrole/bandersnatch-wasm/params.ts
+var Method;
+(function (Method) {
+    Method[Method["RingCommitment"] = 0] = "RingCommitment";
+    Method[Method["BatchVerifyTickets"] = 1] = "BatchVerifyTickets";
+    Method[Method["VerifySeal"] = 2] = "VerifySeal";
+})(Method || (Method = {}));
+class params_Response {
+    data;
+    constructor(data) {
+        this.data = data;
+    }
+    getTransferList() {
+        return [this.data.buffer];
+    }
+}
+class Params {
+    params;
+    constructor(params) {
+        this.params = params;
+    }
+    getTransferList() {
+        return [];
+    }
+}
+
+;// CONCATENATED MODULE: ./packages/jam/safrole/bandersnatch-wasm/worker.ts
+
+
+
+
+const worker = ConcurrentWorker.new(async (p) => {
+    await initAll();
+    const params = p.params;
+    const method = params.method;
+    if (method === Method.RingCommitment) {
+        return Promise.resolve(new params_Response(bandersnatch_exports.ring_commitment(params.keys)));
+    }
+    if (method === Method.BatchVerifyTickets) {
+        return Promise.resolve(new params_Response(bandersnatch_exports.batch_verify_tickets(params.ringSize, params.commitment, params.ticketsData, params.contextLength)));
+    }
+    if (method === Method.VerifySeal) {
+        return Promise.resolve(new params_Response(bandersnatch_exports.verify_seal(params.authorKey, params.signature, params.payload, params.auxData)));
+    }
+    debug_assertNever(method);
+}, null);
+
+;// CONCATENATED MODULE: ./packages/jam/safrole/bandersnatch-wasm/index.ts
+
+
+
+
+const workerFile = __nccwpck_require__.ab + "bootstrap-bandersnatch.mjs";
 class BandernsatchWasm {
-    constructor() { }
-    static async new() {
-        await initAll();
-        return new BandernsatchWasm();
+    executor;
+    constructor(executor) {
+        this.executor = executor;
+    }
+    destroy() {
+        return this.executor.destroy();
+    }
+    static async new({ synchronous }) {
+        const workers = external_node_os_default().cpus().length;
+        return new BandernsatchWasm(!synchronous
+            ? await Executor.initialize(workerFile, {
+                minWorkers: Math.max(1, Math.floor(workers / 2)),
+                maxWorkers: workers,
+            })
+            : worker);
     }
     async verifySeal(authorKey, signature, payload, auxData) {
-        return bandersnatch_exports.verify_seal(authorKey, signature, payload, auxData);
+        const x = await this.executor.run(new Params({
+            method: Method.VerifySeal,
+            authorKey,
+            signature,
+            payload,
+            auxData,
+        }));
+        return x.data;
     }
     async getRingCommitment(keys) {
-        return bandersnatch_exports.ring_commitment(keys);
+        const x = await this.executor.run(new Params({
+            method: Method.RingCommitment,
+            keys,
+        }));
+        return x.data;
     }
     async batchVerifyTicket(ringSize, commitment, ticketsData, contextLength) {
-        return bandersnatch_exports.batch_verify_tickets(ringSize, commitment, ticketsData, contextLength);
+        const x = await this.executor.run(new Params({
+            method: Method.BatchVerifyTickets,
+            ringSize,
+            commitment,
+            ticketsData,
+            contextLength,
+        }));
+        return x.data;
     }
 }
 
@@ -19526,7 +19888,7 @@ class Safrole {
     chainSpec;
     state;
     bandersnatch;
-    constructor(chainSpec, state, bandersnatch = BandernsatchWasm.new()) {
+    constructor(chainSpec, state, bandersnatch = BandernsatchWasm.new({ synchronous: true })) {
         this.chainSpec = chainSpec;
         this.state = state;
         this.bandersnatch = bandersnatch;
@@ -19904,7 +20266,7 @@ var SafroleSealError;
 const BANDERSNATCH_ZERO_KEY = bytes_Bytes.zero(BANDERSNATCH_KEY_BYTES).asOpaque();
 class SafroleSeal {
     bandersnatch;
-    constructor(bandersnatch = BandernsatchWasm.new()) {
+    constructor(bandersnatch = BandernsatchWasm.new({ synchronous: true })) {
         this.bandersnatch = bandersnatch;
     }
     /**
@@ -24847,11 +25209,11 @@ class OnChain {
     authorization;
     // chapter 13: https://graypaper.fluffylabs.dev/#/68eaa1f/18b60118b601?v=0.6.4
     statistics;
-    constructor(chainSpec, state, blocks, hasher) {
+    constructor(chainSpec, state, blocks, hasher, { enableParallelSealVerification }) {
         this.chainSpec = chainSpec;
         this.state = state;
         this.hasher = hasher;
-        const bandersnatch = BandernsatchWasm.new();
+        const bandersnatch = BandernsatchWasm.new({ synchronous: !enableParallelSealVerification });
         this.statistics = new Statistics(chainSpec, state);
         this.safrole = new Safrole(chainSpec, state, bandersnatch);
         this.safroleSeal = new SafroleSeal(bandersnatch);
@@ -24869,16 +25231,16 @@ class OnChain {
         const sealState = this.safrole.getSafroleSealState(timeSlot);
         return await this.safroleSeal.verifyHeaderSeal(block.header.view(), sealState);
     }
-    async transition(block, headerHash, omitSealVerification = false) {
+    async transition(block, headerHash, preverifiedSeal = null, omitSealVerification = false) {
         const headerView = block.header.view();
         const header = block.header.materialize();
         const timeSlot = header.timeSlotIndex;
         // safrole seal
-        let newEntropyHash;
+        let newEntropyHash = preverifiedSeal;
         if (omitSealVerification) {
             newEntropyHash = hashBytes(header.seal).asOpaque();
         }
-        else {
+        if (newEntropyHash === null) {
             const sealResult = await this.verifySeal(timeSlot, block);
             if (sealResult.isError) {
                 return stfError(StfErrorKind.SafroleSeal, sealResult);
@@ -24985,7 +25347,7 @@ class OnChain {
         assertEmpty(deferredTransfersRest);
         const accumulateRoot = await this.accumulateOutput.transition({ accumulationOutputLog });
         // recent history
-        const recentHistoryUpdate = this.recentHistory.transition({
+        const recentHistoryUpdate = await this.recentHistory.transition({
             partial: recentHistoryPartialUpdate,
             headerHash,
             accumulateRoot,
@@ -25065,7 +25427,6 @@ function checkOffendersMatch(offendersMark, headerOffendersMark) {
 
 
 
-
 var ImporterErrorKind;
 (function (ImporterErrorKind) {
     ImporterErrorKind[ImporterErrorKind["Verifier"] = 0] = "Verifier";
@@ -25093,28 +25454,29 @@ class Importer {
             throw new Error(`Unable to load best state from header hash: ${currentBestHeaderHash}.`);
         }
         this.verifier = new BlockVerifier(hasher, blocks);
-        this.stf = new OnChain(spec, state, blocks, hasher);
+        this.stf = new OnChain(spec, state, blocks, hasher, { enableParallelSealVerification: true });
         this.state = state;
         this.currentHash = currentBestHeaderHash;
         logger.info(`😎 Best time slot: ${state.timeslot} (header hash: ${currentBestHeaderHash})`);
     }
-    async importBlock(block, omitSealVerification) {
-        const timer = measure("importBlock");
-        const timeSlot = extractTimeSlot(block);
-        const maybeBestHeader = await this.importBlockInternal(block, omitSealVerification);
-        if (maybeBestHeader.isOk) {
-            const bestHeader = maybeBestHeader.ok;
-            this.logger.info(`🧊 Best block: #${timeSlot} (${bestHeader.hash})`);
-            this.logger.log(timer());
-            return maybeBestHeader;
-        }
-        this.logger.log(`❌ Rejected block #${timeSlot}: ${resultToString(maybeBestHeader)}`);
-        this.logger.log(timer());
-        return maybeBestHeader;
-    }
-    async importBlockInternal(block, omitSealVerification = false) {
+    /** Attempt to pre-verify the seal to speed up importing. */
+    async preverifySeal(timeSlot, block) {
+        try {
+            const res = await this.stf.verifySeal(timeSlot, block);
+            if (res.isOk) {
+                return res.ok;
+            }
+            this.logger.warn(`Unable to pre-verify the seal: ${resultToString(res)}`);
+            return null;
+        }
+        catch (e) {
+            this.logger.warn(`Error while trying to pre-verify the seal: ${e}`);
+            return null;
+        }
+    }
+    async importBlock(block, preverifiedSeal, omitSealVerification = false) {
         const logger = this.logger;
-        logger.log("🧱 Attempting to import a new block");
+        logger.log(`🧱 Attempting to import a new block ${preverifiedSeal !== null ? "(seal preverified)" : ""}`);
         const timerVerify = measure("import:verify");
         const hash = await this.verifier.verifyBlock(block);
         logger.log(timerVerify());
@@ -25139,7 +25501,7 @@ class Importer {
         const headerHash = hash.ok;
         logger.log(`🧱 Verified block: Got hash ${headerHash} for block at slot ${timeSlot}.`);
         const timerStf = measure("import:stf");
-        const res = await this.stf.transition(block, headerHash, omitSealVerification);
+        const res = await this.stf.transition(block, headerHash, preverifiedSeal, omitSealVerification);
         logger.log(timerStf());
         if (res.isError) {
             return importerError(ImporterErrorKind.Stf, res);
@@ -25189,19 +25551,6 @@ class Importer {
         return stateEntries ?? null;
     }
 }
-/**
- * Attempt to safely extract timeslot of a block.
- *
- * NOTE: it may fail if encoding is invalid.
- */
-function extractTimeSlot(block) {
-    try {
-        return block.header.view().timeSlotIndex.materialize();
-    }
-    catch {
-        return tryAsTimeSlot(2 ** 32 - 1);
-    }
-}
 
 ;// CONCATENATED MODULE: ./workers/generic/finished.ts
 
@@ -25465,6 +25814,7 @@ class ImporterReady extends State {
             response: rootHash === null ? bytes_Bytes.zero(hash_HASH_SIZE).raw : rootHash.raw,
         };
     }
+    // NOTE [ToDr] This should rather be using the import queue, instead of going directly.
     async importBlock(block) {
         if (this.importer === null) {
             state_machine_logger.error(`${this.constructor.name} importer not initialized yet!`);
@@ -25476,13 +25826,17 @@ class ImporterReady extends State {
         if (block instanceof Uint8Array) {
             const config = this.getConfig();
             const blockView = decoder_Decoder.decodeObject(Block.Codec.View, block, config.chainSpec);
+            const headerView = blockView.header.view();
+            const timeSlot = headerView.timeSlotIndex.materialize();
             let response;
             try {
-                const res = await this.importer.importBlock(blockView, config.omitSealVerification);
+                const res = await this.importer.importBlock(blockView, null, config.omitSealVerification);
                 if (res.isOk) {
-                    response = result_Result.ok(this.importer.getBestStateRootHash() ?? ZERO_HASH.asOpaque());
+                    state_machine_logger.info(`🧊 Best block: #${timeSlot} (${res.ok.hash})`);
+                    response = result_Result.ok(this.importer.getBestStateRootHash() ?? bytes_Bytes.zero(hash_HASH_SIZE).asOpaque());
                 }
                 else {
+                    state_machine_logger.log(`❌ Rejected block #${timeSlot}: ${resultToString(res)}`);
                     response = result_Result.error(resultToString(res));
                 }
             }
@@ -25530,6 +25884,8 @@ class ImporterReady extends State {
 
 
 
+
+
 const importer_logger = Logger.new(import.meta.filename, "importer");
 if (!external_node_worker_threads_namespaceObject.isMainThread) {
     Logger.configureAll(process.env.JAM_LOG ?? "", Level.LOG);
@@ -25546,6 +25902,7 @@ async function createImporter(config) {
     const importer = new Importer(config.chainSpec, hasher, importer_logger, blocks, states);
     return {
         lmdb,
+        blocks,
         importer,
     };
 }
@@ -25560,27 +25917,65 @@ async function main(channel) {
     importer_logger.info(`📥 Importer starting ${channel.currentState()}`);
     // Await the configuration object
     const ready = await channel.waitForState("ready(importer)");
-    let closeDb = async () => { };
     const finished = await ready.doUntil("finished", async (worker, port) => {
         const config = worker.getConfig();
-        const { lmdb, importer } = await createImporter(config);
-        closeDb = async () => {
-            await lmdb.close();
-        };
+        const { blocks, importer } = await createImporter(config);
         // TODO [ToDr] this is shit, since we have circular dependency.
         worker.setImporter(importer);
         importer_logger.info("📥 Importer waiting for blocks.");
+        // TODO [ToDr] back pressure?
+        let isProcessing = false;
+        const importingQueue = new ImportQueue(config.chainSpec, importer);
         worker.onBlock.on(async (block) => {
-            const res = await importer.importBlock(block, config.omitSealVerification);
-            if (res.isOk) {
-                worker.announce(port, res.ok);
+            const details = ImportQueue.getBlockDetails(block);
+            // ignore invalid blocks.
+            if (details.isError) {
+                importer_logger.trace("🧊 Ignoring invalid block.");
+                return;
+            }
+            // ignore already known blocks
+            if (blocks.getHeader(details.ok.hash) !== null) {
+                importer_logger.trace(`🧊 Already imported block: #${details.ok.data.timeSlot}.`);
+                return;
+            }
+            const importResult = importingQueue.push(details.ok);
+            // ignore blocks that are already queued
+            if (importResult.isError) {
+                importer_logger.trace(`🧊 Already queued block: #${details.ok.data.timeSlot}.`);
+                return;
+            }
+            importer_logger.log(`🧊 Queued block: #${details.ok.data.timeSlot} (skip seal: ${config.omitSealVerification})`);
+            if (isProcessing) {
+                return;
+            }
+            isProcessing = true;
+            try {
+                for (;;) {
+                    const entry = importingQueue.shift();
+                    if (entry === undefined) {
+                        return;
+                    }
+                    const { block, seal, timeSlot } = entry;
+                    const timer = measure("importBlock");
+                    const maybeBestHeader = await importer.importBlock(block, await seal, config.omitSealVerification);
+                    if (maybeBestHeader.isOk) {
+                        const bestHeader = maybeBestHeader.ok;
+                        worker.announce(port, bestHeader);
+                        importer_logger.info(`🧊 Best block: #${bestHeader.data.timeSlotIndex.materialize()} (${bestHeader.hash})`);
+                    }
+                    else {
+                        importer_logger.log(`❌ Rejected block #${timeSlot}: ${resultToString(maybeBestHeader)}`);
+                    }
+                    importer_logger.log(timer());
+                }
+            }
+            finally {
+                isProcessing = false;
             }
         });
         await wasmPromise;
     });
     importer_logger.info("📥 Importer finished. Closing channel.");
-    // close the database
-    await closeDb();
     // Close the comms to gracefuly close the app.
     finished.currentState().close(channel);
 }