@typeberry/jam 0.5.8-857ec24 → 0.5.8-b3da767

package/index.js

@@ -152750,6 +152750,9 @@ class InMemorySerializedStates {
         });
         return serialized_state_SerializedState.new(this.spec, this.blake2b, leafDb);
     }
+    markUnused(header) {
+        this.db.delete(header);
+    }
     async close() { }
 }
 
@@ -152806,6 +152809,9 @@ class InMemoryStates {
         }
         return InMemoryState.copyFrom(this.spec, state, state.intoServicesData());
     }
+    markUnused(header) {
+        this.db.delete(header);
+    }
     async close() { }
 }
 
@@ -153050,6 +153056,9 @@ class LmdbStates {
         }
         return serialized_state_SerializedState.new(this.spec, this.blake2b, leafDbResult.ok);
     }
+    markUnused(header) {
+        this.states.remove(header.raw);
+    }
     async close() {
         await Promise.all([this.states.close(), this.values.close()]);
     }
@@ -166533,6 +166542,113 @@ class TransitionHasher {
 
 
 
+;// CONCATENATED MODULE: ./packages/workers/importer/finality.ts
+
+const finality_logger = logger_Logger.new(import.meta.filename, "finality");
+/**
+ * A simple finalizer that considers a block finalized when N blocks
+ * have been built on top of it.
+ *
+ * Maintains an array of fork chains starting from the last finalized block.
+ * When any chain reaches `depth`, the earliest blocks are finalized and
+ * dead forks (branching from before the finalized point) are discarded.
+ */
+class DummyFinalizer {
+    blocks;
+    depth;
+    lastFinalizedHash;
+    unfinalized = [];
+    static create(blocks, depth) {
+        return new DummyFinalizer(blocks, depth);
+    }
+    constructor(blocks, depth) {
+        this.blocks = blocks;
+        this.depth = depth;
+        this.lastFinalizedHash = blocks.getBestHeaderHash();
+        finality_logger.info `🦭 Dummy Finalizer running with depth=${depth}`;
+    }
+    onBlockImported(headerHash) {
+        const header = this.blocks.getHeader(headerHash);
+        if (header === null) {
+            return null;
+        }
+        const parentHash = header.parentHeaderHash.materialize();
+        // Try to attach the block to an existing chain at its tip.
+        let extendedChain = null;
+        for (const chain of this.unfinalized) {
+            if (chain.length > 0 && chain[chain.length - 1].isEqualTo(parentHash)) {
+                chain.push(headerHash);
+                extendedChain = chain;
+                break;
+            }
+        }
+        if (extendedChain === null) {
+            if (this.lastFinalizedHash.isEqualTo(parentHash)) {
+                // Parent is the finalized block — start a new chain.
+                const newChain = [headerHash];
+                this.unfinalized.push(newChain);
+                extendedChain = newChain;
+            }
+            else {
+                // Fork from the middle of an existing chain — copy the prefix and branch.
+                for (const chain of this.unfinalized) {
+                    const forkIdx = chain.findIndex((h) => h.isEqualTo(parentHash));
+                    if (forkIdx !== -1) {
+                        const newChain = [...chain.slice(0, forkIdx + 1), headerHash];
+                        this.unfinalized.push(newChain);
+                        extendedChain = newChain;
+                        break;
+                    }
+                }
+            }
+        }
+        if (extendedChain === null) {
+            // Orphan block — cannot attach to any known chain.
+            return null;
+        }
+        // Check if the extended chain is long enough to trigger finality.
+        // A chain of length N has N-1 blocks built on top of chain[0].
+        // We finalize chain[0] when there are >= depth blocks after it,
+        // i.e. chain.length > depth.
+        if (extendedChain.length <= this.depth) {
+            return null;
+        }
+        // The newly finalized block sits at index (length - 1 - depth).
+        const finalizedIdx = extendedChain.length - 1 - this.depth;
+        const finalizedHash = extendedChain[finalizedIdx];
+        // Collect prunable hashes and rebuild the unfinalized set.
+        // The previously finalized block's state is no longer needed.
+        const prunable = [this.lastFinalizedHash];
+        const newUnfinalized = [];
+        for (const chain of this.unfinalized) {
+            // Find the finalized block in this chain.
+            const finIdx = chain.findIndex((h) => h.isEqualTo(finalizedHash));
+            if (finIdx !== -1) {
+                // Chain contains the finalized block — it's still alive.
+                // Prune states for blocks before the finalized block.
+                for (let i = 0; i < finIdx; i++) {
+                    prunable.push(chain[i]);
+                }
+                // Keep blocks after the finalized block.
+                const remaining = chain.slice(finIdx + 1);
+                if (remaining.length > 0) {
+                    newUnfinalized.push(remaining);
+                }
+            }
+            else {
+                // Dead fork — branches from a block that is no longer finalized.
+                // Prune all its states.
+                for (const h of chain) {
+                    prunable.push(h);
+                }
+            }
+        }
+        this.lastFinalizedHash = finalizedHash;
+        this.unfinalized = newUnfinalized;
+        return { finalizedHash, prunableStateHashes: prunable };
+    }
+}
+
 ;// CONCATENATED MODULE: ./packages/workers/importer/metrics.ts
 
 
@@ -166757,6 +166873,14 @@ class Importer {
         logger.log `${timerDb()}`;
         // finally update the best block
         await this.blocks.setBestHeaderHash(headerHash);
+        // check for finality and prune old states
+        const finality = this.options.finalizer?.onBlockImported(headerHash) ?? null;
+        if (finality !== null) {
+            this.logger.info `🦭 Finalized block: ${finality.finalizedHash} (${finality.prunableStateHashes.length} to prune)`;
+            for (const hash of finality.prunableStateHashes) {
+                this.states.markUnused(hash);
+            }
+        }
         return result_Result.ok(new WithHash(headerHash, block.header.view()));
     }
     getBestStateRootHash() {
@@ -166798,6 +166922,7 @@ function extractTimeSlot(block) {
 
 
 
+
 const main_logger = logger_Logger.new(import.meta.filename, "importer");
 const keccakHasher = KeccakHasher.create();
 const blake2b = blake2b_Blake2b.createHasher();
@@ -166807,8 +166932,13 @@ async function createImporter(config, options = {}) {
     const pvm = config.workerParams.pvm;
     const blocks = db.getBlocksDb();
     const states = db.getStatesDb();
+    const dummyFinalityDepth = config.workerParams.dummyFinalityDepth ?? 0;
+    const finalizer = dummyFinalityDepth > 0 ? DummyFinalizer.create(blocks, dummyFinalityDepth) : undefined;
     const hasher = new TransitionHasher(await keccakHasher, await blake2b);
-    const importer = new Importer(chainSpec, pvm, hasher, main_logger, blocks, states, options);
+    const importer = new Importer(chainSpec, pvm, hasher, main_logger, blocks, states, {
+        ...options,
+        finalizer,
+    });
     return {
         importer,
         db,
@@ -166926,6 +167056,7 @@ const protocol_protocol = createProtocol("importer", {
 });
 class ImporterConfig {
     pvm;
+    dummyFinalityDepth;
     static Codec = codec_codec.Class(ImporterConfig, {
         pvm: codec_codec.u8.convert((i) => tryAsU8(i), (o) => {
             if (o === PvmBackend.BuiltIn) {
@@ -166936,12 +167067,16 @@ class ImporterConfig {
             }
             throw new Error(`Invalid PvmBackend: ${o}`);
         }),
+        dummyFinalityDepth: codec_codec.u16,
     });
-    static create({ pvm }) {
-        return new ImporterConfig(pvm);
+    static create({ pvm, dummyFinalityDepth }) {
+        return new ImporterConfig(pvm, dummyFinalityDepth);
     }
-    constructor(pvm) {
+    constructor(pvm, 
+    /** Dummy finality depth. 0 means disabled, any positive value enables dummy finality with that depth. */
+    dummyFinalityDepth = numbers_tryAsU16(0)) {
         this.pvm = pvm;
+        this.dummyFinalityDepth = dummyFinalityDepth;
     }
 }
 
@@ -170304,6 +170439,7 @@ async function node_main_main(config, withRelPath, telemetry) {
         ...baseConfig,
         workerParams: ImporterConfig.create({
             pvm: config.pvmBackend,
+            dummyFinalityDepth: numbers_tryAsU16(config.devValidatorIndex !== null ? 100 : 0),
         }),
     };
     const importerConfig = isInMemory
@@ -170498,6 +170634,7 @@ const initNetwork = async (importer, params, baseConfig, genesisHeaderHash, netw
 
 
 
+
 const zeroHash = bytes_Bytes.zero(hash_HASH_SIZE).asOpaque();
 async function mainImporter(config, withRelPath, options = {}) {
     await initAll();
@@ -170510,23 +170647,23 @@ async function mainImporter(config, withRelPath, options = {}) {
     const blake2b = await blake2b_Blake2b.createHasher();
     const nodeName = config.nodeName;
     const { dbPath, genesisHeaderHash } = getDatabasePath(blake2b, config.nodeName, config.node.chainSpec.genesisHeader, withRelPath(config.node.databaseBasePath ?? "<in-memory>"));
+    const workerParams = ImporterConfig.create({
+        pvm: config.pvmBackend,
+        dummyFinalityDepth: numbers_tryAsU16(options.dummyFinalityDepth ?? 0),
+    });
     const workerConfig = config.node.databaseBasePath === undefined
         ? InMemWorkerConfig.new({
             nodeName,
             chainSpec,
             blake2b,
-            workerParams: {
-                pvm: config.pvmBackend,
-            },
+            workerParams,
         })
         : config_LmdbWorkerConfig.new({
             nodeName,
             chainSpec,
             blake2b,
             dbPath,
-            workerParams: {
-                pvm: config.pvmBackend,
-            },
+            workerParams,
         });
     // Initialize the database with genesis state and block if there isn't one.
     common_logger.info `🛢️ Opening database at ${dbPath}`;
@@ -170635,7 +170772,10 @@ async function mainFuzz(fuzzConfig, withRelPath) {
                 },
                 ancestry,
                 network: null,
-            }, withRelPath, { initGenesisFromAncestry: fuzzConfig.initGenesisFromAncestry });
+            }, withRelPath, {
+                initGenesisFromAncestry: fuzzConfig.initGenesisFromAncestry,
+                dummyFinalityDepth: 10_000,
+            });
             runningNode = newNode;
             return await newNode.getBestStateRootHash();
         },