@lodestar/beacon-node 1.22.0 → 1.23.0-dev.0d1fd9c839

package/lib/chain/shufflingCache.js

@@ -1,8 +1,6 @@
-import { getShufflingDecisionBlock } from "@lodestar/state-transition";
-import { ssz } from "@lodestar/types";
-import { MapDef, pruneSetToMax, toRootHex } from "@lodestar/utils";
-import { GENESIS_SLOT } from "@lodestar/params";
-import { computeAnchorCheckpoint } from "./initState.js";
+import { computeEpochShuffling, } from "@lodestar/state-transition";
+import { LodestarError, MapDef, pruneSetToMax } from "@lodestar/utils";
+import { callInNextEventLoop } from "../util/eventLoop.js";
 /**
  * Same value to CheckpointBalancesCache, with the assumption that we don't have to use it for old epochs. In the worse case:
  * - when loading state bytes from disk, we need to compute shuffling for all epochs (~1s as of Sep 2023)
@@ -27,71 +25,31 @@ var CacheItemType;
  * - skip computing shuffling when loading state bytes from disk
  */
 export class ShufflingCache {
-    constructor(metrics = null, opts = {}) {
+    constructor(metrics = null, logger = null, opts = {}, precalculatedShufflings) {
         this.metrics = metrics;
+        this.logger = logger;
         /** LRU cache implemented as a map, pruned every time we add an item */
         this.itemsByDecisionRootByEpoch = new MapDef(() => new Map());
         if (metrics) {
             metrics.shufflingCache.size.addCollect(() => metrics.shufflingCache.size.set(Array.from(this.itemsByDecisionRootByEpoch.values()).reduce((total, innerMap) => total + innerMap.size, 0)));
         }
         this.maxEpochs = opts.maxShufflingCacheEpochs ?? MAX_EPOCHS;
-    }
-    /**
-     * Extract shuffling from state and add to cache
-     */
-    processState(state, shufflingEpoch) {
-        const decisionBlockHex = getDecisionBlock(state, shufflingEpoch);
-        let shuffling;
-        switch (shufflingEpoch) {
-            case state.epochCtx.nextShuffling.epoch:
-                shuffling = state.epochCtx.nextShuffling;
-                break;
-            case state.epochCtx.currentShuffling.epoch:
-                shuffling = state.epochCtx.currentShuffling;
-                break;
-            case state.epochCtx.previousShuffling.epoch:
-                shuffling = state.epochCtx.previousShuffling;
-                break;
-            default:
-                throw new Error(`Shuffling not found from state ${state.slot} for epoch ${shufflingEpoch}`);
-        }
-        let cacheItem = this.itemsByDecisionRootByEpoch.getOrDefault(shufflingEpoch).get(decisionBlockHex);
-        if (cacheItem !== undefined) {
-            // update existing promise
-            if (isPromiseCacheItem(cacheItem)) {
-                // unblock consumers of this promise
-                cacheItem.resolveFn(shuffling);
-                // then update item type to shuffling
-                cacheItem = {
-                    type: CacheItemType.shuffling,
-                    shuffling,
-                };
-                this.add(shufflingEpoch, decisionBlockHex, cacheItem);
-                // we updated type to CacheItemType.shuffling so the above fields are not used anyway
-                this.metrics?.shufflingCache.processStateUpdatePromise.inc();
-            }
-            else {
-                // ShufflingCacheItem, do nothing
-                this.metrics?.shufflingCache.processStateNoOp.inc();
+        precalculatedShufflings?.map(({ shuffling, decisionRoot }) => {
+            if (shuffling !== null) {
+                this.set(shuffling, decisionRoot);
             }
-        }
-        else {
-            // not found, new shuffling
-            this.add(shufflingEpoch, decisionBlockHex, { type: CacheItemType.shuffling, shuffling });
-            this.metrics?.shufflingCache.processStateInsertNew.inc();
-        }
-        return shuffling;
+        });
     }
     /**
      * Insert a promise to make sure we don't regen state for the same shuffling.
      * Bound by MAX_SHUFFLING_PROMISE to make sure our node does not blow up.
      */
-    insertPromise(shufflingEpoch, decisionRootHex) {
+    insertPromise(epoch, decisionRoot) {
         const promiseCount = Array.from(this.itemsByDecisionRootByEpoch.values())
             .flatMap((innerMap) => Array.from(innerMap.values()))
             .filter((item) => isPromiseCacheItem(item)).length;
         if (promiseCount >= MAX_PROMISES) {
-            throw new Error(`Too many shuffling promises: ${promiseCount}, shufflingEpoch: ${shufflingEpoch}, decisionRootHex: ${decisionRootHex}`);
+            throw new Error(`Too many shuffling promises: ${promiseCount}, shufflingEpoch: ${epoch}, decisionRootHex: ${decisionRoot}`);
         }
         let resolveFn = null;
         const promise = new Promise((resolve) => {
@@ -102,10 +60,11 @@ export class ShufflingCache {
         }
         const cacheItem = {
             type: CacheItemType.promise,
+            timeInsertedMs: Date.now(),
             promise,
             resolveFn,
         };
-        this.add(shufflingEpoch, decisionRootHex, cacheItem);
+        this.itemsByDecisionRootByEpoch.getOrDefault(epoch).set(decisionRoot, cacheItem);
         this.metrics?.shufflingCache.insertPromiseCount.inc();
     }
     /**
@@ -113,35 +72,89 @@ export class ShufflingCache {
      * If there's a promise, it means we are computing the same shuffling, so we wait for the promise to resolve.
      * Return null if we don't have a shuffling for this epoch and dependentRootHex.
      */
-    async get(shufflingEpoch, decisionRootHex) {
-        const cacheItem = this.itemsByDecisionRootByEpoch.getOrDefault(shufflingEpoch).get(decisionRootHex);
+    async get(epoch, decisionRoot) {
+        const cacheItem = this.itemsByDecisionRootByEpoch.getOrDefault(epoch).get(decisionRoot);
         if (cacheItem === undefined) {
+            this.metrics?.shufflingCache.miss.inc();
             return null;
         }
         if (isShufflingCacheItem(cacheItem)) {
+            this.metrics?.shufflingCache.hit.inc();
             return cacheItem.shuffling;
         }
         else {
-            // promise
+            this.metrics?.shufflingCache.shufflingPromiseNotResolved.inc();
             return cacheItem.promise;
         }
     }
     /**
-     * Same to get() function but synchronous.
+     * Gets a cached shuffling via the epoch and decision root.  If the shuffling is not
+     * available it will build it synchronously and return the shuffling.
+     *
+     * NOTE: If a shuffling is already queued and not calculated it will build and resolve
+     * the promise but the already queued build will happen at some later time
      */
-    getSync(shufflingEpoch, decisionRootHex) {
-        const cacheItem = this.itemsByDecisionRootByEpoch.getOrDefault(shufflingEpoch).get(decisionRootHex);
-        if (cacheItem === undefined) {
-            return null;
+    getSync(epoch, decisionRoot, buildProps) {
+        const cacheItem = this.itemsByDecisionRootByEpoch.getOrDefault(epoch).get(decisionRoot);
+        if (!cacheItem) {
+            this.metrics?.shufflingCache.miss.inc();
         }
-        if (isShufflingCacheItem(cacheItem)) {
+        else if (isShufflingCacheItem(cacheItem)) {
+            this.metrics?.shufflingCache.hit.inc();
             return cacheItem.shuffling;
         }
-        // ignore promise
-        return null;
+        else if (buildProps) {
+            // TODO: (@matthewkeil) This should possible log a warning??
+            this.metrics?.shufflingCache.shufflingPromiseNotResolvedAndThrownAway.inc();
+        }
+        else {
+            this.metrics?.shufflingCache.shufflingPromiseNotResolved.inc();
+        }
+        let shuffling = null;
+        if (buildProps) {
+            const timer = this.metrics?.shufflingCache.shufflingCalculationTime.startTimer({ source: "getSync" });
+            shuffling = computeEpochShuffling(buildProps.state, buildProps.activeIndices, epoch);
+            timer?.();
+            this.set(shuffling, decisionRoot);
+        }
+        return shuffling;
     }
-    add(shufflingEpoch, decisionBlock, cacheItem) {
-        this.itemsByDecisionRootByEpoch.getOrDefault(shufflingEpoch).set(decisionBlock, cacheItem);
+    /**
+     * Queue asynchronous build for an EpochShuffling, triggered from state-transition
+     */
+    build(epoch, decisionRoot, state, activeIndices) {
+        this.insertPromise(epoch, decisionRoot);
+        /**
+         * TODO: (@matthewkeil) This will get replaced by a proper build queue and a worker to do calculations
+         * on a NICE thread with a rust implementation
+         */
+        callInNextEventLoop(() => {
+            const timer = this.metrics?.shufflingCache.shufflingCalculationTime.startTimer({ source: "build" });
+            const shuffling = computeEpochShuffling(state, activeIndices, epoch);
+            timer?.();
+            this.set(shuffling, decisionRoot);
+        });
+    }
+    /**
+     * Add an EpochShuffling to the ShufflingCache. If a promise for the shuffling is present it will
+     * resolve the promise with the built shuffling
+     */
+    set(shuffling, decisionRoot) {
+        const shufflingAtEpoch = this.itemsByDecisionRootByEpoch.getOrDefault(shuffling.epoch);
+        // if a pending shuffling promise exists, resolve it
+        const cacheItem = shufflingAtEpoch.get(decisionRoot);
+        if (cacheItem) {
+            if (isPromiseCacheItem(cacheItem)) {
+                cacheItem.resolveFn(shuffling);
+                this.metrics?.shufflingCache.shufflingPromiseResolutionTime.observe((Date.now() - cacheItem.timeInsertedMs) / 1000);
+            }
+            else {
+                this.metrics?.shufflingCache.shufflingBuiltMultipleTimes.inc();
+            }
+        }
+        // set the shuffling
+        shufflingAtEpoch.set(decisionRoot, { type: CacheItemType.shuffling, shuffling });
+        // prune the cache
         pruneSetToMax(this.itemsByDecisionRootByEpoch, this.maxEpochs);
     }
 }
@@ -151,14 +164,10 @@ function isShufflingCacheItem(item) {
 function isPromiseCacheItem(item) {
     return item.type === CacheItemType.promise;
 }
-/**
- * Get the shuffling decision block root for the given epoch of given state
- *   - Special case close to genesis block, return the genesis block root
- *   - This is similar to forkchoice.getDependentRoot() function, otherwise we cannot get cached shuffing in attestation verification when syncing from genesis.
- */
-function getDecisionBlock(state, epoch) {
-    return state.slot > GENESIS_SLOT
-        ? getShufflingDecisionBlock(state, epoch)
-        : toRootHex(ssz.phase0.BeaconBlockHeader.hashTreeRoot(computeAnchorCheckpoint(state.config, state).blockHeader));
+export var ShufflingCacheErrorCode;
+(function (ShufflingCacheErrorCode) {
+    ShufflingCacheErrorCode["NO_SHUFFLING_FOUND"] = "SHUFFLING_CACHE_ERROR_NO_SHUFFLING_FOUND";
+})(ShufflingCacheErrorCode || (ShufflingCacheErrorCode = {}));
+export class ShufflingCacheError extends LodestarError {
 }
 //# sourceMappingURL=shufflingCache.js.map

package/lib/chain/shufflingCache.js.map

@@ -1 +1 @@
-{"version":3,"file":"shufflingCache.js","sourceRoot":"","sources":["../../src/chain/shufflingCache.ts"],"names":[],"mappings":"AAAA,OAAO,EAA4C,yBAAyB,EAAC,MAAM,4BAA4B,CAAC;AAChH,OAAO,EAAiB,GAAG,EAAC,MAAM,iBAAiB,CAAC;AACpD,OAAO,EAAC,MAAM,EAAE,aAAa,EAAE,SAAS,EAAC,MAAM,iBAAiB,CAAC;AACjE,OAAO,EAAC,YAAY,EAAC,MAAM,kBAAkB,CAAC;AAE9C,OAAO,EAAC,uBAAuB,EAAC,MAAM,gBAAgB,CAAC;AAEvD;;;;;IAKI;AACJ,MAAM,UAAU,GAAG,CAAC,CAAC;AAErB;;;GAGG;AACH,MAAM,YAAY,GAAG,CAAC,CAAC;AAEvB,IAAK,aAGJ;AAHD,WAAK,aAAa;IAChB,2DAAS,CAAA;IACT,uDAAO,CAAA;AACT,CAAC,EAHI,aAAa,KAAb,aAAa,QAGjB;AAmBD;;;;;GAKG;AACH,MAAM,OAAO,cAAc;IAQzB,YACmB,UAA0B,IAAI,EAC/C,OAA2B,EAAE;QADZ,YAAO,GAAP,OAAO,CAAuB;QARjD,uEAAuE;QACtD,+BAA0B,GAA2C,IAAI,MAAM,CAC9F,GAAG,EAAE,CAAC,IAAI,GAAG,EAAsB,CACpC,CAAC;QAQA,IAAI,OAAO,EAAE,CAAC;YACZ,OAAO,CAAC,cAAc,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,EAAE,CAC1C,OAAO,CAAC,cAAc,CAAC,IAAI,CAAC,GAAG,CAC7B,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,0BAA0B,CAAC,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,QAAQ,EAAE,EAAE,CAAC,KAAK,GAAG,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC,CAC3G,CACF,CAAC;QACJ,CAAC;QAED,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,uBAAuB,IAAI,UAAU,CAAC;IAC9D,CAAC;IAED;;OAEG;IACH,YAAY,CAAC,KAAgC,EAAE,cAAqB;QAClE,MAAM,gBAAgB,GAAG,gBAAgB,CAAC,KAAK,EAAE,cAAc,CAAC,CAAC;QACjE,IAAI,SAAyB,CAAC;QAC9B,QAAQ,cAAc,EAAE,CAAC;YACvB,KAAK,KAAK,CAAC,QAAQ,CAAC,aAAa,CAAC,KAAK;gBACrC,SAAS,GAAG,KAAK,CAAC,QAAQ,CAAC,aAAa,CAAC;gBACzC,MAAM;YACR,KAAK,KAAK,CAAC,QAAQ,CAAC,gBAAgB,CAAC,KAAK;gBACxC,SAAS,GAAG,KAAK,CAAC,QAAQ,CAAC,gBAAgB,CAAC;gBAC5C,MAAM;YACR,KAAK,KAAK,CAAC,QAAQ,CAAC,iBAAiB,CAAC,KAAK;gBACzC,SAAS,GAAG,KAAK,CAAC,QAAQ,CAAC,iBAAiB,CAAC;gBAC7C,MAAM;YACR;gBACE,MAAM,IAAI,KAAK,CAAC,kCAAkC,KAAK,CAAC,IAAI,cAAc,cAAc,EAAE,CAAC,CAAC;QAChG,CAAC;QAED,IAAI,SAAS,GAAG,IAAI,CAAC,0BAA0B,CAAC,YAAY,CAAC,cAAc,CAAC,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC;QACnG,IAAI,SAAS,KAAK,SAAS,EAAE,CAAC;YAC5B,0BAA0B;YAC1B,IAAI,kBAAkB,CAAC,SAAS,CAAC,EAAE,CAAC;gBAClC,oCAAoC;gBACpC,SAAS,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC;gBAC/B,qCAAqC;gBACrC,SAAS,GAAG;oBACV,IAAI,EAAE,aAAa,CAAC,SAAS;oBAC7B,SAAS;iBACV,CAAC;gBACF,IAAI,CAAC,GAAG,CAAC,cAAc,EAAE,gBAAgB,EAAE,SAAS,CAAC,CAAC;gBACtD,qFAAqF;gBACrF,IAAI,CAAC,OAAO,EAAE,cAAc,CAAC,yBAAyB,CAAC,GAAG,EAAE,CAAC;YAC/D,CAAC;iBAAM,CAAC;gBACN,iCAAiC;gBACjC,IAAI,CAAC,OAAO,EAAE,cAAc,CAAC,gBAAgB,CAAC,GAAG,EAAE,CAAC;YACtD,CAAC;QACH,CAAC;aAAM,CAAC;YACN,2BAA2B;YAC3B,IAAI,CAAC,GAAG,CAAC,cAAc,EAAE,gBAAgB,EAAE,EAAC,IAAI,EAAE,aAAa,CAAC,SAAS,EAAE,SAAS,EAAC,CAAC,CAAC;YACvF,IAAI,CAAC,OAAO,EAAE,cAAc,CAAC,qBAAqB,CAAC,GAAG,EAAE,CAAC;QAC3D,CAAC;QAED,OAAO,SAAS,CAAC;IACnB,CAAC;IAED;;;OAGG;IACH,aAAa,CAAC,cAAqB,EAAE,eAAwB;QAC3D,MAAM,YAAY,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,0BAA0B,CAAC,MAAM,EAAE,CAAC;aACtE,OAAO,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;aACpD,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,kBAAkB,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC;QACrD,IAAI,YAAY,IAAI,YAAY,EAAE,CAAC;YACjC,MAAM,IAAI,KAAK,CACb,gCAAgC,YAAY,qBAAqB,cAAc,sBAAsB,eAAe,EAAE,CACvH,CAAC;QACJ,CAAC;QACD,IAAI,SAAS,GAAiD,IAAI,CAAC;QACnE,MAAM,OAAO,GAAG,IAAI,OAAO,CAAiB,CAAC,OAAO,EAAE,EAAE;YACtD,SAAS,GAAG,OAAO,CAAC;QACtB,CAAC,CAAC,CAAC;QACH,IAAI,SAAS,KAAK,IAAI,EAAE,CAAC;YACvB,MAAM,IAAI,KAAK,CAAC,kDAAkD,CAAC,CAAC;QACtE,CAAC;QAED,MAAM,SAAS,GAAqB;YAClC,IAAI,EAAE,aAAa,CAAC,OAAO;YAC3B,OAAO;YACP,SAAS;SACV,CAAC;QACF,IAAI,CAAC,GAAG,CAAC,cAAc,EAAE,eAAe,EAAE,SAAS,CAAC,CAAC;QACrD,IAAI,CAAC,OAAO,EAAE,cAAc,CAAC,kBAAkB,CAAC,GAAG,EAAE,CAAC;IACxD,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,GAAG,CAAC,cAAqB,EAAE,eAAwB;QACvD,MAAM,SAAS,GAAG,IAAI,CAAC,0BAA0B,CAAC,YAAY,CAAC,cAAc,CAAC,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC;QACpG,IAAI,SAAS,KAAK,SAAS,EAAE,CAAC;YAC5B,OAAO,IAAI,CAAC;QACd,CAAC;QAED,IAAI,oBAAoB,CAAC,SAAS,CAAC,EAAE,CAAC;YACpC,OAAO,SAAS,CAAC,SAAS,CAAC;QAC7B,CAAC;aAAM,CAAC;YACN,UAAU;YACV,OAAO,SAAS,CAAC,OAAO,CAAC;QAC3B,CAAC;IACH,CAAC;IAED;;OAEG;IACH,OAAO,CAAC,cAAqB,EAAE,eAAwB;QACrD,MAAM,SAAS,GAAG,IAAI,CAAC,0BAA0B,CAAC,YAAY,CAAC,cAAc,CAAC,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC;QACpG,IAAI,SAAS,KAAK,SAAS,EAAE,CAAC;YAC5B,OAAO,IAAI,CAAC;QACd,CAAC;QAED,IAAI,oBAAoB,CAAC,SAAS,CAAC,EAAE,CAAC;YACpC,OAAO,SAAS,CAAC,SAAS,CAAC;QAC7B,CAAC;QAED,iBAAiB;QACjB,OAAO,IAAI,CAAC;IACd,CAAC;IAEO,GAAG,CAAC,cAAqB,EAAE,aAAsB,EAAE,SAAoB;QAC7E,IAAI,CAAC,0BAA0B,CAAC,YAAY,CAAC,cAAc,CAAC,CAAC,GAAG,CAAC,aAAa,EAAE,SAAS,CAAC,CAAC;QAC3F,aAAa,CAAC,IAAI,CAAC,0BAA0B,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;IACjE,CAAC;CACF;AAED,SAAS,oBAAoB,CAAC,IAAe;IAC3C,OAAO,IAAI,CAAC,IAAI,KAAK,aAAa,CAAC,SAAS,CAAC;AAC/C,CAAC;AAED,SAAS,kBAAkB,CAAC,IAAe;IACzC,OAAO,IAAI,CAAC,IAAI,KAAK,aAAa,CAAC,OAAO,CAAC;AAC7C,CAAC;AAED;;;;GAIG;AACH,SAAS,gBAAgB,CAAC,KAAgC,EAAE,KAAY;IACtE,OAAO,KAAK,CAAC,IAAI,GAAG,YAAY;QAC9B,CAAC,CAAC,yBAAyB,CAAC,KAAK,EAAE,KAAK,CAAC;QACzC,CAAC,CAAC,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,iBAAiB,CAAC,YAAY,CAAC,uBAAuB,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC;AACrH,CAAC"}
+{"version":3,"file":"shufflingCache.js","sourceRoot":"","sources":["../../src/chain/shufflingCache.ts"],"names":[],"mappings":"AAAA,OAAO,EAKL,qBAAqB,GACtB,MAAM,4BAA4B,CAAC;AAEpC,OAAO,EAAC,aAAa,EAAU,MAAM,EAAE,aAAa,EAAC,MAAM,iBAAiB,CAAC;AAE7E,OAAO,EAAC,mBAAmB,EAAC,MAAM,sBAAsB,CAAC;AAEzD;;;;;IAKI;AACJ,MAAM,UAAU,GAAG,CAAC,CAAC;AAErB;;;GAGG;AACH,MAAM,YAAY,GAAG,CAAC,CAAC;AAEvB,IAAK,aAGJ;AAHD,WAAK,aAAa;IAChB,2DAAS,CAAA;IACT,uDAAO,CAAA;AACT,CAAC,EAHI,aAAa,KAAb,aAAa,QAGjB;AAoBD;;;;;GAKG;AACH,MAAM,OAAO,cAAc;IAQzB,YACW,UAA0B,IAAI,EAC9B,SAAwB,IAAI,EACrC,OAA2B,EAAE,EAC7B,uBAAqF;QAH5E,YAAO,GAAP,OAAO,CAAuB;QAC9B,WAAM,GAAN,MAAM,CAAsB;QATvC,uEAAuE;QACtD,+BAA0B,GAA2C,IAAI,MAAM,CAC9F,GAAG,EAAE,CAAC,IAAI,GAAG,EAAsB,CACpC,CAAC;QAUA,IAAI,OAAO,EAAE,CAAC;YACZ,OAAO,CAAC,cAAc,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,EAAE,CAC1C,OAAO,CAAC,cAAc,CAAC,IAAI,CAAC,GAAG,CAC7B,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,0BAA0B,CAAC,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,QAAQ,EAAE,EAAE,CAAC,KAAK,GAAG,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC,CAC3G,CACF,CAAC;QACJ,CAAC;QAED,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,uBAAuB,IAAI,UAAU,CAAC;QAE5D,uBAAuB,EAAE,GAAG,CAAC,CAAC,EAAC,SAAS,EAAE,YAAY,EAAC,EAAE,EAAE;YACzD,IAAI,SAAS,KAAK,IAAI,EAAE,CAAC;gBACvB,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,YAAY,CAAC,CAAC;YACpC,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC;IAED;;;OAGG;IACH,aAAa,CAAC,KAAY,EAAE,YAAqB;QAC/C,MAAM,YAAY,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,0BAA0B,CAAC,MAAM,EAAE,CAAC;aACtE,OAAO,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;aACpD,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,kBAAkB,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC;QACrD,IAAI,YAAY,IAAI,YAAY,EAAE,CAAC;YACjC,MAAM,IAAI,KAAK,CACb,gCAAgC,YAAY,qBAAqB,KAAK,sBAAsB,YAAY,EAAE,CAC3G,CAAC;QACJ,CAAC;QACD,IAAI,SAAS,GAAiD,IAAI,CAAC;QACnE,MAAM,OAAO,GAAG,IAAI,OAAO,CAAiB,CAAC,OAAO,EAAE,EAAE;YACtD,SAAS,GAAG,OAAO,CAAC;QACtB,CAAC,CAAC,CAAC;QACH,IAAI,SAAS,KAAK,IAAI,EAAE,CAAC;YACvB,MAAM,IAAI,KAAK,CAAC,kDAAkD,CAAC,CAAC;QACtE,CAAC;QAED,MAAM,SAAS,GAAqB;YAClC,IAAI,EAAE,aAAa,CAAC,OAAO;YAC3B,cAAc,EAAE,IAAI,CAAC,GAAG,EAAE;YAC1B,OAAO;YACP,SAAS;SACV,CAAC;QACF,IAAI,CAAC,0BAA0B,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,YAAY,EAAE,SAAS,CAAC,CAAC;QACjF,IAAI,CAAC,OAAO,EAAE,cAAc,CAAC,kBAAkB,CAAC,GAAG,EAAE,CAAC;IACxD,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,GAAG,CAAC,KAAY,EAAE,YAAqB;QAC3C,MAAM,SAAS,GAAG,IAAI,CAAC,0BAA0B,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;QACxF,IAAI,SAAS,KAAK,SAAS,EAAE,CAAC;YAC5B,IAAI,CAAC,OAAO,EAAE,cAAc,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC;YACxC,OAAO,IAAI,CAAC;QACd,CAAC;QAED,IAAI,oBAAoB,CAAC,SAAS,CAAC,EAAE,CAAC;YACpC,IAAI,CAAC,OAAO,EAAE,cAAc,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC;YACvC,OAAO,SAAS,CAAC,SAAS,CAAC;QAC7B,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,OAAO,EAAE,cAAc,CAAC,2BAA2B,CAAC,GAAG,EAAE,CAAC;YAC/D,OAAO,SAAS,CAAC,OAAO,CAAC;QAC3B,CAAC;IACH,CAAC;IAED;;;;;;OAMG;IACH,OAAO,CACL,KAAY,EACZ,YAAqB,EACrB,UAAc;QAEd,MAAM,SAAS,GAAG,IAAI,CAAC,0BAA0B,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;QACxF,IAAI,CAAC,SAAS,EAAE,CAAC;YACf,IAAI,CAAC,OAAO,EAAE,cAAc,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC;QAC1C,CAAC;aAAM,IAAI,oBAAoB,CAAC,SAAS,CAAC,EAAE,CAAC;YAC3C,IAAI,CAAC,OAAO,EAAE,cAAc,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC;YACvC,OAAO,SAAS,CAAC,SAAS,CAAC;QAC7B,CAAC;aAAM,IAAI,UAAU,EAAE,CAAC;YACtB,4DAA4D;YAC5D,IAAI,CAAC,OAAO,EAAE,cAAc,CAAC,wCAAwC,CAAC,GAAG,EAAE,CAAC;QAC9E,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,OAAO,EAAE,cAAc,CAAC,2BAA2B,CAAC,GAAG,EAAE,CAAC;QACjE,CAAC;QAED,IAAI,SAAS,GAA0B,IAAI,CAAC;QAC5C,IAAI,UAAU,EAAE,CAAC;YACf,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,EAAE,cAAc,CAAC,wBAAwB,CAAC,UAAU,CAAC,EAAC,MAAM,EAAE,SAAS,EAAC,CAAC,CAAC;YACpG,SAAS,GAAG,qBAAqB,CAAC,UAAU,CAAC,KAAK,EAAE,UAAU,CAAC,aAAa,EAAE,KAAK,CAAC,CAAC;YACrF,KAAK,EAAE,EAAE,CAAC;YACV,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,YAAY,CAAC,CAAC;QACpC,CAAC;QACD,OAAO,SAAmF,CAAC;IAC7F,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,KAAa,EAAE,YAAoB,EAAE,KAA0B,EAAE,aAA0B;QAC/F,IAAI,CAAC,aAAa,CAAC,KAAK,EAAE,YAAY,CAAC,CAAC;QACxC;;;WAGG;QACH,mBAAmB,CAAC,GAAG,EAAE;YACvB,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,EAAE,cAAc,CAAC,wBAAwB,CAAC,UAAU,CAAC,EAAC,MAAM,EAAE,OAAO,EAAC,CAAC,CAAC;YAClG,MAAM,SAAS,GAAG,qBAAqB,CAAC,KAAK,EAAE,aAAa,EAAE,KAAK,CAAC,CAAC;YACrE,KAAK,EAAE,EAAE,CAAC;YACV,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,YAAY,CAAC,CAAC;QACpC,CAAC,CAAC,CAAC;IACL,CAAC;IAED;;;OAGG;IACK,GAAG,CAAC,SAAyB,EAAE,YAAoB;QACzD,MAAM,gBAAgB,GAAG,IAAI,CAAC,0BAA0B,CAAC,YAAY,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;QACvF,oDAAoD;QACpD,MAAM,SAAS,GAAG,gBAAgB,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;QACrD,IAAI,SAAS,EAAE,CAAC;YACd,IAAI,kBAAkB,CAAC,SAAS,CAAC,EAAE,CAAC;gBAClC,SAAS,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC;gBAC/B,IAAI,CAAC,OAAO,EAAE,cAAc,CAAC,8BAA8B,CAAC,OAAO,CACjE,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC,cAAc,CAAC,GAAG,IAAI,CAC/C,CAAC;YACJ,CAAC;iBAAM,CAAC;gBACN,IAAI,CAAC,OAAO,EAAE,cAAc,CAAC,2BAA2B,CAAC,GAAG,EAAE,CAAC;YACjE,CAAC;QACH,CAAC;QACD,oBAAoB;QACpB,gBAAgB,CAAC,GAAG,CAAC,YAAY,EAAE,EAAC,IAAI,EAAE,aAAa,CAAC,SAAS,EAAE,SAAS,EAAC,CAAC,CAAC;QAC/E,kBAAkB;QAClB,aAAa,CAAC,IAAI,CAAC,0BAA0B,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;IACjE,CAAC;CACF;AAED,SAAS,oBAAoB,CAAC,IAAe;IAC3C,OAAO,IAAI,CAAC,IAAI,KAAK,aAAa,CAAC,SAAS,CAAC;AAC/C,CAAC;AAED,SAAS,kBAAkB,CAAC,IAAe;IACzC,OAAO,IAAI,CAAC,IAAI,KAAK,aAAa,CAAC,OAAO,CAAC;AAC7C,CAAC;AAED,MAAM,CAAN,IAAY,uBAEX;AAFD,WAAY,uBAAuB;IACjC,0FAA+D,CAAA;AACjE,CAAC,EAFW,uBAAuB,KAAvB,uBAAuB,QAElC;AAQD,MAAM,OAAO,mBAAoB,SAAQ,aAAsC;CAAG"}

package/lib/chain/stateCache/datastore/file.js

@@ -1,6 +1,6 @@
 import path from "node:path";
-import { toHexString, fromHexString } from "@chainsafe/ssz";
 import { ssz } from "@lodestar/types";
+import { fromHex, toHex } from "@lodestar/utils";
 import { ensureDir, readFile, readFileNames, removeFile, writeIfNotExist } from "../../../util/file.js";
 const CHECKPOINT_STATES_FOLDER = "checkpoint_states";
 const CHECKPOINT_FILE_NAME_LENGTH = 82;
@@ -22,23 +22,23 @@ export class FileCPStateDatastore {
     }
     async write(cpKey, stateBytes) {
         const serializedCheckpoint = ssz.phase0.Checkpoint.serialize(cpKey);
-        const filePath = path.join(this.folderPath, toHexString(serializedCheckpoint));
+        const filePath = path.join(this.folderPath, toHex(serializedCheckpoint));
         await writeIfNotExist(filePath, stateBytes);
         return serializedCheckpoint;
     }
     async remove(serializedCheckpoint) {
-        const filePath = path.join(this.folderPath, toHexString(serializedCheckpoint));
+        const filePath = path.join(this.folderPath, toHex(serializedCheckpoint));
         await removeFile(filePath);
     }
     async read(serializedCheckpoint) {
-        const filePath = path.join(this.folderPath, toHexString(serializedCheckpoint));
+        const filePath = path.join(this.folderPath, toHex(serializedCheckpoint));
         return readFile(filePath);
     }
     async readKeys() {
         const fileNames = await readFileNames(this.folderPath);
         return fileNames
             .filter((fileName) => fileName.startsWith("0x") && fileName.length === CHECKPOINT_FILE_NAME_LENGTH)
-            .map((fileName) => fromHexString(fileName));
+            .map((fileName) => fromHex(fileName));
     }
 }
 //# sourceMappingURL=file.js.map

package/lib/chain/stateCache/datastore/file.js.map

@@ -1 +1 @@
-{"version":3,"file":"file.js","sourceRoot":"","sources":["../../../../src/chain/stateCache/datastore/file.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAC,WAAW,EAAE,aAAa,EAAC,MAAM,gBAAgB,CAAC;AAC1D,OAAO,EAAS,GAAG,EAAC,MAAM,iBAAiB,CAAC;AAC5C,OAAO,EAAC,SAAS,EAAE,QAAQ,EAAE,aAAa,EAAE,UAAU,EAAE,eAAe,EAAC,MAAM,uBAAuB,CAAC;AAGtG,MAAM,wBAAwB,GAAG,mBAAmB,CAAC;AACrD,MAAM,2BAA2B,GAAG,EAAE,CAAC;AAEvC;;GAEG;AACH,MAAM,OAAO,oBAAoB;IAG/B,YAAY,YAAoB,GAAG;QACjC,+DAA+D;QAC/D,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,wBAAwB,CAAC,CAAC;IACnE,CAAC;IAED,KAAK,CAAC,IAAI;QACR,IAAI,CAAC;YACH,MAAM,SAAS,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QACnC,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,aAAa;QACf,CAAC;IACH,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,KAAwB,EAAE,UAAsB;QAC1D,MAAM,oBAAoB,GAAG,GAAG,CAAC,MAAM,CAAC,UAAU,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;QACpE,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,WAAW,CAAC,oBAAoB,CAAC,CAAC,CAAC;QAC/E,MAAM,eAAe,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;QAC5C,OAAO,oBAAoB,CAAC;IAC9B,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,oBAAkC;QAC7C,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,WAAW,CAAC,oBAAoB,CAAC,CAAC,CAAC;QAC/E,MAAM,UAAU,CAAC,QAAQ,CAAC,CAAC;IAC7B,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,oBAAkC;QAC3C,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,WAAW,CAAC,oBAAoB,CAAC,CAAC,CAAC;QAC/E,OAAO,QAAQ,CAAC,QAAQ,CAAC,CAAC;IAC5B,CAAC;IAED,KAAK,CAAC,QAAQ;QACZ,MAAM,SAAS,GAAG,MAAM,aAAa,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QACvD,OAAO,SAAS;aACb,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,QAAQ,CAAC,MAAM,KAAK,2BAA2B,CAAC;aAClG,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC,CAAC;IAChD,CAAC;CACF"}
+{"version":3,"file":"file.js","sourceRoot":"","sources":["../../../../src/chain/stateCache/datastore/file.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAS,GAAG,EAAC,MAAM,iBAAiB,CAAC;AAC5C,OAAO,EAAC,OAAO,EAAE,KAAK,EAAC,MAAM,iBAAiB,CAAC;AAC/C,OAAO,EAAC,SAAS,EAAE,QAAQ,EAAE,aAAa,EAAE,UAAU,EAAE,eAAe,EAAC,MAAM,uBAAuB,CAAC;AAGtG,MAAM,wBAAwB,GAAG,mBAAmB,CAAC;AACrD,MAAM,2BAA2B,GAAG,EAAE,CAAC;AAEvC;;GAEG;AACH,MAAM,OAAO,oBAAoB;IAG/B,YAAY,YAAoB,GAAG;QACjC,+DAA+D;QAC/D,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,wBAAwB,CAAC,CAAC;IACnE,CAAC;IAED,KAAK,CAAC,IAAI;QACR,IAAI,CAAC;YACH,MAAM,SAAS,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QACnC,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,aAAa;QACf,CAAC;IACH,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,KAAwB,EAAE,UAAsB;QAC1D,MAAM,oBAAoB,GAAG,GAAG,CAAC,MAAM,CAAC,UAAU,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;QACpE,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,KAAK,CAAC,oBAAoB,CAAC,CAAC,CAAC;QACzE,MAAM,eAAe,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;QAC5C,OAAO,oBAAoB,CAAC;IAC9B,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,oBAAkC;QAC7C,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,KAAK,CAAC,oBAAoB,CAAC,CAAC,CAAC;QACzE,MAAM,UAAU,CAAC,QAAQ,CAAC,CAAC;IAC7B,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,oBAAkC;QAC3C,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,KAAK,CAAC,oBAAoB,CAAC,CAAC,CAAC;QACzE,OAAO,QAAQ,CAAC,QAAQ,CAAC,CAAC;IAC5B,CAAC;IAED,KAAK,CAAC,QAAQ;QACZ,MAAM,SAAS,GAAG,MAAM,aAAa,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QACvD,OAAO,SAAS;aACb,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,QAAQ,CAAC,MAAM,KAAK,2BAA2B,CAAC;aAClG,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC;IAC1C,CAAC;CACF"}

package/lib/chain/stateCache/fifoBlockStateCache.d.ts

@@ -8,9 +8,14 @@ export type FIFOBlockStateCacheOpts = {
     maxBlockStates?: number;
 };
 /**
- * Regen state if there's a reorg distance > 32 slots.
+ * Given `maxSkipSlots` = 32 and `DEFAULT_EARLIEST_PERMISSIBLE_SLOT_DISTANCE` = 32, lodestar doesn't need to
+ * reload states in order to process a gossip block.
+ *
+ * |-----------------------------------------------|-----------------------------------------------|
+ *                 maxSkipSlots                      DEFAULT_EARLIEST_PERMISSIBLE_SLOT_DISTANCE    ^
+ *                                                                                             clock slot
  */
-export declare const DEFAULT_MAX_BLOCK_STATES = 32;
+export declare const DEFAULT_MAX_BLOCK_STATES = 64;
 /**
  * New implementation of BlockStateCache that keeps the most recent n states consistently
  *  - Maintain a linked list (FIFO) with special handling for head state, which is always the first item in the list

package/lib/chain/stateCache/fifoBlockStateCache.js

@@ -2,9 +2,14 @@ import { toRootHex } from "@lodestar/utils";
 import { LinkedList } from "../../util/array.js";
 import { MapTracker } from "./mapMetrics.js";
 /**
- * Regen state if there's a reorg distance > 32 slots.
+ * Given `maxSkipSlots` = 32 and `DEFAULT_EARLIEST_PERMISSIBLE_SLOT_DISTANCE` = 32, lodestar doesn't need to
+ * reload states in order to process a gossip block.
+ *
+ * |-----------------------------------------------|-----------------------------------------------|
+ *                 maxSkipSlots                      DEFAULT_EARLIEST_PERMISSIBLE_SLOT_DISTANCE    ^
+ *                                                                                             clock slot
  */
-export const DEFAULT_MAX_BLOCK_STATES = 32;
+export const DEFAULT_MAX_BLOCK_STATES = 64;
 /**
  * New implementation of BlockStateCache that keeps the most recent n states consistently
  *  - Maintain a linked list (FIFO) with special handling for head state, which is always the first item in the list

package/lib/chain/stateCache/fifoBlockStateCache.js.map

@@ -1 +1 @@
-{"version":3,"file":"fifoBlockStateCache.js","sourceRoot":"","sources":["../../../src/chain/stateCache/fifoBlockStateCache.ts"],"names":[],"mappings":"AAGA,OAAO,EAAC,SAAS,EAAC,MAAM,iBAAiB,CAAC;AAE1C,OAAO,EAAC,UAAU,EAAC,MAAM,qBAAqB,CAAC;AAE/C,OAAO,EAAC,UAAU,EAAC,MAAM,iBAAiB,CAAC;AAO3C;;GAEG;AACH,MAAM,CAAC,MAAM,wBAAwB,GAAG,EAAE,CAAC;AAE3C;;;;;;;;;;;;;;;;;;GAkBG;AACH,MAAM,OAAO,mBAAmB;IAa9B,YAAY,IAA6B,EAAE,EAAC,OAAO,EAA6B;QAC9E,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,cAAc,IAAI,wBAAwB,CAAC;QACjE,IAAI,CAAC,KAAK,GAAG,IAAI,UAAU,CAAC,OAAO,EAAE,UAAU,CAAC,CAAC;QACjD,IAAI,OAAO,EAAE,CAAC;YACZ,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,UAAU,CAAC;YAClC,OAAO,CAAC,UAAU,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;QACzF,CAAC;QACD,IAAI,CAAC,QAAQ,GAAG,IAAI,UAAU,EAAE,CAAC;IACnC,CAAC;IAED;;OAEG;IACH,YAAY,CAAC,IAAsC;QACjD,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;YAClB,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;QACvB,CAAC;IACH,CAAC;IAED;;;;OAIG;IACH,YAAY;QACV,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC;QAC9C,IAAI,UAAU,CAAC,IAAI,EAAE,CAAC;YACpB,oBAAoB;YACpB,MAAM,KAAK,CAAC,iCAAiC,CAAC,CAAC;QACjD,CAAC;QAED,MAAM,UAAU,GAAG,UAAU,CAAC,KAAK,CAAC;QACpC,8FAA8F;QAC9F,OAAO,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAChC,CAAC;IAED;;OAEG;IACH,GAAG,CAAC,OAAgB,EAAE,IAAqB;QACzC,IAAI,CAAC,OAAO,EAAE,OAAO,CAAC,GAAG,EAAE,CAAC;QAC5B,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QACrC,IAAI,CAAC,IAAI,EAAE,CAAC;YACV,OAAO,IAAI,CAAC;QACd,CAAC;QAED,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC;QACzB,IAAI,CAAC,OAAO,EAAE,gBAAgB,CAAC,OAAO,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QAEzD,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,iBAAiB,CAAC,CAAC;IAC7C,CAAC;IAED;;;;OAIG;IACH,GAAG,CAAC,IAA+B,EAAE,MAAM,GAAG,KAAK;QACjD,MAAM,GAAG,GAAG,SAAS,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC,CAAC;QAC3C,IAAI,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE,CAAC;YAChC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;gBAC5B,MAAM,KAAK,CAAC,+CAA+C,GAAG,EAAE,CAAC,CAAC;YACpE,CAAC;YACD,IAAI,MAAM,EAAE,CAAC;gBACX,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;YAChC,CAAC;iBAAM,CAAC;gBACN,IAAI,CAAC,QAAQ,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC;YAClC,CAAC;YACD,sBAAsB;YACtB,OAAO;QACT,CAAC;QAED,YAAY;QACZ,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC;QACzB,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;QAC1B,IAAI,MAAM,EAAE,CAAC;YACX,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QAC7B,CAAC;aAAM,CAAC;YACN,oBAAoB;YACpB,MAAM,IAAI,GAAG,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAC;YACnC,IAAI,IAAI,IAAI,IAAI,EAAE,CAAC;gBACjB,iDAAiD;gBACjD,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;YAC7B,CAAC;iBAAM,CAAC;gBACN,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;YACvC,CAAC;QACH,CAAC;QACD,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAClB,CAAC;IAED,IAAI,IAAI;QACN,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC;IACzB,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,YAAoB;QACxB,OAAO,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC;YAC7C,MAAM,GAAG,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;YACjC,uDAAuD;YACvD,kEAAkE;YAClE,IAAI,GAAG,KAAK,YAAY,EAAE,CAAC;gBACzB,MAAM;YACR,CAAC;YACD,IAAI,CAAC,GAAG,EAAE,CAAC;gBACT,oBAAoB;gBACpB,MAAM,IAAI,KAAK,CAAC,QAAQ,CAAC,CAAC;YAC5B,CAAC;YACD,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,CAAC;YACpB,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QACzB,CAAC;IACH,CAAC;IAED;;;OAGG;IACH,oBAAoB,KAAU,CAAC;IAE/B;;OAEG;IACH,KAAK;QACH,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;IACrB,CAAC;IAED,0DAA0D;IAC1D,WAAW;QACT,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC;YAC7D,IAAI,EAAE,KAAK,CAAC,IAAI;YAChB,IAAI,EAAE,SAAS,CAAC,KAAK,CAAC,YAAY,EAAE,CAAC;YACrC,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC;YACzC,QAAQ,EAAE,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC;YAC3C,eAAe,EAAE,KAAK;SACvB,CAAC,CAAC,CAAC;IACN,CAAC;IAED,SAAS;QACP,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;IAC7B,CAAC;IAED;;OAEG;IACH,YAAY;QACV,OAAO,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC;IACjC,CAAC;CACF"}
+{"version":3,"file":"fifoBlockStateCache.js","sourceRoot":"","sources":["../../../src/chain/stateCache/fifoBlockStateCache.ts"],"names":[],"mappings":"AAGA,OAAO,EAAC,SAAS,EAAC,MAAM,iBAAiB,CAAC;AAE1C,OAAO,EAAC,UAAU,EAAC,MAAM,qBAAqB,CAAC;AAE/C,OAAO,EAAC,UAAU,EAAC,MAAM,iBAAiB,CAAC;AAO3C;;;;;;;GAOG;AACH,MAAM,CAAC,MAAM,wBAAwB,GAAG,EAAE,CAAC;AAE3C;;;;;;;;;;;;;;;;;;GAkBG;AACH,MAAM,OAAO,mBAAmB;IAa9B,YAAY,IAA6B,EAAE,EAAC,OAAO,EAA6B;QAC9E,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,cAAc,IAAI,wBAAwB,CAAC;QACjE,IAAI,CAAC,KAAK,GAAG,IAAI,UAAU,CAAC,OAAO,EAAE,UAAU,CAAC,CAAC;QACjD,IAAI,OAAO,EAAE,CAAC;YACZ,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,UAAU,CAAC;YAClC,OAAO,CAAC,UAAU,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;QACzF,CAAC;QACD,IAAI,CAAC,QAAQ,GAAG,IAAI,UAAU,EAAE,CAAC;IACnC,CAAC;IAED;;OAEG;IACH,YAAY,CAAC,IAAsC;QACjD,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;YAClB,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;QACvB,CAAC;IACH,CAAC;IAED;;;;OAIG;IACH,YAAY;QACV,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC;QAC9C,IAAI,UAAU,CAAC,IAAI,EAAE,CAAC;YACpB,oBAAoB;YACpB,MAAM,KAAK,CAAC,iCAAiC,CAAC,CAAC;QACjD,CAAC;QAED,MAAM,UAAU,GAAG,UAAU,CAAC,KAAK,CAAC;QACpC,8FAA8F;QAC9F,OAAO,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAChC,CAAC;IAED;;OAEG;IACH,GAAG,CAAC,OAAgB,EAAE,IAAqB;QACzC,IAAI,CAAC,OAAO,EAAE,OAAO,CAAC,GAAG,EAAE,CAAC;QAC5B,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QACrC,IAAI,CAAC,IAAI,EAAE,CAAC;YACV,OAAO,IAAI,CAAC;QACd,CAAC;QAED,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC;QACzB,IAAI,CAAC,OAAO,EAAE,gBAAgB,CAAC,OAAO,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QAEzD,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,iBAAiB,CAAC,CAAC;IAC7C,CAAC;IAED;;;;OAIG;IACH,GAAG,CAAC,IAA+B,EAAE,MAAM,GAAG,KAAK;QACjD,MAAM,GAAG,GAAG,SAAS,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC,CAAC;QAC3C,IAAI,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE,CAAC;YAChC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;gBAC5B,MAAM,KAAK,CAAC,+CAA+C,GAAG,EAAE,CAAC,CAAC;YACpE,CAAC;YACD,IAAI,MAAM,EAAE,CAAC;gBACX,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;YAChC,CAAC;iBAAM,CAAC;gBACN,IAAI,CAAC,QAAQ,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC;YAClC,CAAC;YACD,sBAAsB;YACtB,OAAO;QACT,CAAC;QAED,YAAY;QACZ,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC;QACzB,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;QAC1B,IAAI,MAAM,EAAE,CAAC;YACX,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QAC7B,CAAC;aAAM,CAAC;YACN,oBAAoB;YACpB,MAAM,IAAI,GAAG,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAC;YACnC,IAAI,IAAI,IAAI,IAAI,EAAE,CAAC;gBACjB,iDAAiD;gBACjD,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;YAC7B,CAAC;iBAAM,CAAC;gBACN,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;YACvC,CAAC;QACH,CAAC;QACD,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAClB,CAAC;IAED,IAAI,IAAI;QACN,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC;IACzB,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,YAAoB;QACxB,OAAO,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC;YAC7C,MAAM,GAAG,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;YACjC,uDAAuD;YACvD,kEAAkE;YAClE,IAAI,GAAG,KAAK,YAAY,EAAE,CAAC;gBACzB,MAAM;YACR,CAAC;YACD,IAAI,CAAC,GAAG,EAAE,CAAC;gBACT,oBAAoB;gBACpB,MAAM,IAAI,KAAK,CAAC,QAAQ,CAAC,CAAC;YAC5B,CAAC;YACD,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,CAAC;YACpB,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QACzB,CAAC;IACH,CAAC;IAED;;;OAGG;IACH,oBAAoB,KAAU,CAAC;IAE/B;;OAEG;IACH,KAAK;QACH,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;IACrB,CAAC;IAED,0DAA0D;IAC1D,WAAW;QACT,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC;YAC7D,IAAI,EAAE,KAAK,CAAC,IAAI;YAChB,IAAI,EAAE,SAAS,CAAC,KAAK,CAAC,YAAY,EAAE,CAAC;YACrC,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC;YACzC,QAAQ,EAAE,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC;YAC3C,eAAe,EAAE,KAAK;SACvB,CAAC,CAAC,CAAC;IACN,CAAC;IAED,SAAS;QACP,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;IAC7B,CAAC;IAED;;OAEG;IACH,YAAY;QACV,OAAO,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC;IACjC,CAAC;CACF"}

package/lib/chain/stateCache/persistentCheckpointsCache.d.ts

@@ -30,10 +30,11 @@ type LoadedStateBytesData = {
     stateBytes: Uint8Array;
 };
 /**
- * Before n-historical states, lodestar keeps mostly 3 states in memory with 1 finalized state
- * Since Jan 2024, lodestar stores the finalized state in disk and keeps up to 2 epochs in memory
+ * Before n-historical states, lodestar keeps all checkpoint states since finalized
+ * Since Sep 2024, lodestar stores 3 most recent checkpoint states in memory and the rest on disk. The finalized state
+ * may not be available in memory, and stay on disk instead.
  */
-export declare const DEFAULT_MAX_CP_STATE_EPOCHS_IN_MEMORY = 2;
+export declare const DEFAULT_MAX_CP_STATE_EPOCHS_IN_MEMORY = 3;
 /**
  * An implementation of CheckpointStateCache that keep up to n epoch checkpoint states in memory and persist the rest to disk
  * - If it's more than `maxEpochsInMemory` epochs old, it will persist n last epochs to disk based on the view of the block

package/lib/chain/stateCache/persistentCheckpointsCache.js

@@ -43,9 +43,8 @@ var __disposeResources = (this && this.__disposeResources) || (function (Suppres
     var e = new Error(message);
     return e.name = "SuppressedError", e.error = error, e.suppressed = suppressed, e;
 });
-import { fromHexString, toHexString } from "@chainsafe/ssz";
 import { computeStartSlotAtEpoch, getBlockRootAtSlot } from "@lodestar/state-transition";
-import { MapDef, sleep, toRootHex } from "@lodestar/utils";
+import { MapDef, fromHex, sleep, toHex, toRootHex } from "@lodestar/utils";
 import { loadCachedBeaconState } from "@lodestar/state-transition";
 import { INTERVALS_PER_SLOT } from "@lodestar/params";
 import { AllocSource } from "../../util/bufferPool.js";
@@ -54,10 +53,11 @@ import { MapTracker } from "./mapMetrics.js";
 import { datastoreKeyToCheckpoint } from "./datastore/index.js";
 import { CacheItemType } from "./types.js";
 /**
- * Before n-historical states, lodestar keeps mostly 3 states in memory with 1 finalized state
- * Since Jan 2024, lodestar stores the finalized state in disk and keeps up to 2 epochs in memory
+ * Before n-historical states, lodestar keeps all checkpoint states since finalized
+ * Since Sep 2024, lodestar stores 3 most recent checkpoint states in memory and the rest on disk. The finalized state
+ * may not be available in memory, and stay on disk instead.
  */
-export const DEFAULT_MAX_CP_STATE_EPOCHS_IN_MEMORY = 2;
+export const DEFAULT_MAX_CP_STATE_EPOCHS_IN_MEMORY = 3;
 /**
  * An implementation of CheckpointStateCache that keep up to n epoch checkpoint states in memory and persist the rest to disk
  * - If it's more than `maxEpochsInMemory` epochs old, it will persist n last epochs to disk based on the view of the block
@@ -98,7 +98,7 @@ export class PersistentCheckpointStateCache {
         this.preComputedCheckpointHits = null;
         this.cache = new MapTracker(metrics?.cpStateCache);
         if (metrics) {
-            this.metrics = metrics.cpStateCache;
+            this.metrics = metrics;
             metrics.cpStateCache.size.addCollect(() => {
                 let persistCount = 0;
                 let inMemoryCount = 0;
@@ -166,36 +166,28 @@ export class PersistentCheckpointStateCache {
             return stateOrStateBytesData?.clone(opts?.dontTransferCache) ?? null;
         }
         const { persistedKey, stateBytes } = stateOrStateBytesData;
-        const logMeta = { persistedKey: toHexString(persistedKey) };
+        const logMeta = { persistedKey: toHex(persistedKey) };
         this.logger.debug("Reload: read state successful", logMeta);
-        this.metrics?.stateReloadSecFromSlot.observe(this.clock?.secFromSlot(this.clock?.currentSlot ?? 0) ?? 0);
+        this.metrics?.cpStateCache.stateReloadSecFromSlot.observe(this.clock?.secFromSlot(this.clock?.currentSlot ?? 0) ?? 0);
         const seedState = this.findSeedStateToReload(cp);
-        this.metrics?.stateReloadEpochDiff.observe(Math.abs(seedState.epochCtx.epoch - cp.epoch));
+        this.metrics?.cpStateCache.stateReloadEpochDiff.observe(Math.abs(seedState.epochCtx.epoch - cp.epoch));
         this.logger.debug("Reload: found seed state", { ...logMeta, seedSlot: seedState.slot });
         try {
             const env_1 = { stack: [], error: void 0, hasError: false };
             try {
                 // 80% of validators serialization time comes from memory allocation, this is to avoid it
-                const sszTimer = this.metrics?.stateReloadValidatorsSerializeDuration.startTimer();
+                const sszTimer = this.metrics?.cpStateCache.stateReloadValidatorsSerializeDuration.startTimer();
                 // automatically free the buffer pool after this scope
                 const validatorsBytesWithKey = __addDisposableResource(env_1, this.serializeStateValidators(seedState), false);
                 let validatorsBytes = validatorsBytesWithKey?.buffer;
                 if (validatorsBytes == null) {
                     // fallback logic in case we can't use the buffer pool
-                    this.metrics?.stateReloadValidatorsSerializeAllocCount.inc();
+                    this.metrics?.cpStateCache.stateReloadValidatorsSerializeAllocCount.inc();
                     validatorsBytes = seedState.validators.serialize();
                 }
                 sszTimer?.();
-                const timer = this.metrics?.stateReloadDuration.startTimer();
-                const newCachedState = loadCachedBeaconState(seedState, stateBytes, {
-                    shufflingGetter: (shufflingEpoch, decisionRootHex) => {
-                        const shuffling = this.shufflingCache.getSync(shufflingEpoch, decisionRootHex);
-                        if (shuffling == null) {
-                            this.metrics?.stateReloadShufflingCacheMiss.inc();
-                        }
-                        return shuffling;
-                    },
-                }, validatorsBytes);
+                const timer = this.metrics?.cpStateCache.stateReloadDuration.startTimer();
+                const newCachedState = loadCachedBeaconState(seedState, stateBytes, {}, validatorsBytes);
                 newCachedState.commit();
                 const stateRoot = toRootHex(newCachedState.hashTreeRoot());
                 timer?.();
@@ -257,7 +249,7 @@ export class PersistentCheckpointStateCache {
             throw new Error("Expected persistent key");
         }
         const persistedKey = cacheItem.value;
-        const dbReadTimer = this.metrics?.stateReloadDbReadTime.startTimer();
+        const dbReadTimer = this.metrics?.cpStateCache.stateReloadDbReadTime.startTimer();
         const stateBytes = await this.datastore.read(persistedKey);
         dbReadTimer?.();
         if (stateBytes === null) {
@@ -269,19 +261,19 @@ export class PersistentCheckpointStateCache {
      * Similar to get() api without reloading from disk
      */
     get(cpOrKey, opts) {
-        this.metrics?.lookups.inc();
+        this.metrics?.cpStateCache.lookups.inc();
         const cpKey = typeof cpOrKey === "string" ? cpOrKey : toCacheKey(cpOrKey);
         const cacheItem = this.cache.get(cpKey);
         if (cacheItem === undefined) {
             return null;
         }
-        this.metrics?.hits.inc();
+        this.metrics?.cpStateCache.hits.inc();
         if (cpKey === this.preComputedCheckpoint) {
             this.preComputedCheckpointHits = (this.preComputedCheckpointHits ?? 0) + 1;
         }
         if (isInMemoryCacheItem(cacheItem)) {
             const { state } = cacheItem;
-            this.metrics?.stateClonedCount.observe(state.clonedCount);
+            this.metrics?.cpStateCache.stateClonedCount.observe(state.clonedCount);
             return state.clone(opts?.dontTransferCache);
         }
         return null;
@@ -293,7 +285,7 @@ export class PersistentCheckpointStateCache {
         const cpHex = toCheckpointHex(cp);
         const key = toCacheKey(cpHex);
         const cacheItem = this.cache.get(key);
-        this.metrics?.adds.inc();
+        this.metrics?.cpStateCache.adds.inc();
         if (cacheItem !== undefined && isPersistedCacheItem(cacheItem)) {
             const persistedKey = cacheItem.value;
             // was persisted to disk, set back to memory
@@ -301,7 +293,7 @@ export class PersistentCheckpointStateCache {
             this.logger.verbose("Added checkpoint state to memory but a persisted key existed", {
                 epoch: cp.epoch,
                 rootHex: cpHex.rootHex,
-                persistedKey: toHexString(persistedKey),
+                persistedKey: toHex(persistedKey),
             });
         }
         else {
@@ -588,7 +580,7 @@ export class PersistentCheckpointStateCache {
     async processPastEpoch(blockRootHex, state, epoch) {
         let persistCount = 0;
         const epochBoundarySlot = computeStartSlotAtEpoch(epoch);
-        const epochBoundaryRoot = epochBoundarySlot === state.slot ? fromHexString(blockRootHex) : getBlockRootAtSlot(state, epochBoundarySlot);
+        const epochBoundaryRoot = epochBoundarySlot === state.slot ? fromHex(blockRootHex) : getBlockRootAtSlot(state, epochBoundarySlot);
         const epochBoundaryHex = toRootHex(epochBoundaryRoot);
         const prevEpochRoot = toRootHex(getBlockRootAtSlot(state, epochBoundarySlot - 1));
         // for each epoch, usually there are 2 rootHexes respective to the 2 checkpoint states: Previous Root Checkpoint State and Current Root Checkpoint State
@@ -614,7 +606,7 @@ export class PersistentCheckpointStateCache {
                     stateSlot: state.slot,
                     rootHex,
                     epochBoundaryHex,
-                    persistedKey: persistedKey ? toHexString(persistedKey) : "",
+                    persistedKey: persistedKey ? toHex(persistedKey) : "",
                 };
                 if (persistedRootHexes.has(rootHex)) {
                     if (persistedKey) {
@@ -623,21 +615,25 @@ export class PersistentCheckpointStateCache {
                     }
                     else {
                         // persist and do not update epochIndex
-                        this.metrics?.statePersistSecFromSlot.observe(this.clock?.secFromSlot(this.clock?.currentSlot ?? 0) ?? 0);
-                        const cpPersist = { epoch: epoch, root: fromHexString(rootHex) };
+                        this.metrics?.cpStateCache.statePersistSecFromSlot.observe(this.clock?.secFromSlot(this.clock?.currentSlot ?? 0) ?? 0);
+                        const cpPersist = { epoch: epoch, root: fromHex(rootHex) };
                         // It's not sustainable to allocate ~240MB for each state every epoch, so we use buffer pool to reuse the memory.
                         // As monitored on holesky as of Jan 2024:
                         //   - This does not increase heap allocation while gc time is the same
                         //   - It helps stabilize persist time and save ~300ms in average (1.5s vs 1.2s)
                         //   - It also helps the state reload to save ~500ms in average (4.3s vs 3.8s)
                         //   - Also `serializeState.test.ts` perf test shows a lot of differences allocating ~240MB once vs per state serialization
-                        const timer = this.metrics?.stateSerializeDuration.startTimer();
-                        persistedKey = await serializeState(state, AllocSource.PERSISTENT_CHECKPOINTS_CACHE_STATE, (stateBytes) => this.datastore.write(cpPersist, stateBytes), this.bufferPool);
-                        timer?.();
+                        const timer = this.metrics?.stateSerializeDuration.startTimer({
+                            source: AllocSource.PERSISTENT_CHECKPOINTS_CACHE_STATE,
+                        });
+                        persistedKey = await serializeState(state, AllocSource.PERSISTENT_CHECKPOINTS_CACHE_STATE, (stateBytes) => {
+                            timer?.();
+                            return this.datastore.write(cpPersist, stateBytes);
+                        }, this.bufferPool);
                         persistCount++;
                         this.logger.verbose("Pruned checkpoint state from memory and persisted to disk", {
                             ...logMeta,
-                            persistedKey: toHexString(persistedKey),
+                            persistedKey: toHex(persistedKey),
                         });
                     }
                     // overwrite cpKey, this means the state is deleted from memory
@@ -655,7 +651,7 @@ export class PersistentCheckpointStateCache {
                         this.cache.delete(cpKey);
                         this.epochIndex.get(epoch)?.delete(rootHex);
                     }
-                    this.metrics?.statePruneFromMemoryCount.inc();
+                    this.metrics?.cpStateCache.statePruneFromMemoryCount.inc();
                     this.logger.verbose("Pruned checkpoint state from memory", logMeta);
                 }
             }
@@ -676,7 +672,7 @@ export class PersistentCheckpointStateCache {
                 if (persistedKey) {
                     await this.datastore.remove(persistedKey);
                     persistCount++;
-                    this.metrics?.persistedStateRemoveCount.inc();
+                    this.metrics?.cpStateCache.persistedStateRemoveCount.inc();
                 }
             }
             this.cache.delete(key);