@lodestar/beacon-node 1.46.0 → 1.47.0-dev.02d2dbea03

package/package.json

@@ -11,7 +11,7 @@
   "bugs": {
     "url": "https://github.com/ChainSafe/lodestar/issues"
   },
-  "version": "1.46.0",
+  "version": "1.47.0-dev.02d2dbea03",
   "type": "module",
   "exports": {
     ".": {
@@ -100,51 +100,51 @@
     "check-readme": "pnpm exec tsx ../../scripts/check_readme.ts"
   },
   "dependencies": {
-    "@chainsafe/as-sha256": "^1.2.4",
-    "@chainsafe/blst": "^2.2.0",
-    "@chainsafe/discv5": "^12.0.1",
-    "@chainsafe/enr": "^6.0.1",
+    "@chainsafe/as-sha256": "^1.2.5",
+    "@chainsafe/discv5": "^12.0.2",
+    "@chainsafe/enr": "^6.0.2",
     "@chainsafe/libp2p-noise": "^17.0.0",
     "@chainsafe/libp2p-quic": "^2.1.3",
-    "@chainsafe/persistent-merkle-tree": "^1.3.0",
+    "@chainsafe/lodestar-z": "catalog:",
+    "@chainsafe/persistent-merkle-tree": "^1.3.1",
     "@chainsafe/prometheus-gc-stats": "^1.0.0",
     "@chainsafe/snappy-wasm": "^0.5.0",
-    "@chainsafe/ssz": "^1.6.2",
+    "@chainsafe/ssz": "^1.6.3",
     "@chainsafe/threads": "^1.11.3",
     "@crate-crypto/node-eth-kzg": "0.9.1",
     "@fastify/bearer-auth": "^10.0.1",
     "@fastify/cors": "^10.0.1",
     "@fastify/swagger": "^9.0.0",
     "@fastify/swagger-ui": "^5.0.1",
-    "@libp2p/bootstrap": "^12.0.14",
-    "@libp2p/crypto": "^5.1.13",
-    "@libp2p/gossipsub": "^16.0.2",
-    "@libp2p/identify": "^4.0.13",
-    "@libp2p/interface": "^3.1.0",
-    "@libp2p/mdns": "^12.0.14",
-    "@libp2p/mplex": "^12.0.14",
-    "@libp2p/peer-id": "^6.0.4",
-    "@libp2p/prometheus-metrics": "^5.0.14",
-    "@libp2p/tcp": "^11.0.13",
-    "@lodestar/api": "^1.46.0",
-    "@lodestar/config": "^1.46.0",
-    "@lodestar/db": "^1.46.0",
-    "@lodestar/fork-choice": "^1.46.0",
-    "@lodestar/logger": "^1.46.0",
-    "@lodestar/params": "^1.46.0",
-    "@lodestar/reqresp": "^1.46.0",
-    "@lodestar/state-transition": "^1.46.0",
-    "@lodestar/types": "^1.46.0",
-    "@lodestar/utils": "^1.46.0",
-    "@lodestar/validator": "^1.46.0",
+    "@libp2p/bootstrap": "^12.0.30",
+    "@libp2p/crypto": "^5.1.23",
+    "@libp2p/gossipsub": "^16.1.1",
+    "@libp2p/identify": "^4.1.13",
+    "@libp2p/interface": "^3.3.0",
+    "@libp2p/mdns": "^12.0.30",
+    "@libp2p/mplex": "^12.0.30",
+    "@libp2p/peer-id": "^6.0.15",
+    "@libp2p/prometheus-metrics": "^5.0.28",
+    "@libp2p/tcp": "^11.0.27",
+    "@lodestar/api": "^1.47.0-dev.02d2dbea03",
+    "@lodestar/config": "^1.47.0-dev.02d2dbea03",
+    "@lodestar/db": "^1.47.0-dev.02d2dbea03",
+    "@lodestar/fork-choice": "^1.47.0-dev.02d2dbea03",
+    "@lodestar/logger": "^1.47.0-dev.02d2dbea03",
+    "@lodestar/params": "^1.47.0-dev.02d2dbea03",
+    "@lodestar/reqresp": "^1.47.0-dev.02d2dbea03",
+    "@lodestar/state-transition": "^1.47.0-dev.02d2dbea03",
+    "@lodestar/types": "^1.47.0-dev.02d2dbea03",
+    "@lodestar/utils": "^1.47.0-dev.02d2dbea03",
+    "@lodestar/validator": "^1.47.0-dev.02d2dbea03",
     "@multiformats/multiaddr": "^13.0.1",
-    "datastore-core": "^11.0.2",
-    "datastore-level": "^12.0.2",
+    "datastore-core": "^12.0.1",
+    "datastore-level": "^13.0.1",
     "deepmerge": "^4.3.1",
     "fastify": "^5.8.1",
-    "interface-datastore": "^9.0.2",
+    "interface-datastore": "^10.0.1",
     "jwt-simple": "0.5.6",
-    "libp2p": "3.1.6",
+    "libp2p": "3.3.9",
     "multiformats": "^13.4.2",
     "prom-client": "^15.1.0",
     "qs": "^6.14.2",
@@ -155,10 +155,10 @@
   },
   "devDependencies": {
     "@chainsafe/swap-or-not-shuffle": "^1.2.1",
-    "@libp2p/interface-internal": "^3.0.13",
-    "@libp2p/logger": "^6.2.2",
-    "@libp2p/utils": "^7.0.13",
-    "@lodestar/spec-test-util": "^1.46.0",
+    "@libp2p/interface-internal": "^3.1.12",
+    "@libp2p/logger": "^6.2.13",
+    "@libp2p/utils": "^7.4.0",
+    "@lodestar/spec-test-util": "^1.47.0-dev.02d2dbea03",
     "@types/js-yaml": "^4.0.5",
     "@types/qs": "^6.9.7",
     "@types/tmp": "^0.2.3",
@@ -175,5 +175,5 @@
     "beacon",
     "blockchain"
   ],
-  "gitHead": "3873dd5b032d0ad82581fc3416e9628b4f6f2642"
+  "gitHead": "4dfb544428ee8bda82f94c10cfa569a5f132f4e0"
 }

package/src/api/impl/beacon/blocks/index.ts

@@ -128,7 +128,8 @@ export function getBeaconBlockApi({
         forkName: fork,
         sampledColumns: chain.custodyConfig.sampledColumns,
         custodyColumns: chain.custodyConfig.custodyColumns,
-        timeCreatedSec: seenTimestampSec,
+        seenTimestampSec,
+        source: PayloadEnvelopeInputSource.api,
       });
     }
 
@@ -220,12 +221,6 @@ export function getBeaconBlockApi({
                   // Block has already been seen, e.g. via gossip racing the publish API. Benign.
                   chain.logger.debug("Ignoring already-known block during publishing", valLogMeta);
                   return;
-                case BlockErrorCode.REPEAT_PROPOSAL:
-                  // The proposer already produced a block for this slot. For a solo setup this is a
-                  // notable signal (duplicate-proposal attempt). For fallback / DVT setups it is
-                  // expected on every block where another node published first.
-                  chain.logger.warn("Ignoring repeat-proposal block during publishing", valLogMeta);
-                  return;
               }
             }
 
@@ -407,7 +402,7 @@ export function getBeaconBlockApi({
       for (let i = 0; i < dataColumnSidecars.length; i++) {
         // + 1 because we publish to beacon_block first
         const {sentPeers, alreadyPublished} = sentPeersArr[i + 1] as {sentPeers: number; alreadyPublished: boolean};
-        // sent peers could be 0 as we set `allowPublishToZeroTopicPeers=true` in network.publishDataColumnSidecar()
+        // sent peers could be 0, data_column_sidecar opts out of the zero peers publish error, see gossipTopicAllowPublishToZeroPeers
         metrics?.dataColumns.sentPeersPerSubnet.observe(sentPeers);
         // A duplicate publish (alreadyPublished=true) means the column is already propagating on the network —
         // expected in self-build flows where peers gossip columns back to us before we publish the envelope.
@@ -1032,6 +1027,7 @@ export function getBeaconBlockApi({
     },
 
     async publishExecutionPayloadBid({signedExecutionPayloadBid}) {
+      const seenTimestampSec = Date.now() / 1000;
       const bid = signedExecutionPayloadBid.message;
       const slot = bid.slot;
       const fork = config.getForkName(slot);
@@ -1042,6 +1038,9 @@ export function getBeaconBlockApi({
 
       await validateApiExecutionPayloadBid(chain, signedExecutionPayloadBid);
 
+      const elapsedSec = chain.clock.secFromSlot(slot, seenTimestampSec);
+      metrics?.gossipExecutionPayloadBid.elapsedTimeTillReceived.observe({source: OpSource.api}, elapsedSec);
+
       try {
         const insertOutcome = chain.executionPayloadBidPool.add(signedExecutionPayloadBid);
         metrics?.opPool.executionPayloadBidPool.apiInsertOutcome.inc({insertOutcome});

package/src/api/impl/beacon/pool/index.ts

@@ -25,6 +25,7 @@ import {validateApiPayloadAttestationMessage} from "../../../../chain/validation
 import {validateApiProposerSlashing} from "../../../../chain/validation/proposerSlashing.js";
 import {validateApiSyncCommittee} from "../../../../chain/validation/syncCommittee.js";
 import {validateApiVoluntaryExit} from "../../../../chain/validation/voluntaryExit.js";
+import {OpSource} from "../../../../chain/validatorMonitor.js";
 import {validateGossipFnRetryUnknownRoot} from "../../../../network/processor/gossipHandlers.js";
 import {ApiError, FailureList, IndexedError} from "../../errors.js";
 import {ApiModules} from "../../types.js";
@@ -99,7 +100,9 @@ export function getBeaconPoolApi({
             const {indexedAttestation, subnet, attDataRootHex, committeeIndex, validatorCommitteeIndex, committeeSize} =
               await validateGossipFnRetryUnknownRoot(validateFn, network, chain, slot, beaconBlockRoot);
 
-            if (network.shouldAggregate(subnet, slot)) {
+            try {
+              // `is_aggregator` only covers aggregating what we receive on the subnet via gossip, not what
+              // a validator client hands us directly, see https://github.com/ChainSafe/lodestar/issues/7548
               const insertOutcome = chain.attestationPool.add(
                 committeeIndex,
                 attestation,
@@ -109,6 +112,10 @@ export function getBeaconPoolApi({
                 priority
               );
               metrics?.opPool.attestationPool.apiInsertOutcome.inc({insertOutcome});
+            } catch (e) {
+              // The pool is a local optimization, failing to insert must not stop us from publishing a
+              // validated attestation, which the route requires us to do. Same handling as the gossip path
+              logger.debug("Error adding unaggregated attestation to pool", {subnet}, e as Error);
             }
 
             if (isForkPostElectra(fork)) {
@@ -172,7 +179,21 @@ export function getBeaconPoolApi({
     },
 
     async submitPoolVoluntaryExit({signedVoluntaryExit}) {
-      await validateApiVoluntaryExit(chain, signedVoluntaryExit);
+      const result = await validateApiVoluntaryExit(chain, signedVoluntaryExit);
+
+      if (result.status === "deferred") {
+        const currentEpoch = chain.clock.currentEpoch;
+        const inserted = chain.deferredVoluntaryExitPool.insert(signedVoluntaryExit, result.validity, currentEpoch);
+        if (!inserted) {
+          throw new ApiError(400, "Deferred voluntary exit pool is full or already contains this validator");
+        }
+        logger.info("Voluntary exit deferred until transient conditions are met", {
+          validatorIndex: signedVoluntaryExit.message.validatorIndex,
+          reason: result.validity,
+        });
+        return;
+      }
+
       chain.opPool.insertVoluntaryExit(signedVoluntaryExit);
       chain.emitter.emit(routes.events.EventType.voluntaryExit, signedVoluntaryExit);
       await network.publishVoluntaryExit(signedVoluntaryExit);
@@ -211,6 +232,7 @@ export function getBeaconPoolApi({
     },
 
     async submitPayloadAttestationMessages({payloadAttestationMessages}) {
+      const seenTimestampSec = Date.now() / 1000;
       const failures: FailureList = [];
 
       await Promise.all(
@@ -226,6 +248,9 @@ export function getBeaconPoolApi({
               beaconBlockRoot
             );
 
+            const delaySec = chain.clock.secFromSlot(slot, seenTimestampSec);
+            metrics?.gossipPayloadAttestationMessage.elapsedTimeTillReceived.observe({source: OpSource.api}, delaySec);
+
             const insertOutcome = chain.payloadAttestationPool.add(
               payloadAttestationMessage,
               attDataRootHex,

package/src/api/impl/beacon/state/utils.ts

@@ -1,7 +1,8 @@
+import {type PubkeyCache} from "@chainsafe/lodestar-z/pubkeys";
 import {routes} from "@lodestar/api";
 import {CheckpointWithHex, IForkChoice} from "@lodestar/fork-choice";
 import {GENESIS_SLOT} from "@lodestar/params";
-import {IBeaconStateView, IBeaconStateViewGloas, PubkeyCache} from "@lodestar/state-transition";
+import {IBeaconStateView, IBeaconStateViewGloas} from "@lodestar/state-transition";
 import {
   BLSPubkey,
   BuilderIndex,

package/src/api/impl/events/index.ts

@@ -4,7 +4,8 @@ import {ApiModules} from "../types.js";
 
 export function getEventsApi({
   chain,
-}: Pick<ApiModules, "chain" | "config">): ApplicationMethods<routes.events.Endpoints> {
+  logger,
+}: Pick<ApiModules, "chain" | "config" | "logger">): ApplicationMethods<routes.events.Endpoints> {
   return {
     async eventstream({topics, signal, onEvent}) {
       const onAbortFns: (() => void)[] = [];
@@ -12,9 +13,16 @@ export function getEventsApi({
       for (const topic of topics) {
         // biome-ignore lint/suspicious/noExplicitAny: We need to use `any` type here
         const handler = (data: any): void => {
-          // TODO: What happens if this handler throws? Does it break the other chain.emitter listeners?
-
-          onEvent({type: topic, message: data});
+          try {
+            // TS only relates `{type, message}` to the `BeaconEvent` discriminated union while
+            // `EventType` stays within its ~25-member cap; without the cast, adding events trips TS2345.
+            onEvent({type: topic, message: data} as routes.events.BeaconEvent);
+          } catch (e) {
+            // `chain.emitter` emits synchronously, a throwing listener would propagate the error to
+            // whoever emitted the event, e.g. a gossip handler, and skip the remaining listeners.
+            // A single misbehaving event or client must not affect the node or other subscribers.
+            logger.warn("Error sending event to client", {topic}, e as Error);
+          }
         };
 
         chain.emitter.on(topic, handler);

package/src/api/impl/lodestar/index.ts

@@ -147,7 +147,7 @@ export function getLodestarApi({
     },
 
     async dropStateCache() {
-      chain.regen.dropCache();
+      await chain.regen.dropCache();
     },
 
     async connectPeer({peerId, multiaddrs}) {
@@ -280,7 +280,7 @@ export function getLodestarApi({
     },
 
     async getFastConfirmationInfo() {
-      const confirmedRoot = chain.forkChoice.getConfirmedRoot();
+      const fcrStore = chain.forkChoice.getFastConfirmationStore();
       const confirmedBlock = chain.forkChoice.getConfirmedBlock();
       const justifiedCheckpoint = chain.forkChoice.getJustifiedCheckpoint();
       const finalizedCheckpoint = chain.forkChoice.getFinalizedCheckpoint();
@@ -290,21 +290,20 @@ export function getLodestarApi({
       return {
         data: {
           confirmed: {
-            rootHex: confirmedRoot,
-            slot: confirmedBlock?.slot ?? null,
+            root: fromHex(fcrStore.confirmedRoot),
+            slot: confirmedBlock?.slot ?? 0,
           },
           head: {
-            rootHex: headRoot,
+            root: fromHex(headRoot),
             slot: head.slot,
           },
-          justifiedCheckpoint: {
-            rootHex: justifiedCheckpoint.rootHex,
-            epoch: justifiedCheckpoint.epoch,
-          },
-          finalizedCheckpoint: {
-            rootHex: finalizedCheckpoint.rootHex,
-            epoch: finalizedCheckpoint.epoch,
-          },
+          justifiedCheckpoint,
+          finalizedCheckpoint,
+          previousEpochObservedJustifiedCheckpoint: fcrStore.previousEpochObservedJustifiedCheckpoint,
+          currentEpochObservedJustifiedCheckpoint: fcrStore.currentEpochObservedJustifiedCheckpoint,
+          previousEpochGreatestUnrealizedCheckpoint: fcrStore.previousEpochGreatestUnrealizedCheckpoint,
+          previousSlotHead: fromHex(fcrStore.previousSlotHead),
+          currentSlotHead: fromHex(fcrStore.currentSlotHead),
         },
       };
     },

package/src/api/impl/validator/index.ts

@@ -92,7 +92,12 @@ import {getStateResponseWithRegen} from "../beacon/state/utils.js";
 import {ApiError, FailureList, IndexedError, NodeIsSyncing, OnlySupportedByDVT} from "../errors.js";
 import {ApiModules} from "../types.js";
 import {notWhileSyncing} from "../utils.js";
-import {computeSubnetForCommitteesAtSlot, getPubkeysForIndices, selectBlockProductionSource} from "./utils.js";
+import {
+  computeSubnetForCommitteesAtSlot,
+  getPubkeysForIndices,
+  selectBlockProductionSource,
+  selectBlockProductionSourceByBoostFactor,
+} from "./utils.js";
 
 /**
  * Cutoff time to wait from start of the slot for execution and builder block production apis to resolve.
@@ -577,8 +582,7 @@ export function getValidatorApi(
       builderSelection,
       isBuilderEnabled,
       strictFeeRecipientCheck,
-      // winston logger doesn't like bigint
-      builderBoostFactor: `${builderBoostFactor}`,
+      builderBoostFactor,
     };
 
     logger.verbose("Assembling block with produceEngineOrBuilderBlock", loggerContext);
@@ -851,8 +855,8 @@ export function getValidatorApi(
       randaoReveal,
       graffiti,
       feeRecipient,
+      strictFeeRecipientCheck,
       includePayload,
-      builderSelection,
       builderBoostFactor,
     }) {
       const fork = config.getForkName(slot);
@@ -861,11 +865,6 @@ export function getValidatorApi(
         throw new ApiError(400, `produceBlockV4 not supported for pre-gloas fork=${fork}`);
       }
 
-      builderSelection = builderSelection ?? routes.validator.BuilderSelection.MaxProfit;
-      if (builderSelection === routes.validator.BuilderSelection.BuilderOnly) {
-        logger.warn("Builder selection builderonly is no longer supported, treating as builderalways");
-        builderSelection = routes.validator.BuilderSelection.BuilderAlways;
-      }
       builderBoostFactor = builderBoostFactor ?? BigInt(100);
       if (builderBoostFactor > MAX_BUILDER_BOOST_FACTOR) {
         throw new ApiError(400, `Invalid builderBoostFactor=${builderBoostFactor} > MAX_BUILDER_BOOST_FACTOR`);
@@ -896,14 +895,11 @@ export function getValidatorApi(
       // TODO GLOAS: add external builder api support when it is implemented
       const isBuildingOnFull = chain.forkChoice.shouldBuildOnFull(parentBlock, slot);
       const bidParentBlockHash = isBuildingOnFull ? parentBlock.executionPayloadBlockHash : parentBlock.parentBlockHash;
-      // Bids are only skipped entirely with executiononly or while the circuit breaker is active,
-      // other engine-preferring selections still build a block with the best bid as fallback in
-      // case local production fails
-      const circuitBreakerActive = chain.builderCircuitBreaker.isActive(slot);
-      const builderBid =
-        builderSelection === routes.validator.BuilderSelection.ExecutionOnly || circuitBreakerActive
-          ? null
-          : chain.executionPayloadBidPool.getBestBid(slot, bidParentBlockHash, parentBlockRootHex);
+      const circuitBreakerActive = chain.builderCircuitBreaker.isActive(slot, parentBlock);
+      // Keep a builder bid as fallback unless the circuit breaker is active
+      const builderBid = circuitBreakerActive
+        ? null
+        : chain.executionPayloadBidPool.getBestBid(slot, bidParentBlockHash, parentBlockRootHex);
 
       const logCtx = {
         slot,
@@ -911,8 +907,8 @@ export function getValidatorApi(
         parentBlockRoot: parentBlockRootHex,
         parentBlockHash: parentBlock.executionPayloadBlockHash,
         fork,
-        builderSelection,
         builderBoostFactor,
+        strictFeeRecipientCheck,
         circuitBreakerActive,
         ...(builderBid !== null
           ? {
@@ -936,6 +932,7 @@ export function getValidatorApi(
         randaoReveal,
         graffiti: graffitiBytes,
         feeRecipient,
+        strictFeeRecipientCheck,
         commonBlockBodyPromise,
       };
 
@@ -960,12 +957,8 @@ export function getValidatorApi(
         chain.produceBlock(baseAttrs)
       ).then((engineBlock) => {
         // No need to wait for the bid block if the engine block will always be selected due to
-        // suspected builder censorship, a builder boost factor of 0 or executionalways selection
-        if (
-          engineBlock.shouldOverrideBuilder ||
-          builderBoostFactor === BigInt(0) ||
-          builderSelection === routes.validator.BuilderSelection.ExecutionAlways
-        ) {
+        // suspected builder censorship or a builder boost factor of 0
+        if (engineBlock.shouldOverrideBuilder || builderBoostFactor === BigInt(0)) {
           controller.abort();
         }
         return engineBlock;
@@ -994,8 +987,7 @@ export function getValidatorApi(
         });
         logger.warn("Selected local block: censorship suspected in builder bid", logCtx);
       } else if (engineResult.status === "fulfilled" && bidResult.status === "fulfilled") {
-        const result = selectBlockProductionSource({
-          builderSelection,
+        const result = selectBlockProductionSourceByBoostFactor({
           builderBoostFactor,
           engineExecutionPayloadValue: engineResult.value.executionPayloadValue,
           // The bid value is the payment to the proposer, in Gwei
@@ -1061,7 +1053,10 @@ export function getValidatorApi(
         root: blockRoot,
       });
       if (chain.opts.persistProducedBlocks) {
-        void chain.persistBlock(block, "produced_engine_block");
+        void chain.persistBlock(
+          block,
+          source === ProducedBlockSource.builder ? "produced_builder_block" : "produced_engine_block"
+        );
       }
 
       // Include the payload for self-builds unless disabled (stateless flow)

package/src/api/impl/validator/utils.ts

@@ -59,24 +59,39 @@ export function selectBlockProductionSource({
       return {source: ProducedBlockSource.engine, reason: EngineBlockSelectionReason.EnginePreferred};
 
     case routes.validator.BuilderSelection.Default:
-    case routes.validator.BuilderSelection.MaxProfit: {
-      if (builderBoostFactor === BigInt(0)) {
-        return {source: ProducedBlockSource.engine, reason: EngineBlockSelectionReason.EnginePreferred};
-      }
-
-      if (builderBoostFactor === MAX_BUILDER_BOOST_FACTOR) {
-        return {source: ProducedBlockSource.builder, reason: BuilderBlockSelectionReason.BuilderPreferred};
-      }
-
-      if (engineExecutionPayloadValue >= (builderExecutionPayloadValue * builderBoostFactor) / BigInt(100)) {
-        return {source: ProducedBlockSource.engine, reason: EngineBlockSelectionReason.BlockValue};
-      }
-
-      return {source: ProducedBlockSource.builder, reason: BuilderBlockSelectionReason.BlockValue};
-    }
+    case routes.validator.BuilderSelection.MaxProfit:
+      return selectBlockProductionSourceByBoostFactor({
+        engineExecutionPayloadValue,
+        builderExecutionPayloadValue,
+        builderBoostFactor,
+      });
 
     case routes.validator.BuilderSelection.BuilderAlways:
     case routes.validator.BuilderSelection.BuilderOnly:
       return {source: ProducedBlockSource.builder, reason: BuilderBlockSelectionReason.BuilderPreferred};
   }
 }
+
+export function selectBlockProductionSourceByBoostFactor({
+  engineExecutionPayloadValue,
+  builderExecutionPayloadValue,
+  builderBoostFactor,
+}: {
+  engineExecutionPayloadValue: bigint;
+  builderExecutionPayloadValue: bigint;
+  builderBoostFactor: bigint;
+}): BlockSelectionResult {
+  if (builderBoostFactor === BigInt(0)) {
+    return {source: ProducedBlockSource.engine, reason: EngineBlockSelectionReason.EnginePreferred};
+  }
+
+  if (builderBoostFactor === MAX_BUILDER_BOOST_FACTOR) {
+    return {source: ProducedBlockSource.builder, reason: BuilderBlockSelectionReason.BuilderPreferred};
+  }
+
+  if (engineExecutionPayloadValue >= (builderExecutionPayloadValue * builderBoostFactor) / BigInt(100)) {
+    return {source: ProducedBlockSource.engine, reason: EngineBlockSelectionReason.BlockValue};
+  }
+
+  return {source: ProducedBlockSource.builder, reason: BuilderBlockSelectionReason.BlockValue};
+}

package/src/api/rest/base.ts

@@ -104,6 +104,13 @@ export class RestApiServer {
 
     // To parse our ApiError -> statusCode
     server.setErrorHandler<FastifyError | Error>((err, _req, res) => {
+      if (res.raw.headersSent) {
+        // A status code can no longer be written, e.g. if a long-lived SSE stream failed mid-flight.
+        // End the response instead, else the socket is left open and the client never notices.
+        if (!res.raw.writableEnded && !res.raw.destroyed) res.raw.end();
+        return;
+      }
+
       const stacktraces = opts.stacktraces ? err.stack?.split("\n") : undefined;
       if ("validation" in err && err.validation) {
         const {instancePath = "unknown", message} = err.validation?.[0] ?? {};

package/src/chain/archiveStore/archiveStore.ts

@@ -192,7 +192,7 @@ export class ArchiveStore {
 
   private onCheckpoint = (): void => {
     const headStateRoot = this.chain.forkChoice.getHead().stateRoot;
-    this.chain.regen.pruneOnCheckpoint(
+    this.chain.regen.onCheckpoint(
       this.chain.forkChoice.getFinalizedCheckpoint().epoch,
       this.chain.forkChoice.getJustifiedCheckpoint().epoch,
       headStateRoot

package/src/chain/archiveStore/historicalState/metrics.ts

@@ -1,3 +1,4 @@
+import {ForkName} from "@lodestar/params";
 import {
   BeaconStateTransitionMetrics,
   EpochTransitionStep,
@@ -45,6 +46,27 @@ export function createHistoricalStateTransitionMetrics(
       labelNames: ["step"],
       buckets: [0.01, 0.05, 0.1, 0.2, 0.5, 0.75, 1],
     }),
+    forkUpgradeTime: metricsRegister.histogram<{fork: ForkName}>({
+      name: "lodestar_historical_state_stfn_fork_upgrade_seconds",
+      help: "Time to upgrade the state at a fork boundary in seconds",
+      labelNames: ["fork"],
+      buckets: [0.05, 0.1, 0.25, 0.5, 1, 2, 5, 10, 30],
+    }),
+    onboardBuildersTime: metricsRegister.histogram({
+      name: "lodestar_historical_state_stfn_gloas_onboard_builders_seconds",
+      help: "Time spent in onboardBuildersFromPendingDeposits at the gloas fork transition",
+      buckets: [0.05, 0.1, 0.25, 0.5, 1, 2, 5, 10, 30],
+    }),
+    onboardBuildersDeposits: metricsRegister.gauge<{outcome: "onboarded" | "topup" | "kept" | "dropped"}>({
+      name: "lodestar_historical_state_stfn_gloas_onboard_builders_deposits",
+      help: "Pending deposits handled at the gloas fork transition, by outcome",
+      labelNames: ["outcome"],
+    }),
+    onboardBuildersSignatureChecks: metricsRegister.gauge<{source: "cache" | "verified"}>({
+      name: "lodestar_historical_state_stfn_gloas_onboard_builders_signature_checks",
+      help: "Builder deposit signature checks at the gloas fork transition, by whether the pre-verify cache served them",
+      labelNames: ["source"],
+    }),
     processBlockTime: metricsRegister.histogram({
       name: "lodestar_historical_state_stfn_process_block_seconds",
       help: "Time to process a single block in seconds",

package/src/chain/blocks/importBlock.ts

@@ -22,7 +22,6 @@ import {
   RootCache,
   computeEpochAtSlot,
   computeTimeAtSlot,
-  isStartSlotOfEpoch,
   isStatePostAltair,
   isStatePostBellatrix,
 } from "@lodestar/state-transition";
@@ -404,11 +403,7 @@ export async function importBlock(
         notOverrideFcuReason = NotReorgedReason.NotProposerOfNextSlot;
       }
     } catch (e) {
-      if (isStartSlotOfEpoch(proposalSlot)) {
-        notOverrideFcuReason = NotReorgedReason.AtEpochBoundary;
-      } else {
-        this.logger.warn("Unable to get beacon proposer. Do not override fcu.", {proposalSlot}, e as Error);
-      }
+      this.logger.warn("Unable to get beacon proposer. Do not override fcu.", {proposalSlot}, e as Error);
     }
 
     if (shouldOverrideFcu) {
@@ -445,7 +440,7 @@ export async function importBlock(
      * zero block hash (pre TTD)
      */
     const safeBlockHash = getSafeExecutionBlockHash(this.forkChoice, this.logger);
-    const finalizedBlockHash = getFinalizedExecutionBlockHash(this.forkChoice, this.logger);
+    const finalizedBlockHash = getFinalizedExecutionBlockHash(this.forkChoice);
     if (headBlockHash !== ZERO_HASH_HEX) {
       this.executionEngine
         .notifyForkchoiceUpdate(