@lodestar/beacon-node 1.51.0-dev.f39fad29a1 → 1.51.0-dev.f41cd750ac

package/package.json

@@ -11,7 +11,7 @@
   "bugs": {
     "url": "https://github.com/ChainSafe/lodestar/issues"
   },
-  "version": "1.51.0-dev.f39fad29a1",
+  "version": "1.51.0-dev.f41cd750ac",
   "type": "module",
   "exports": {
     ".": {
@@ -105,7 +105,6 @@
     "@chainsafe/enr": "^6.0.2",
     "@chainsafe/libp2p-noise": "^17.0.0",
     "@chainsafe/libp2p-quic": "^2.1.5",
-    "@chainsafe/lodestar-z": "catalog:",
     "@chainsafe/persistent-merkle-tree": "^1.3.1",
     "@chainsafe/prometheus-gc-stats": "^1.0.0",
     "@chainsafe/snappy-wasm": "^0.5.2",
@@ -127,17 +126,18 @@
     "@libp2p/prometheus-metrics": "^5.0.31",
     "@libp2p/tcp": "^11.0.30",
     "@libp2p/utils": "^7.4.3",
-    "@lodestar/api": "^1.51.0-dev.f39fad29a1",
-    "@lodestar/config": "^1.51.0-dev.f39fad29a1",
-    "@lodestar/db": "^1.51.0-dev.f39fad29a1",
-    "@lodestar/fork-choice": "^1.51.0-dev.f39fad29a1",
-    "@lodestar/logger": "^1.51.0-dev.f39fad29a1",
-    "@lodestar/params": "^1.51.0-dev.f39fad29a1",
-    "@lodestar/reqresp": "^1.51.0-dev.f39fad29a1",
-    "@lodestar/state-transition": "^1.51.0-dev.f39fad29a1",
-    "@lodestar/types": "^1.51.0-dev.f39fad29a1",
-    "@lodestar/utils": "^1.51.0-dev.f39fad29a1",
-    "@lodestar/validator": "^1.51.0-dev.f39fad29a1",
+    "@lodestar/api": "^1.51.0-dev.f41cd750ac",
+    "@lodestar/config": "^1.51.0-dev.f41cd750ac",
+    "@lodestar/db": "^1.51.0-dev.f41cd750ac",
+    "@lodestar/fork-choice": "^1.51.0-dev.f41cd750ac",
+    "@lodestar/logger": "^1.51.0-dev.f41cd750ac",
+    "@lodestar/native": "^1.51.0-dev.f41cd750ac",
+    "@lodestar/params": "^1.51.0-dev.f41cd750ac",
+    "@lodestar/reqresp": "^1.51.0-dev.f41cd750ac",
+    "@lodestar/state-transition": "^1.51.0-dev.f41cd750ac",
+    "@lodestar/types": "^1.51.0-dev.f41cd750ac",
+    "@lodestar/utils": "^1.51.0-dev.f41cd750ac",
+    "@lodestar/validator": "^1.51.0-dev.f41cd750ac",
     "@multiformats/multiaddr": "^13.0.1",
     "datastore-core": "^12.0.1",
     "datastore-level": "^13.0.1",
@@ -159,14 +159,13 @@
     "@ethereumjs/rlp": "^4.0.1",
     "@libp2p/interface-internal": "^3.2.0",
     "@libp2p/logger": "^6.2.15",
-    "@lodestar/spec-test-util": "^1.51.0-dev.f39fad29a1",
-    "@lodestar/test-utils": "^1.51.0-dev.f39fad29a1",
+    "@lodestar/spec-test-util": "^1.51.0-dev.f41cd750ac",
+    "@lodestar/test-utils": "^1.51.0-dev.f41cd750ac",
     "@types/js-yaml": "^4.0.5",
     "@types/qs": "^6.9.7",
     "@types/tmp": "^0.2.3",
     "dotenv": "^16.4.5",
     "js-yaml": "^4.1.0",
-    "rimraf": "^4.4.1",
     "tmp": "^0.2.1"
   },
   "keywords": [
@@ -175,5 +174,5 @@
     "beacon",
     "blockchain"
   ],
-  "gitHead": "e52b8a101948ed725175de63e644cdf314ea7e91"
+  "gitHead": "cdf182625bc5a8c3719d89192626d23a2585748b"
 }

package/src/api/impl/beacon/blocks/index.ts

@@ -316,18 +316,8 @@ export function getBeaconBlockApi(
         break;
       }
 
-      case routes.beacon.BroadcastValidation.none: {
-        chain.logger.debug("Skipping broadcast validation", valLogMeta);
-        break;
-      }
-
       default: {
-        // error or log warning we do not support this validation
-        const message = `Broadcast validation of ${broadcastValidation} type not implemented yet`;
-        if (chain.opts.broadcastValidationStrictness === "error") {
-          throw Error(message);
-        }
-        chain.logger.warn(message, valLogMeta);
+        throw new ApiError(400, `Invalid broadcast validation level ${broadcastValidation}`);
       }
     }
 
@@ -397,26 +387,24 @@ export function getBeaconBlockApi(
       ...blobSidecars.map((blobSidecar) => () => network.publishBlobSidecar(blobSidecar)),
       () =>
         // there is no rush to persist block since we published it to gossip anyway
-        chain
-          .processBlock(blockForImport, opts)
-          .catch((e) => {
-            if (e instanceof BlockError) {
-              switch (e.type.code) {
-                case BlockErrorCode.ALREADY_KNOWN:
-                  // Block was imported while publishing, e.g. received via gossip
-                  chain.logger.debug("Ignoring already-known block during publishing", valLogMeta);
-                  return;
-                case BlockErrorCode.PARENT_BLOCK_UNKNOWN:
-                case BlockErrorCode.PARENT_PAYLOAD_UNKNOWN:
-                  chain.emitter.emit(ChainEvent.blockUnknownParent, {
-                    blockInput: blockForImport,
-                    peer: IDENTITY_PEER_ID,
-                    source: BlockInputSource.api,
-                  });
-              }
+        chain.processBlock(blockForImport, opts).catch((e) => {
+          if (e instanceof BlockError) {
+            switch (e.type.code) {
+              case BlockErrorCode.ALREADY_KNOWN:
+                // Block was imported while publishing, e.g. received via gossip
+                chain.logger.debug("Ignoring already-known block during publishing", valLogMeta);
+                return;
+              case BlockErrorCode.PARENT_BLOCK_UNKNOWN:
+              case BlockErrorCode.PARENT_PAYLOAD_UNKNOWN:
+                chain.emitter.emit(ChainEvent.blockUnknownParent, {
+                  blockInput: blockForImport,
+                  peer: IDENTITY_PEER_ID,
+                  source: BlockInputSource.api,
+                });
             }
-            throw e;
-          }),
+          }
+          throw e;
+        }),
     ];
     const sentPeersArr = await promiseAllMaybeAsync<{sentPeers: number; alreadyPublished: boolean} | number | void>(
       publishPromises
@@ -804,11 +792,6 @@ export function getBeaconBlockApi(
       };
       try {
         switch (broadcastValidation) {
-          case routes.beacon.BroadcastValidation.none: {
-            chain.logger.debug("Skipping broadcast validation of execution payload envelope", valLogMeta);
-            break;
-          }
-
           case routes.beacon.BroadcastValidation.gossip: {
             await validateApiExecutionPayloadEnvelope(chain, signedExecutionPayloadEnvelope);
             break;
@@ -847,11 +830,7 @@ export function getBeaconBlockApi(
           }
 
           default: {
-            const message = `Broadcast validation of ${broadcastValidation} type not implemented yet`;
-            if (chain.opts.broadcastValidationStrictness === "error") {
-              throw Error(message);
-            }
-            chain.logger.warn(message, valLogMeta);
+            throw new ApiError(400, `Invalid broadcast validation level ${broadcastValidation}`);
           }
         }
       } catch (error) {
@@ -1003,9 +982,13 @@ export function getBeaconBlockApi(
         () => network.publishSignedExecutionPayloadEnvelope(signedExecutionPayloadEnvelope),
         // Publish all data column sidecars
         ...dataColumnSidecars.map((dataColumnSidecar) => () => network.publishDataColumnSidecar(dataColumnSidecar)),
-        // Import execution payload. Signature is verified during broadcast validation, an already
-        // known envelope was verified by its original source and `none` deliberately skips validation
-        () => chain.processExecutionPayload(payloadInput, {validSignature: true}),
+        // Import execution payload
+        () =>
+          chain.processExecutionPayload(payloadInput, {
+            // Signature is verified during broadcast validation only for the submitted envelope,
+            // an envelope cached by another source is verified at import
+            validSignature: payloadInput.getPayloadEnvelope() === signedExecutionPayloadEnvelope,
+          }),
       ];
 
       const publishPromise = promiseAllMaybeAsync<{sentPeers: number; alreadyPublished: boolean} | number | void>(
@@ -1109,7 +1092,7 @@ export function getBeaconBlockApi(
     },
 
     async getSignedExecutionPayloadEnvelope({blockId}, context) {
-      const {block, executionOptimistic, finalized} = await getBlockResponse(chain, blockId);
+      const {block, finalized} = await getBlockResponse(chain, blockId);
       const slot = block.message.slot;
       const fork = config.getForkName(slot);
 
@@ -1123,6 +1106,13 @@ export function getBeaconBlockApi(
       const blockRoot = config.getForkTypes(slot).BeaconBlock.hashTreeRoot(block.message);
       const blockRootHex = toRootHex(blockRoot);
 
+      // The block's own flag is the status of the payload it builds on, the envelope's is that of its
+      // FULL variant. An unfinalized block without one has not imported its envelope yet.
+      const payload = finalized ? null : chain.forkChoice.getBlockHex(blockRootHex, PayloadStatus.FULL);
+      if (!finalized && payload === null) {
+        throw new ApiError(404, `Execution payload envelope not found for slot=${slot}, blockRoot=${blockRootHex}`);
+      }
+
       let data: Uint8Array | gloas.SignedExecutionPayloadEnvelope | null;
       try {
         data = context?.returnBytes
@@ -1147,7 +1137,7 @@ export function getBeaconBlockApi(
 
       return {
         data,
-        meta: {executionOptimistic, finalized, version: fork},
+        meta: {executionOptimistic: payload !== null && isOptimisticBlock(payload), finalized, version: fork},
       };
     },
 

package/src/api/impl/beacon/pool/index.ts

@@ -17,6 +17,7 @@ import {
   PayloadAttestationError,
   PayloadAttestationErrorCode,
   SyncCommitteeError,
+  SyncCommitteeErrorCode,
 } from "../../../../chain/errors/index.js";
 import {validateApiAttesterSlashing} from "../../../../chain/validation/attesterSlashing.js";
 import {validateApiBlsToExecutionChange} from "../../../../chain/validation/blsToExecutionChange.js";
@@ -340,7 +341,10 @@ export function getBeaconPoolApi({
             const synCommittee = state.getIndexedSyncCommittee(signature.slot);
             const indexesInCommittee = synCommittee.validatorIndexMap.get(signature.validatorIndex);
             if (indexesInCommittee === undefined || indexesInCommittee.length === 0) {
-              return; // Not a sync committee member
+              throw new SyncCommitteeError(GossipAction.REJECT, {
+                code: SyncCommitteeErrorCode.VALIDATOR_NOT_IN_SYNC_COMMITTEE,
+                validatorIndex: signature.validatorIndex,
+              });
             }
 
             // Verify signature only, all other data is very likely to be correct, since the `signature` object is created by this node.

package/src/api/impl/beacon/state/utils.ts

@@ -1,6 +1,6 @@
-import {type PubkeyCache} from "@chainsafe/lodestar-z/pubkeys";
 import {routes} from "@lodestar/api";
 import {CheckpointWithHex, IForkChoice} from "@lodestar/fork-choice";
+import {type PubkeyCache} from "@lodestar/native/pubkeys";
 import {GENESIS_SLOT} from "@lodestar/params";
 import {IBeaconStateView, IBeaconStateViewGloas} from "@lodestar/state-transition";
 import {

package/src/api/impl/validator/index.ts

@@ -1756,7 +1756,10 @@ export function getValidatorApi(
         }
       }
 
-      const dependentRoot = fromHex(state.getShufflingDecisionRoot(epoch));
+      // The fork epoch PTC is computed at the fork from effective balances updated at the end of the
+      // last pre-fork epoch, so its dependent root is one epoch later than for other epochs
+      const dependentRootEpoch = epoch === config.GLOAS_FORK_EPOCH ? epoch + 1 : epoch;
+      const dependentRoot = fromHex(state.getShufflingDecisionRoot(dependentRootEpoch));
 
       return {
         data: duties,

package/src/chain/GetBlobsTracker.ts

@@ -1,10 +1,9 @@
 import {ChainForkConfig} from "@lodestar/config";
 import {computeEpochAtSlot} from "@lodestar/state-transition";
-import {Logger} from "@lodestar/utils";
+import {Logger, pruneSetToMax} from "@lodestar/utils";
 import {BLOB_AND_PROOF_V2_RPC_BYTES} from "../execution/engine/types.js";
 import {IExecutionEngine} from "../execution/index.js";
 import {Metrics} from "../metrics/metrics.js";
-import {callInNextEventLoop} from "../util/eventLoop.js";
 import {
   DataColumnEngineResult,
   getBlobSidecarsFromExecution,
@@ -14,6 +13,11 @@ import {IBlockInput, isBlockInputBlobs} from "./blocks/blockInput/index.js";
 import {PayloadEnvelopeInput} from "./blocks/payloadEnvelopeInput/index.js";
 import {ChainEventEmitter} from "./emitter.js";
 
+/** A `null` answer only changes if the EL receives the transactions later, so retries are few and spaced out */
+export const MAX_GET_BLOBS_ATTEMPTS = 3;
+export const GET_BLOBS_RETRY_INTERVAL_MS = 1000;
+const MAX_TRACKED_BLOCK_ROOTS = 64;
+
 export type GetBlobsTrackerInit = {
   logger: Logger;
   executionEngine: IExecutionEngine;
@@ -32,6 +36,8 @@ export class GetBlobsTracker {
   metrics: Metrics | null;
   config: ChainForkConfig;
   activeReconstructions = new Set<string>();
+  failedAttempts = new Map<string, {count: number; lastAttemptMs: number; retryTimer?: NodeJS.Timeout}>();
+  pendingTriggers = new Set<string>();
   // Preallocate buffers for getBlobsV2 RPC calls
   // See https://github.com/ChainSafe/lodestar/pull/8282 for context
   blobsAndProofsBuffers: {buffers: Uint8Array[]; inUse: boolean}[] = [];
@@ -46,23 +52,55 @@ export class GetBlobsTracker {
 
   triggerGetBlobs(input: IBlockInput | PayloadEnvelopeInput): void {
     if (this.activeReconstructions.has(input.blockRootHex)) {
+      this.pendingTriggers.add(input.blockRootHex);
       return;
     }
 
+    const failed = this.failedAttempts.get(input.blockRootHex);
+    if (failed) {
+      if (failed.count >= MAX_GET_BLOBS_ATTEMPTS) {
+        return;
+      }
+      const waitMs = GET_BLOBS_RETRY_INTERVAL_MS - (Date.now() - failed.lastAttemptMs);
+      if (waitMs > 0) {
+        // Columns tend to arrive in one burst, so keep a single trailing retry instead of relying on a later trigger
+        failed.retryTimer ??= setTimeout(() => {
+          failed.retryTimer = undefined;
+          if (this.failedAttempts.get(input.blockRootHex) === failed && !input.hasAllData()) {
+            this.triggerGetBlobs(input);
+          }
+        }, waitMs).unref();
+        return;
+      }
+    }
+
+    // The request is sent right away, before block processing issues newPayload: ELs that drop a
+    // transaction's blobs once the payload is validated can only answer a request that gets there first
     if (!(input instanceof PayloadEnvelopeInput) && isBlockInputBlobs(input)) {
       // there is not preallocation for blob sidecars like there is for columns sidecars so no need to
       // store the index for the preallocated buffers
       this.activeReconstructions.add(input.blockRootHex);
-      callInNextEventLoop(() => {
-        const logCtx = {slot: input.slot, root: input.blockRootHex};
-        this.logger.verbose("Trigger getBlobsV1 for block", logCtx);
-        getBlobSidecarsFromExecution(this.config, this.executionEngine, this.metrics, this.emitter, input).finally(
-          () => {
-            this.logger.verbose("Completed getBlobsV1 for block", logCtx);
-            this.activeReconstructions.delete(input.blockRootHex);
+      const logCtx = {slot: input.slot, root: input.blockRootHex};
+      this.logger.verbose("Trigger getBlobsV1 for block", logCtx);
+      let requestFailed = false;
+      getBlobSidecarsFromExecution(this.config, this.executionEngine, this.metrics, this.emitter, input)
+        .then(() => {
+          // EL errors and misses are handled inside the helper, blobs still missing means the attempt failed
+          if (!input.hasAllData()) {
+            requestFailed = true;
+            this.recordFailedAttempt(input.blockRootHex);
           }
-        );
-      });
+        })
+        .catch((error) => {
+          this.logger.debug("Error during getBlobsV1 for block", logCtx, error as Error);
+          requestFailed = true;
+          this.recordFailedAttempt(input.blockRootHex);
+        })
+        .finally(() => {
+          this.logger.verbose("Completed getBlobsV1 for block", logCtx);
+          this.activeReconstructions.delete(input.blockRootHex);
+          this.retryPendingTrigger(input, requestFailed);
+        });
 
       return;
     }
@@ -83,34 +121,55 @@ export class GetBlobsTracker {
     }
     timer?.();
 
-    // We don't care about the outcome of this call,
-    // just that it has been triggered for this block root.
     this.activeReconstructions.add(input.blockRootHex);
     this.blobsAndProofsBuffers[freeIndex].inUse = true;
-    callInNextEventLoop(() => {
-      const logCtx = {slot: input.slot, root: input.blockRootHex};
-      this.logger.verbose("Trigger getBlobsV2 for block", logCtx);
-      getDataColumnSidecarsFromExecution(
-        this.config,
-        this.executionEngine,
-        this.emitter,
-        input,
-        this.metrics,
-        this.blobsAndProofsBuffers[freeIndex].buffers
-      )
-        .then((result) => {
-          this.logger.debug("getBlobsV2 result for block", {...logCtx, result});
-          this.metrics?.dataColumns.dataColumnEngineResult.inc({result});
-        })
-        .catch((error) => {
-          this.logger.debug("Error during getBlobsV2 for block", logCtx, error as Error);
-          this.metrics?.dataColumns.dataColumnEngineResult.inc({result: DataColumnEngineResult.Failed});
-        })
-        .finally(() => {
-          this.logger.verbose("Completed getBlobsV2 for block", logCtx);
-          this.activeReconstructions.delete(input.blockRootHex);
-          this.blobsAndProofsBuffers[freeIndex].inUse = false;
-        });
-    });
+    const logCtx = {slot: input.slot, root: input.blockRootHex};
+    this.logger.verbose("Trigger getBlobsV2 for block", logCtx);
+    let requestFailed = false;
+    getDataColumnSidecarsFromExecution(
+      this.config,
+      this.executionEngine,
+      this.emitter,
+      input,
+      this.metrics,
+      this.blobsAndProofsBuffers[freeIndex].buffers
+    )
+      .then((result) => {
+        this.logger.debug("getBlobsV2 result for block", {...logCtx, result});
+        this.metrics?.dataColumns.dataColumnEngineResult.inc({result});
+        if (result === DataColumnEngineResult.NullResponse) {
+          requestFailed = true;
+          this.recordFailedAttempt(input.blockRootHex);
+        }
+      })
+      .catch((error) => {
+        this.logger.debug("Error during getBlobsV2 for block", logCtx, error as Error);
+        this.metrics?.dataColumns.dataColumnEngineResult.inc({result: DataColumnEngineResult.Failed});
+        requestFailed = true;
+        this.recordFailedAttempt(input.blockRootHex);
+      })
+      .finally(() => {
+        this.logger.verbose("Completed getBlobsV2 for block", logCtx);
+        this.activeReconstructions.delete(input.blockRootHex);
+        this.blobsAndProofsBuffers[freeIndex].inUse = false;
+        this.retryPendingTrigger(input, requestFailed);
+      });
+  }
+
+  private retryPendingTrigger(input: IBlockInput | PayloadEnvelopeInput, requestFailed: boolean): void {
+    // Triggers dropped while the request was in flight re-enter once it failed, the attempt that was just
+    // recorded makes this arm the trailing retry instead of sending another request right away.
+    // Post-gloas only the bid triggers a request, so every failed attempt re-enters to arm its own retry
+    const hadPendingTrigger = this.pendingTriggers.delete(input.blockRootHex);
+    if (requestFailed && (hadPendingTrigger || input instanceof PayloadEnvelopeInput)) {
+      this.triggerGetBlobs(input);
+    }
+  }
+
+  private recordFailedAttempt(blockRootHex: string): void {
+    const failed = this.failedAttempts.get(blockRootHex);
+    clearTimeout(failed?.retryTimer);
+    this.failedAttempts.set(blockRootHex, {count: (failed?.count ?? 0) + 1, lastAttemptMs: Date.now()});
+    pruneSetToMax(this.failedAttempts, MAX_TRACKED_BLOCK_ROOTS);
   }
 }

package/src/chain/archiveStore/archiveStore.ts

@@ -197,6 +197,7 @@ export class ArchiveStore {
     //
     // `jobQueue.push()` can throw synchronously (queue aborted / full) or reject asynchronously;
     // awaiting it inside the deferred callback funnels both failure modes into the one catch.
+    // biome-ignore lint/nursery/noMisusedPromises: the callback catches all errors itself
     callInNextEventLoop(async () => {
       try {
         await this.jobQueue.push(finalized);

package/src/chain/archiveStore/utils/archiveBlocks.ts

@@ -1,7 +1,14 @@
 import path from "node:path";
 import {ChainForkConfig} from "@lodestar/config";
 import {KeyValue} from "@lodestar/db";
-import {CheckpointWithHex, ExecutionStatus, IForkChoice, PayloadStatus, ProtoBlock} from "@lodestar/fork-choice";
+import {
+  CheckpointWithHex,
+  ExecutionStatus,
+  IForkChoice,
+  PayloadStatus,
+  ProtoBlock,
+  isGloasBlock,
+} from "@lodestar/fork-choice";
 import {ForkSeq, SLOTS_PER_EPOCH} from "@lodestar/params";
 import {computeEpochAtSlot, computeStartSlotAtEpoch, getBlobDataRetentionStartEpoch} from "@lodestar/state-transition";
 import {Epoch, Slot} from "@lodestar/types";
@@ -49,6 +56,17 @@ async function persistOrphanedBlock(
   await writeIfNotExist(filepath, bytes);
 }
 
+/**
+ * Oldest epoch for which data column sidecars are kept
+ */
+export function getDataColumnSidecarsMinEpoch(
+  config: ChainForkConfig,
+  currentEpoch: Epoch,
+  archiveDataEpochs?: number
+): Epoch {
+  return Math.min(getBlobDataRetentionStartEpoch(config, currentEpoch), currentEpoch - (archiveDataEpochs ?? 0));
+}
+
 /**
  * Archives finalized blocks from active bucket to archive bucket.
  *
@@ -152,44 +170,34 @@ export async function archiveBlocks(
     }
   }
 
-  // deleteNonCanonicalBlocks
-  // loop through forkchoice single time
+  const canonicalRoots = new Set(finalizedCanonicalBlocks.map((block) => block.blockRoot));
+  const orphanedBlocks: ProtoBlock[] = [];
+  const orphanedPayloads: ProtoBlock[] = [];
 
-  const nonCanonicalBlockRoots = finalizedNonCanonicalBlocks.map((summary) => fromHex(summary.blockRoot));
-  if (nonCanonicalBlockRoots.length > 0) {
-    if (persistOrphanedBlocks) {
-      // Persist orphaned blocks to disk before deleting them from hot db
-      await Promise.all(
-        nonCanonicalBlockRoots.map(async (root, index) => {
-          const block = finalizedNonCanonicalBlocks[index];
-          const blockBytes = await db.block.getBinary(root);
-          const blockLogCtx = {slot: block.slot, root: block.blockRoot};
-          if (blockBytes) {
-            await persistOrphanedBlock(block.slot, block.blockRoot, blockBytes, {
-              persistOrphanedBlocksDir: persistOrphanedBlocksDir ?? "orphaned_blocks",
-            });
-            logger.verbose("Persisted orphaned block", {...logCtx, ...blockLogCtx});
-          } else {
-            logger.warn("Tried to persist orphaned block but no block found", {...logCtx, ...blockLogCtx});
-          }
-        })
-      );
+  for (const block of finalizedNonCanonicalBlocks) {
+    if (block.payloadStatus === PayloadStatus.FULL) {
+      orphanedPayloads.push(block);
+    }
+    // Every Gloas block gets EMPTY before FULL. If only FULL survives pruning, an earlier archive run
+    // already handled the root's block-level data.
+    const isBlockNode = !isGloasBlock(block) || block.payloadStatus === PayloadStatus.EMPTY;
+    if (isBlockNode && !canonicalRoots.has(block.blockRoot)) {
+      orphanedBlocks.push(block);
     }
+  }
+  const orphanedBlockRoots = orphanedBlocks.map((block) => fromHex(block.blockRoot));
 
-    const nonCanonicalSlots = finalizedNonCanonicalBlocks.map((summary) => summary.slot).sort((a, b) => a - b);
-    const nonCanonicalLogCtx = {
+  if (orphanedPayloads.length > 0) {
+    const orphanedPayloadRoots = orphanedPayloads.map((block) => fromHex(block.blockRoot));
+    const orphanedPayloadLogCtx = {
       ...logCtx,
-      count: nonCanonicalBlockRoots.length,
-      slotRange: prettyPrintIndices(nonCanonicalSlots),
+      count: orphanedPayloads.length,
+      slotRange: prettyPrintIndices(orphanedPayloads.map((block) => block.slot).sort((a, b) => a - b)),
     };
 
-    const columnItems = finalizedNonCanonicalBlocks
-      // Gloas EMPTY and FULL variants share a block root. EMPTY has no sidecars, so deleting by its root could
-      // remove the canonical FULL variant's columns. Pre-Gloas blocks are always FULL.
-      .filter(
-        (summary) => config.getForkSeq(summary.slot) >= ForkSeq.fulu && summary.payloadStatus === PayloadStatus.FULL
-      )
-      .map((summary) => ({slot: summary.slot, blockRoot: summary.blockRoot}));
+    const columnItems = orphanedPayloads
+      .filter((block) => config.getForkSeq(block.slot) >= ForkSeq.fulu)
+      .map((block) => ({slot: block.slot, blockRoot: block.blockRoot}));
     if (columnItems.length > 0) {
       // Delete sidecars first so their block roots remain available to retry cleanup after a failure or crash.
       await db.dataColumns.deleteMany(columnItems);
@@ -200,20 +208,44 @@ export async function archiveBlocks(
       });
     }
 
-    await db.block.batchDelete(nonCanonicalBlockRoots);
-    logger.verbose("Deleted non canonical blocks from hot DB", nonCanonicalLogCtx);
-
     if (finalizedPostDeneb) {
-      await db.blobSidecars.batchDelete(nonCanonicalBlockRoots);
-      logger.verbose("Deleted non canonical blobSidecars from hot DB", nonCanonicalLogCtx);
+      await db.blobSidecars.batchDelete(orphanedPayloadRoots);
+      logger.verbose("Deleted non canonical blobSidecars from hot DB", orphanedPayloadLogCtx);
     }
 
     if (finalizedPostGloas) {
-      await db.executionPayloadEnvelope.batchDelete(nonCanonicalBlockRoots);
-      logger.verbose("Deleted non canonical executionPayloadEnvelopes from hot DB", nonCanonicalLogCtx);
+      await db.executionPayloadEnvelope.batchDelete(orphanedPayloadRoots);
+      logger.verbose("Deleted non canonical executionPayloadEnvelopes from hot DB", orphanedPayloadLogCtx);
     }
   }
 
+  if (orphanedBlocks.length > 0) {
+    if (persistOrphanedBlocks) {
+      // Persist orphaned blocks to disk before deleting them from hot db
+      await Promise.all(
+        orphanedBlocks.map(async (block) => {
+          const blockBytes = await db.block.getBinary(fromHex(block.blockRoot));
+          const blockLogCtx = {slot: block.slot, root: block.blockRoot};
+          if (blockBytes) {
+            await persistOrphanedBlock(block.slot, block.blockRoot, blockBytes, {
+              persistOrphanedBlocksDir: persistOrphanedBlocksDir ?? "orphaned_blocks",
+            });
+            logger.verbose("Persisted orphaned block", {...logCtx, ...blockLogCtx});
+          } else {
+            logger.debug("Tried to persist orphaned block but no block found", {...logCtx, ...blockLogCtx});
+          }
+        })
+      );
+    }
+
+    await db.block.batchDelete(orphanedBlockRoots);
+    logger.verbose("Deleted non canonical blocks from hot DB", {
+      ...logCtx,
+      count: orphanedBlocks.length,
+      slotRange: prettyPrintIndices(orphanedBlocks.map((block) => block.slot).sort((a, b) => a - b)),
+    });
+  }
+
   // Delete expired blobs
   // Keep only `[current_epoch - max(MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS, archiveDataEpochs)]`
   // if archiveDataEpochs set to Infinity do not prune`
@@ -239,10 +271,7 @@ export async function archiveBlocks(
   // Keep only `[min(compute_blob_data_retention_start_epoch(current_epoch), current_epoch - archiveDataEpochs)]`
   if (finalizedPostFulu) {
     if (archiveDataEpochs !== Infinity) {
-      const dataColumnSidecarsMinEpoch = Math.min(
-        getBlobDataRetentionStartEpoch(config, currentEpoch),
-        currentEpoch - (archiveDataEpochs ?? 0)
-      );
+      const dataColumnSidecarsMinEpoch = getDataColumnSidecarsMinEpoch(config, currentEpoch, archiveDataEpochs);
       if (dataColumnSidecarsMinEpoch >= config.FULU_FORK_EPOCH) {
         const columnsPruneSlot = computeStartSlotAtEpoch(dataColumnSidecarsMinEpoch);
         const prunedColumnSlots = await db.dataColumns.pruneBefore(columnsPruneSlot);
@@ -266,9 +295,9 @@ export async function archiveBlocks(
     }
   }
 
-  // Prunning potential checkpoint data
+  // Pruning potential checkpoint data
   const finalizedCanonicalNonCheckpointBlocks = getNonCheckpointBlocks(finalizedCanonicalBlockRoots);
-  const nonCheckpointBlockRoots: Uint8Array[] = [...nonCanonicalBlockRoots];
+  const nonCheckpointBlockRoots: Uint8Array[] = [...orphanedBlockRoots];
   for (const block of finalizedCanonicalNonCheckpointBlocks) {
     nonCheckpointBlockRoots.push(block.root);
   }