@lodestar/fork-choice 1.51.0-dev.f8eff664d2 → 1.51.0-dev.f921fbf6b7

package/src/protoArray/protoArray.ts

@@ -122,6 +122,11 @@ export class ProtoArray {
    * when called with the negative parameter value.
    */
   private ptcAttested = new Map<RootHex, BitArray>();
+  /**
+   * Execution engine inclusion list verdict per block, recorded on payload import from heze.
+   * Spec: heze/fork-choice.md#modified-store (payload_inclusion_list_satisfaction)
+   */
+  private payloadInclusionListSatisfaction = new Map<RootHex, boolean>();
 
   constructor({
     pruneThreshold,
@@ -938,6 +943,27 @@ export class ProtoArray {
     return this.getParentPayloadStatus(block) === PayloadStatus.FULL;
   }
 
+  /** Spec: heze/fork-choice.md#new-record_payload_inclusion_list_satisfaction. Returns whether the verdict changed */
+  recordPayloadInclusionListSatisfaction(blockRoot: RootHex, satisfied: boolean): boolean {
+    const previous = this.payloadInclusionListSatisfaction.get(blockRoot);
+    this.payloadInclusionListSatisfaction.set(blockRoot, satisfied);
+    return previous !== satisfied;
+  }
+
+  /**
+   * Spec: heze/fork-choice.md#new-is_payload_inclusion_list_satisfied
+   *
+   * False until the payload is delivered. A delivered payload without a verdict is pre-heze, where
+   * no inclusion list constraints apply.
+   */
+  isPayloadInclusionListSatisfied(blockRoot: RootHex): boolean {
+    return this.hasPayload(blockRoot) && this.payloadInclusionListSatisfaction.get(blockRoot) !== false;
+  }
+
+  getPayloadInclusionListSatisfaction(blockRoot: RootHex): boolean | null {
+    return this.payloadInclusionListSatisfaction.get(blockRoot) ?? null;
+  }
+
   /**
    * Determine if we should extend the payload (prefer FULL over EMPTY)
    * Spec: gloas/fork-choice.md#new-should_extend_payload
@@ -956,6 +982,10 @@ export class ProtoArray {
       return false;
     }
 
+    if (this.payloadInclusionListSatisfaction.get(blockRoot) === false) {
+      return false;
+    }
+
     // Condition 1: Payload is timely AND data is available
     if (this.isPayloadTimely(blockRoot) && this.isPayloadDataAvailable(blockRoot)) {
       return true;
@@ -989,12 +1019,23 @@ export class ProtoArray {
   }
 
   /**
-   * Optimistic sync validate till validated latest hash, invalidate any descendant branch
-   * if invalidate till hash provided. If consensus fails, this will invalidate entire
-   * forkChoice which will throw on any call to findHead
+   * Apply an execution engine verdict with a latest valid hash (LVH) to the tree. VALID flips Syncing nodes up to the
+   * LVH node. INVALID invalidates the node identified by `invalidateFromParentBlockRoot/Hash` and its ancestors up to
+   * the LVH node (exclusive), then every descendant. Returns whether any status changed.
+   *
+   * A null or unknown LVH means the engine only vouches for the queried payload being invalid: the queried node and
+   * the nodes that inherit that payload (gloas PENDING and EMPTY variants) are invalidated, nothing above them.
+   *
+   * `strict` keeps the consensus failure semantics for verdicts about a payload the engine just processed: a VALID
+   * or PreMerge node on the invalidated chain poisons the tree. Verdicts from forkchoiceUpdated or about an ancestor
+   * chain are applied with `strict: false`: a node the engine already validated is never invalidated and the justified
+   * node is never crossed. If the walk to the LVH meets either, the LVH is treated as unknown.
    */
-  // TODO GLOAS: Review usage of this post-gloas
-  validateLatestHash(execResponse: LVHExecResponse, currentSlot: Slot): void {
+  validateLatestHash(
+    execResponse: LVHExecResponse,
+    currentSlot: Slot,
+    {strict = true}: {strict?: boolean} = {}
+  ): boolean {
     // Look reverse because its highly likely node with latestValidExecHash is towards the
     // the leaves of the forkchoice
     //
@@ -1005,71 +1046,128 @@ export class ProtoArray {
     // https://github.com/ChainSafe/lodestar/pull/4182#discussion_r914770167
 
     if (execResponse.executionStatus === ExecutionStatus.Valid) {
-      const {latestValidExecHash} = execResponse;
+      const {latestValidExecHash, blockRoot} = execResponse;
       // We use -1 for denoting not found
       let latestValidHashIndex = -1;
 
-      for (let nodeIndex = this.nodes.length - 1; nodeIndex >= 0; nodeIndex--) {
-        if (this.nodes[nodeIndex].executionPayloadBlockHash === latestValidExecHash) {
-          latestValidHashIndex = nodeIndex;
-          // We found the block corresponding to latestValidHashIndex, exit the loop
-          break;
+      if (blockRoot !== undefined) {
+        latestValidHashIndex = this.getNodeIndexByRootAndBlockHash(blockRoot, latestValidExecHash) ?? -1;
+      } else {
+        for (let nodeIndex = this.nodes.length - 1; nodeIndex >= 0; nodeIndex--) {
+          if (this.nodes[nodeIndex].executionPayloadBlockHash === latestValidExecHash) {
+            latestValidHashIndex = nodeIndex;
+            // We found the block corresponding to latestValidHashIndex, exit the loop
+            break;
+          }
         }
       }
 
       // We are trying to be as forgiving as possible here because ideally latestValidHashIndex
       // should be found in the forkchoice
-      if (latestValidHashIndex >= 0) {
-        this.propagateValidExecutionStatusByIndex(latestValidHashIndex);
+      if (latestValidHashIndex < 0) {
+        return false;
       }
-    } else {
-      // In case of invalidation, ideally:
-      //  i) Find the invalid payload
-      //  ii) Obtain a chain [LVH.child, LVH.child.child, ....., invalid_payload]
-      //  iii) Obtain a chain [Last_known_valid_node,  ...., LVH]
-      //
-      // Mark chain iii) as Valid if LVH is non null but right now LVH can be non null without
-      //  gurranteing chain iii) to be valid: for e.g. in following scenario LVH can be returned
-      //  as any of SYNCING: SYNCING, SYNCING, SYNCING, INVALID (due to simple check)/
-      //  So we currently ignore this chain and hope eventually it gets resolved
-      //
-      // Mark chain ii) as Invalid if LVH is found and non null, else only invalidate invalid_payload
-      // if its in fcU.
-      //
-      const {invalidateFromParentBlockRoot, invalidateFromParentBlockHash, latestValidExecHash} = execResponse;
-      const invalidateFromParentIndex = this.getNodeIndexByRootAndBlockHash(
-        invalidateFromParentBlockRoot,
-        invalidateFromParentBlockHash
-      );
-      if (invalidateFromParentIndex === undefined) {
-        throw Error(
-          `Unable to find invalidateFromParentBlockRoot=${invalidateFromParentBlockRoot} invalidateFromParentBlockHash=${invalidateFromParentBlockHash} in forkChoice`
-        );
+      const {executionStatus} = this.nodes[latestValidHashIndex];
+      if (executionStatus === ExecutionStatus.Valid || executionStatus === ExecutionStatus.PreMerge) {
+        return false;
       }
-      const latestValidHashIndex =
-        latestValidExecHash !== null ? this.getNodeIndexFromLVH(latestValidExecHash, invalidateFromParentIndex) : null;
-      if (latestValidHashIndex === null) {
-        /**
-         * The LVH (latest valid hash) is null or not found.
-         *
-         * The spec gives an allowance for the EL being able to return a nullish LVH if it could not
-         * "determine" one. There are two interpretations:
-         *
-         * - "the LVH is unknown" - simply throw and move on. We can't determine which chain to invalidate
-         *   since we don't know which ancestor is valid.
-         *
-         * - "the LVH doesn't exist" - this means that the entire ancestor chain is invalid, and should
-         *   be marked as such.
-         *
-         * The more robust approach is to treat nullish LVH as "the LVH is unknown" rather than
-         * "the LVH doesn't exist". The alternative means that we will poison a valid chain when the
-         * EL is lazy (or buggy) with its LVH response.
-         */
+      if (!strict) {
+        // The walk up flips every Syncing ancestor, a lenient verdict must not contradict an invalidated one
+        let nodeIndex: number | undefined = latestValidHashIndex;
+        while (nodeIndex !== undefined) {
+          const node: ProtoNode = this.nodes[nodeIndex];
+          if (node.executionStatus === ExecutionStatus.Valid || node.executionStatus === ExecutionStatus.PreMerge) {
+            break;
+          }
+          if (node.executionStatus === ExecutionStatus.Invalid) {
+            return false;
+          }
+          nodeIndex = node.parent;
+        }
+      }
+      this.propagateValidExecutionStatusByIndex(latestValidHashIndex);
+      return true;
+    }
+
+    // In case of invalidation, ideally:
+    //  i) Find the invalid payload
+    //  ii) Obtain a chain [LVH.child, LVH.child.child, ....., invalid_payload]
+    //  iii) Obtain a chain [Last_known_valid_node,  ...., LVH]
+    //
+    // Mark chain iii) as Valid if LVH is non null but right now LVH can be non null without
+    //  gurranteing chain iii) to be valid: for e.g. in following scenario LVH can be returned
+    //  as any of SYNCING: SYNCING, SYNCING, SYNCING, INVALID (due to simple check)/
+    //  So we currently ignore this chain and hope eventually it gets resolved
+    //
+    // Mark chain ii) as Invalid if LVH is found and non null, else only invalidate invalid_payload
+    // if its in fcU.
+    //
+    const {invalidateFromParentBlockRoot, invalidateFromParentBlockHash, latestValidExecHash} = execResponse;
+    const invalidateFromParentIndex = this.getNodeIndexByRootAndBlockHash(
+      invalidateFromParentBlockRoot,
+      invalidateFromParentBlockHash
+    );
+    if (invalidateFromParentIndex === undefined) {
+      throw Error(
+        `Unable to find invalidateFromParentBlockRoot=${invalidateFromParentBlockRoot} invalidateFromParentBlockHash=${invalidateFromParentBlockHash} in forkChoice`
+      );
+    }
+    if (!strict && this.nodes[invalidateFromParentIndex].executionStatus !== ExecutionStatus.Syncing) {
+      return false;
+    }
+
+    let latestValidHashIndex =
+      latestValidExecHash !== null ? this.getNodeIndexFromLVH(latestValidExecHash, invalidateFromParentIndex) : null;
+    if (latestValidHashIndex === null) {
+      if (strict) {
+        // The strict from node is the parent of the rejected payload and may be valid, never guess here
         throw Error(`Unable to find latestValidExecHash=${latestValidExecHash} in the forkchoice`);
       }
+      latestValidHashIndex = this.getUnknownLVHIndex(invalidateFromParentIndex);
+    }
+
+    if (!strict) {
+      // Never invalidate the justified block, the EMPTY variant sharing its chain, or anything at or below it in
+      // the tree. The justified block's own payload (FULL variant) is not part of the checkpoint
+      const justifiedIndex = this.getDefaultNodeIndex(this.justifiedRoot) ?? -1;
+      let nodeIndex: number | undefined = invalidateFromParentIndex;
+      while (nodeIndex !== undefined && nodeIndex > latestValidHashIndex) {
+        const node: ProtoNode = this.nodes[nodeIndex];
+        if (
+          node.executionStatus !== ExecutionStatus.Syncing ||
+          nodeIndex <= justifiedIndex ||
+          (node.blockRoot === this.justifiedRoot && node.payloadStatus === PayloadStatus.EMPTY)
+        ) {
+          // Conflicting LVH: fall back to the unknown LVH bound, but never cross this node
+          latestValidHashIndex = Math.max(nodeIndex, this.getUnknownLVHIndex(invalidateFromParentIndex));
+          break;
+        }
+        nodeIndex = node.parent;
+      }
+    }
+
+    if (invalidateFromParentIndex <= latestValidHashIndex) {
+      return false;
+    }
+
+    this.propagateInValidExecutionStatusByIndex(invalidateFromParentIndex, latestValidHashIndex, currentSlot);
+    return true;
+  }
 
-      this.propagateInValidExecutionStatusByIndex(invalidateFromParentIndex, latestValidHashIndex, currentSlot);
+  /**
+   * Upper bound (exclusive) for invalidating from `fromIndex` when the engine gave no usable LVH: the parent of the
+   * last ancestor that still inherits the queried payload. Pre-gloas this is the parent of `fromIndex`. For a gloas
+   * EMPTY or PENDING node it reaches up to the FULL node that owns the payload. -1 when the chain reaches the root.
+   */
+  private getUnknownLVHIndex(fromIndex: number): number {
+    const payloadBlockHash = this.nodes[fromIndex].executionPayloadBlockHash;
+    let nodeIndex = fromIndex;
+    let parentIndex = this.nodes[nodeIndex].parent;
+    while (parentIndex !== undefined && this.nodes[parentIndex].executionPayloadBlockHash === payloadBlockHash) {
+      nodeIndex = parentIndex;
+      parentIndex = this.nodes[nodeIndex].parent;
     }
+    return parentIndex ?? -1;
   }
 
   private propagateValidExecutionStatusByIndex(validNodeIndex: number): void {
@@ -1265,6 +1363,15 @@ export class ProtoArray {
 
     // For Gloas: check if from previous slot
     if (node.slot + 1 !== currentSlot) {
+      // During the block's own slot a payload known to not satisfy the inclusion list
+      // constraints ranks below its EMPTY variant, as it will from the next slot on
+      if (
+        node.slot === currentSlot &&
+        node.payloadStatus === PayloadStatus.FULL &&
+        this.payloadInclusionListSatisfaction.get(node.blockRoot) === false
+      ) {
+        return PayloadStatus.PENDING;
+      }
       return node.payloadStatus;
     }
 
@@ -1399,6 +1506,7 @@ export class ProtoArray {
       this.payloadTimelinessVotes.delete(root);
       this.ptcAttested.delete(root);
       this.payloadDataAvailabilityVotes.delete(root);
+      this.payloadInclusionListSatisfaction.delete(root);
     }
 
     // Store nodes prior to finalization