@lodestar/beacon-node 1.35.0-dev.7a3b3a996e → 1.35.0-dev.811265f1e2

package/lib/chain/seenCache/seenCommitteeContribution.js

@@ -11,11 +11,10 @@ const MAX_SLOTS_IN_CACHE = 8;
  * This stays in-memory and should be pruned per slot.
  */
 export class SeenContributionAndProof {
-    metrics;
-    seenAggregatorBySlot = new MapDef(() => new Set());
-    seenContributionBySlot = new MapDef(() => new MapDef(() => []));
     constructor(metrics) {
         this.metrics = metrics;
+        this.seenAggregatorBySlot = new MapDef(() => new Set());
+        this.seenContributionBySlot = new MapDef(() => new MapDef(() => []));
     }
     /**
      * _[IGNORE]_ A valid sync committee contribution with equal `slot`, `beacon_block_root` and `subcommittee_index` whose

package/lib/chain/seenCache/seenCommitteeContribution.js.map

@@ -1 +1 @@
-{"version":3,"file":"seenCommitteeContribution.js","sourceRoot":"","sources":["../../../src/chain/seenCache/seenCommitteeContribution.ts"],"names":[],"mappings":"AAEA,OAAO,EAAC,MAAM,EAAE,SAAS,EAAC,MAAM,iBAAiB,CAAC;AAElD,OAAO,EAAC,iBAAiB,EAAC,MAAM,wBAAwB,CAAC;AACzD,OAAO,EAAkB,UAAU,EAAC,MAAM,4BAA4B,CAAC;AAEvE;;GAEG;AACH,MAAM,kBAAkB,GAAG,CAAC,CAAC;AAQ7B;;;;GAIG;AACH,MAAM,OAAO,wBAAwB;IASN;IARZ,oBAAoB,GAAG,IAAI,MAAM,CAChD,GAAG,EAAE,CAAC,IAAI,GAAG,EAAuB,CACrC,CAAC;IAEe,sBAAsB,GAAG,IAAI,MAAM,CAClD,GAAG,EAAE,CAAC,IAAI,MAAM,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAC3B,CAAC;IAEF,YAA6B,OAAuB;QAAvB,YAAO,GAAP,OAAO,CAAgB;IAAG,CAAC;IAExD;;;OAGG;IACH,iBAAiB,CAAC,YAAuC;QACvD,MAAM,EAAC,eAAe,EAAE,IAAI,EAAC,GAAG,YAAY,CAAC;QAC7C,MAAM,eAAe,GAAG,IAAI,CAAC,sBAAsB,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;QACvE,MAAM,sBAAsB,GAAG,eAAe,CAAC,YAAY,CAAC,qBAAqB,CAAC,YAAY,CAAC,CAAC,CAAC;QACjG,IAAI,CAAC,OAAO,EAAE,SAAS,CAAC,sBAAsB,CAAC,YAAY,CAAC,GAAG,EAAE,CAAC;QAClE,yDAAyD;QAEzD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,sBAAsB,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACvD,IAAI,iBAAiB,CAAC,sBAAsB,CAAC,CAAC,CAAC,CAAC,eAAe,EAAE,eAAe,CAAC,EAAE,CAAC;gBAClF,IAAI,CAAC,OAAO,EAAE,SAAS,CAAC,sBAAsB,CAAC,WAAW,CAAC,GAAG,EAAE,CAAC;gBACjE,IAAI,CAAC,OAAO,EAAE,SAAS,CAAC,sBAAsB,CAAC,kBAAkB,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;gBACjF,OAAO,IAAI,CAAC;YACd,CAAC;QACH,CAAC;QAED,IAAI,CAAC,OAAO,EAAE,SAAS,CAAC,sBAAsB,CAAC,kBAAkB,CAAC,OAAO,CAAC,sBAAsB,CAAC,MAAM,CAAC,CAAC;QACzG,OAAO,KAAK,CAAC;IACf,CAAC;IAED;;;;OAIG;IACH,iBAAiB,CAAC,IAAU,EAAE,iBAAyB,EAAE,eAA+B;QACtF,OAAO,IAAI,CAAC,oBAAoB,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,GAAG,CAAC,iBAAiB,CAAC,iBAAiB,EAAE,eAAe,CAAC,CAAC,KAAK,IAAI,CAAC;IAClH,CAAC;IAED,yCAAyC;IACzC,GAAG,CAAC,oBAA0C,EAAE,YAAoB;QAClE,MAAM,EAAC,YAAY,EAAE,eAAe,EAAC,GAAG,oBAAoB,CAAC;QAC7D,MAAM,EAAC,iBAAiB,EAAE,IAAI,EAAE,eAAe,EAAC,GAAG,YAAY,CAAC;QAEhE,8BAA8B;QAC9B,IAAI,CAAC,oBAAoB,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,iBAAiB,CAAC,iBAAiB,EAAE,eAAe,CAAC,CAAC,CAAC;QAExG,gCAAgC;QAChC,MAAM,eAAe,GAAG,IAAI,CAAC,sBAAsB,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;QACvE,MAAM,sBAAsB,GAAG,eAAe,CAAC,YAAY,CAAC,qBAAqB,CAAC,YAAY,CAAC,CAAC,CAAC;QACjG,UAAU,CAAC,sBAAsB,EAAE,EAAC,eAAe,EAAE,YAAY,EAAC,CAAC,CAAC;IACtE,CAAC;IAED,0BAA0B;IAC1B,KAAK,CAAC,QAAc;QAClB,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,oBAAoB,CAAC,IAAI,EAAE,EAAE,CAAC;YACpD,IAAI,IAAI,GAAG,QAAQ,GAAG,kBAAkB,EAAE,CAAC;gBACzC,IAAI,CAAC,oBAAoB,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;YACzC,CAAC;QACH,CAAC;QAED,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,sBAAsB,CAAC,IAAI,EAAE,EAAE,CAAC;YACtD,IAAI,IAAI,GAAG,QAAQ,GAAG,kBAAkB,EAAE,CAAC;gBACzC,IAAI,CAAC,sBAAsB,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;YAC3C,CAAC;QACH,CAAC;IACH,CAAC;CACF;AAED,SAAS,iBAAiB,CAAC,iBAAyB,EAAE,eAA+B;IACnF,OAAO,GAAG,iBAAiB,IAAI,eAAe,EAAE,CAAC;AACnD,CAAC;AAED,SAAS,qBAAqB,CAAC,YAAuC;IACpE,MAAM,EAAC,IAAI,EAAE,eAAe,EAAE,iBAAiB,EAAC,GAAG,YAAY,CAAC;IAChE,OAAO,GAAG,IAAI,MAAM,SAAS,CAAC,eAAe,CAAC,MAAM,iBAAiB,EAAE,CAAC;AAC1E,CAAC"}
+{"version":3,"file":"seenCommitteeContribution.js","sourceRoot":"","sources":["../../../src/chain/seenCache/seenCommitteeContribution.ts"],"names":[],"mappings":"AAEA,OAAO,EAAC,MAAM,EAAE,SAAS,EAAC,MAAM,iBAAiB,CAAC;AAElD,OAAO,EAAC,iBAAiB,EAAC,MAAM,wBAAwB,CAAC;AACzD,OAAO,EAAkB,UAAU,EAAC,MAAM,4BAA4B,CAAC;AAEvE;;GAEG;AACH,MAAM,kBAAkB,GAAG,CAAC,CAAC;AAQ7B;;;;GAIG;AACH,MAAM,OAAO,wBAAwB;IASnC,YAA6B,OAAuB;QAAvB,YAAO,GAAP,OAAO,CAAgB;QARnC,yBAAoB,GAAG,IAAI,MAAM,CAChD,GAAG,EAAE,CAAC,IAAI,GAAG,EAAuB,CACrC,CAAC;QAEe,2BAAsB,GAAG,IAAI,MAAM,CAClD,GAAG,EAAE,CAAC,IAAI,MAAM,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAC3B,CAAC;IAEqD,CAAC;IAExD;;;OAGG;IACH,iBAAiB,CAAC,YAAuC;QACvD,MAAM,EAAC,eAAe,EAAE,IAAI,EAAC,GAAG,YAAY,CAAC;QAC7C,MAAM,eAAe,GAAG,IAAI,CAAC,sBAAsB,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;QACvE,MAAM,sBAAsB,GAAG,eAAe,CAAC,YAAY,CAAC,qBAAqB,CAAC,YAAY,CAAC,CAAC,CAAC;QACjG,IAAI,CAAC,OAAO,EAAE,SAAS,CAAC,sBAAsB,CAAC,YAAY,CAAC,GAAG,EAAE,CAAC;QAClE,yDAAyD;QAEzD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,sBAAsB,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACvD,IAAI,iBAAiB,CAAC,sBAAsB,CAAC,CAAC,CAAC,CAAC,eAAe,EAAE,eAAe,CAAC,EAAE,CAAC;gBAClF,IAAI,CAAC,OAAO,EAAE,SAAS,CAAC,sBAAsB,CAAC,WAAW,CAAC,GAAG,EAAE,CAAC;gBACjE,IAAI,CAAC,OAAO,EAAE,SAAS,CAAC,sBAAsB,CAAC,kBAAkB,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;gBACjF,OAAO,IAAI,CAAC;YACd,CAAC;QACH,CAAC;QAED,IAAI,CAAC,OAAO,EAAE,SAAS,CAAC,sBAAsB,CAAC,kBAAkB,CAAC,OAAO,CAAC,sBAAsB,CAAC,MAAM,CAAC,CAAC;QACzG,OAAO,KAAK,CAAC;IACf,CAAC;IAED;;;;OAIG;IACH,iBAAiB,CAAC,IAAU,EAAE,iBAAyB,EAAE,eAA+B;QACtF,OAAO,IAAI,CAAC,oBAAoB,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,GAAG,CAAC,iBAAiB,CAAC,iBAAiB,EAAE,eAAe,CAAC,CAAC,KAAK,IAAI,CAAC;IAClH,CAAC;IAED,yCAAyC;IACzC,GAAG,CAAC,oBAA0C,EAAE,YAAoB;QAClE,MAAM,EAAC,YAAY,EAAE,eAAe,EAAC,GAAG,oBAAoB,CAAC;QAC7D,MAAM,EAAC,iBAAiB,EAAE,IAAI,EAAE,eAAe,EAAC,GAAG,YAAY,CAAC;QAEhE,8BAA8B;QAC9B,IAAI,CAAC,oBAAoB,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,iBAAiB,CAAC,iBAAiB,EAAE,eAAe,CAAC,CAAC,CAAC;QAExG,gCAAgC;QAChC,MAAM,eAAe,GAAG,IAAI,CAAC,sBAAsB,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;QACvE,MAAM,sBAAsB,GAAG,eAAe,CAAC,YAAY,CAAC,qBAAqB,CAAC,YAAY,CAAC,CAAC,CAAC;QACjG,UAAU,CAAC,sBAAsB,EAAE,EAAC,eAAe,EAAE,YAAY,EAAC,CAAC,CAAC;IACtE,CAAC;IAED,0BAA0B;IAC1B,KAAK,CAAC,QAAc;QAClB,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,oBAAoB,CAAC,IAAI,EAAE,EAAE,CAAC;YACpD,IAAI,IAAI,GAAG,QAAQ,GAAG,kBAAkB,EAAE,CAAC;gBACzC,IAAI,CAAC,oBAAoB,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;YACzC,CAAC;QACH,CAAC;QAED,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,sBAAsB,CAAC,IAAI,EAAE,EAAE,CAAC;YACtD,IAAI,IAAI,GAAG,QAAQ,GAAG,kBAAkB,EAAE,CAAC;gBACzC,IAAI,CAAC,sBAAsB,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;YAC3C,CAAC;QACH,CAAC;IACH,CAAC;CACF;AAED,SAAS,iBAAiB,CAAC,iBAAyB,EAAE,eAA+B;IACnF,OAAO,GAAG,iBAAiB,IAAI,eAAe,EAAE,CAAC;AACnD,CAAC;AAED,SAAS,qBAAqB,CAAC,YAAuC;IACpE,MAAM,EAAC,IAAI,EAAE,eAAe,EAAE,iBAAiB,EAAC,GAAG,YAAY,CAAC;IAChE,OAAO,GAAG,IAAI,MAAM,SAAS,CAAC,eAAe,CAAC,MAAM,iBAAiB,EAAE,CAAC;AAC1E,CAAC"}

package/lib/chain/seenCache/seenGossipBlockInput.d.ts

@@ -1,90 +1,94 @@
 import { ChainForkConfig } from "@lodestar/config";
-import { CheckpointWithHex } from "@lodestar/fork-choice";
-import { RootHex, deneb, fulu } from "@lodestar/types";
+import { ForkName } from "@lodestar/params";
+import { RootHex, SignedBeaconBlock, deneb, fulu } from "@lodestar/types";
 import { Logger } from "@lodestar/utils";
-import { Metrics } from "../../metrics/metrics.js";
+import { IExecutionEngine } from "../../execution/index.js";
+import { Metrics } from "../../metrics/index.js";
 import { IClock } from "../../util/clock.js";
 import { CustodyConfig } from "../../util/dataColumns.js";
-import { BlockInput, BlockInputBlobs, BlockInputColumns, BlockWithSource, IBlockInput, SourceMeta } from "../blocks/blockInput/index.js";
+import { BlockInput, CachedData, GossipedInputType, NullBlockInput } from "../blocks/types.js";
 import { ChainEventEmitter } from "../emitter.js";
-export type SeenBlockInputCacheModules = {
-    config: ChainForkConfig;
-    clock: IClock;
-    chainEvents: ChainEventEmitter;
-    signal: AbortSignal;
-    custodyConfig: CustodyConfig;
-    metrics: Metrics | null;
-    logger?: Logger;
+export declare enum BlockInputAvailabilitySource {
+    GOSSIP = "gossip",
+    RECOVERED = "recovered",
+    UNKNOWN_SYNC = "unknown_sync"
+}
+type GossipedBlockInput = {
+    type: GossipedInputType.block;
+    signedBlock: SignedBeaconBlock;
+} | {
+    type: GossipedInputType.blob;
+    blobSidecar: deneb.BlobSidecar;
+} | {
+    type: GossipedInputType.dataColumn;
+    dataColumnSidecar: fulu.DataColumnSidecar;
+    dataColumnBytes: Uint8Array | null;
+};
+export type BlockInputCacheType = {
+    fork: ForkName;
+    block?: SignedBeaconBlock;
+    cachedData?: CachedData;
+    blockInputPromise: Promise<BlockInput>;
+    resolveBlockInput: (blockInput: BlockInput) => void;
+};
+type GossipBlockInputResponseWithBlock = {
+    blockInput: BlockInput;
+    blockInputMeta: {
+        pending: GossipedInputType.blob | null;
+        haveBlobs: number;
+        expectedBlobs: number;
+    } | {
+        pending: GossipedInputType.dataColumn | null;
+        haveColumns: number;
+        expectedColumns: number;
+    };
+};
+type BlockInputPendingBlock = {
+    pending: GossipedInputType.block;
+};
+export type BlockInputMetaPendingBlockWithBlobs = BlockInputPendingBlock & {
+    haveBlobs: number;
+    expectedBlobs: null;
+};
+type BlockInputMetaPendingBlockWithColumns = BlockInputPendingBlock & {
+    haveColumns: number;
+    expectedColumns: null;
 };
-export type GetByBlobOptions = {
-    throwErrorIfAlreadyKnown?: boolean;
+type GossipBlockInputResponseWithNullBlock = {
+    blockInput: NullBlockInput;
+    blockInputMeta: BlockInputMetaPendingBlockWithBlobs | BlockInputMetaPendingBlockWithColumns;
 };
+type GossipBlockInputResponse = GossipBlockInputResponseWithBlock | GossipBlockInputResponseWithNullBlock;
 /**
- * Consumers that create BlockInputs or change types of old BlockInputs
+ * For predeneb, SeenGossipBlockInput only tracks and caches block so that we don't need to download known block
+ * roots. From deneb, it serves same purpose plus tracks and caches the live blobs and blocks on the network to
+ * solve data availability for the blockInput. If no block has been seen yet for some already seen blobs, it
+ * responds will null, but on the first block or the consequent blobs it responds with blobs promise till all blobs
+ * become available.
  *
- * - gossipHandlers (block and blob)
- * - beaconBlocksMaybeBlobsByRange
- * - unavailableBeaconBlobsByRoot (beaconBlocksMaybeBlobsByRoot)
- * - publishBlock in the beacon/blocks/index.ts API
- *   https://github.com/ChainSafe/lodestar/blob/unstable/packages/beacon-node/src/api/impl/beacon/blocks/index.ts#L62
- * - maybeValidateBlobs in verifyBlocksDataAvailability (is_data_available spec function)
- *   https://github.com/ChainSafe/lodestar/blob/unstable/packages/beacon-node/src/chain/blocks/verifyBlocksDataAvailability.ts#L111
- *
- *
- * Pruning management for SeenBlockInputCache
- * ------------------------------------------
- * There are four cases for how pruning needs to be handled
- * - Normal operation following head via gossip (and/or reqresp). For this situation the consumer (process pipeline or
- *   caller of processBlock) will call the `prune` method to remove any processed BlockInputs from the cache. This will
- *   also remove any ancestors of the processed BlockInput as that will also need to have been successfully processed
- *   for import to work correctly
- * - onFinalized event handler will help to prune any non-canonical forks once the chain finalizes. Any block-slots that
- *   are before the finalized checkpoint will be pruned.
- * - Range-sync periods.  The range process uses this cache to store and sync blocks with DA data as the chain is pulled
- *   from peers.  We pull batches, by epoch, so 32 slots are pulled at a time and several batches are pulled concurrently.
- *   It is important to set the MAX_BLOCK_INPUT_CACHE_SIZE high enough to support range sync activities.  Currently the
- *   value is set for 5 batches of 32 slots.  As process block is called (similar to following head) the BlockInput and
- *   its ancestors will be pruned.
- * - Non-Finality times.  This is a bit more tricky.  There can be long periods of non-finality and storing everything
- *   will cause OOM.  The pruneToMax will help ensure a hard limit on the number of stored blocks (with DA) that are held
- *   in memory at any one time.  The value for MAX_BLOCK_INPUT_CACHE_SIZE is set to accommodate range-sync but in
- *   practice this value may need to be massaged in the future if we find issues when debugging non-finality
+ * One can start processing block on blobs promise blockInput response and can await on the promise before
+ * fully importing the block. The blobs promise is gets resolved as soon as all blobs corresponding to that
+ * block are seen by SeenGossipBlockInput
  */
-export declare class SeenBlockInput {
-    private readonly config;
+export declare class SeenGossipBlockInput {
+    private readonly blockInputCache;
     private readonly custodyConfig;
+    private readonly executionEngine;
     private readonly clock;
-    private readonly chainEvents;
-    private readonly signal;
-    private readonly metrics;
-    private readonly logger?;
-    private blockInputs;
-    constructor({ config, custodyConfig, clock, chainEvents, signal, metrics, logger }: SeenBlockInputCacheModules);
-    has(rootHex: RootHex): boolean;
-    get(rootHex: RootHex): IBlockInput | undefined;
-    /**
-     * Removes the single BlockInput from the cache
-     */
-    remove(rootHex: RootHex): void;
-    /**
-     * Removes a processed BlockInput from the cache and also removes any ancestors of processed blocks
-     */
-    prune(rootHex: RootHex): void;
-    onFinalized: (checkpoint: CheckpointWithHex) => void;
-    getByBlock({ blockRootHex, block, source, seenTimestampSec, peerIdStr }: BlockWithSource): BlockInput;
-    getByBlob({ blockRootHex, blobSidecar, source, seenTimestampSec, peerIdStr, }: SourceMeta & {
-        blockRootHex: RootHex;
-        blobSidecar: deneb.BlobSidecar;
-    }, opts?: GetByBlobOptions): BlockInputBlobs;
-    getByColumn({ blockRootHex, columnSidecar, seenTimestampSec, source, peerIdStr, }: SourceMeta & {
-        blockRootHex: RootHex;
-        columnSidecar: fulu.DataColumnSidecar;
-    }, opts?: GetByBlobOptions): BlockInputColumns;
-    private buildCommonProps;
+    private readonly emitter;
+    private readonly logger;
+    constructor(custodyConfig: CustodyConfig, executionEngine: IExecutionEngine, emitter: ChainEventEmitter, clock: IClock, logger: Logger);
+    globalCacheId: number;
+    prune(): void;
+    hasBlock(blockRoot: RootHex): boolean;
     /**
-     * Use custom implementation of pruneSetToMax to allow for sorting by slot
-     * and deleting via key/rootHex
+     * Intended to be used for gossip validation, specifically this check:
+     * [IGNORE] The sidecar is the first sidecar for the tuple (block_header.slot, block_header.proposer_index,
+     *          sidecar.index) with valid header signature, sidecar inclusion proof, and kzg proof
      */
-    private pruneToMaxSize;
+    hasDataColumnSidecar(sidecar: fulu.DataColumnSidecar): boolean;
+    getGossipBlockInput(config: ChainForkConfig, gossipedInput: GossipedBlockInput, metrics: Metrics | null): GossipBlockInputResponse;
 }
+export declare function getEmptyBlockInputCacheEntry(fork: ForkName, globalCacheId: number): BlockInputCacheType;
+export {};
 //# sourceMappingURL=seenGossipBlockInput.d.ts.map