@lodestar/beacon-node 1.34.1 → 1.35.0-dev.092cc3954a

package/lib/chain/seenCache/seenCommitteeContribution.js.map

@@ -1 +1 @@
-{"version":3,"file":"seenCommitteeContribution.js","sourceRoot":"","sources":["../../../src/chain/seenCache/seenCommitteeContribution.ts"],"names":[],"mappings":"AAEA,OAAO,EAAC,MAAM,EAAE,SAAS,EAAC,MAAM,iBAAiB,CAAC;AAElD,OAAO,EAAC,iBAAiB,EAAC,MAAM,wBAAwB,CAAC;AACzD,OAAO,EAAkB,UAAU,EAAC,MAAM,4BAA4B,CAAC;AAEvE;;GAEG;AACH,MAAM,kBAAkB,GAAG,CAAC,CAAC;AAQ7B;;;;GAIG;AACH,MAAM,OAAO,wBAAwB;IASnC,YAA6B,OAAuB;QAAvB,YAAO,GAAP,OAAO,CAAgB;QARnC,yBAAoB,GAAG,IAAI,MAAM,CAChD,GAAG,EAAE,CAAC,IAAI,GAAG,EAAuB,CACrC,CAAC;QAEe,2BAAsB,GAAG,IAAI,MAAM,CAClD,GAAG,EAAE,CAAC,IAAI,MAAM,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAC3B,CAAC;IAEqD,CAAC;IAExD;;;OAGG;IACH,iBAAiB,CAAC,YAAuC;QACvD,MAAM,EAAC,eAAe,EAAE,IAAI,EAAC,GAAG,YAAY,CAAC;QAC7C,MAAM,eAAe,GAAG,IAAI,CAAC,sBAAsB,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;QACvE,MAAM,sBAAsB,GAAG,eAAe,CAAC,YAAY,CAAC,qBAAqB,CAAC,YAAY,CAAC,CAAC,CAAC;QACjG,IAAI,CAAC,OAAO,EAAE,SAAS,CAAC,sBAAsB,CAAC,YAAY,CAAC,GAAG,EAAE,CAAC;QAClE,yDAAyD;QAEzD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,sBAAsB,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACvD,IAAI,iBAAiB,CAAC,sBAAsB,CAAC,CAAC,CAAC,CAAC,eAAe,EAAE,eAAe,CAAC,EAAE,CAAC;gBAClF,IAAI,CAAC,OAAO,EAAE,SAAS,CAAC,sBAAsB,CAAC,WAAW,CAAC,GAAG,EAAE,CAAC;gBACjE,IAAI,CAAC,OAAO,EAAE,SAAS,CAAC,sBAAsB,CAAC,kBAAkB,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;gBACjF,OAAO,IAAI,CAAC;YACd,CAAC;QACH,CAAC;QAED,IAAI,CAAC,OAAO,EAAE,SAAS,CAAC,sBAAsB,CAAC,kBAAkB,CAAC,OAAO,CAAC,sBAAsB,CAAC,MAAM,CAAC,CAAC;QACzG,OAAO,KAAK,CAAC;IACf,CAAC;IAED;;;;OAIG;IACH,iBAAiB,CAAC,IAAU,EAAE,iBAAyB,EAAE,eAA+B;QACtF,OAAO,IAAI,CAAC,oBAAoB,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,GAAG,CAAC,iBAAiB,CAAC,iBAAiB,EAAE,eAAe,CAAC,CAAC,KAAK,IAAI,CAAC;IAClH,CAAC;IAED,yCAAyC;IACzC,GAAG,CAAC,oBAA0C,EAAE,YAAoB;QAClE,MAAM,EAAC,YAAY,EAAE,eAAe,EAAC,GAAG,oBAAoB,CAAC;QAC7D,MAAM,EAAC,iBAAiB,EAAE,IAAI,EAAE,eAAe,EAAC,GAAG,YAAY,CAAC;QAEhE,8BAA8B;QAC9B,IAAI,CAAC,oBAAoB,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,iBAAiB,CAAC,iBAAiB,EAAE,eAAe,CAAC,CAAC,CAAC;QAExG,gCAAgC;QAChC,MAAM,eAAe,GAAG,IAAI,CAAC,sBAAsB,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;QACvE,MAAM,sBAAsB,GAAG,eAAe,CAAC,YAAY,CAAC,qBAAqB,CAAC,YAAY,CAAC,CAAC,CAAC;QACjG,UAAU,CAAC,sBAAsB,EAAE,EAAC,eAAe,EAAE,YAAY,EAAC,CAAC,CAAC;IACtE,CAAC;IAED,0BAA0B;IAC1B,KAAK,CAAC,QAAc;QAClB,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,oBAAoB,CAAC,IAAI,EAAE,EAAE,CAAC;YACpD,IAAI,IAAI,GAAG,QAAQ,GAAG,kBAAkB,EAAE,CAAC;gBACzC,IAAI,CAAC,oBAAoB,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;YACzC,CAAC;QACH,CAAC;QAED,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,sBAAsB,CAAC,IAAI,EAAE,EAAE,CAAC;YACtD,IAAI,IAAI,GAAG,QAAQ,GAAG,kBAAkB,EAAE,CAAC;gBACzC,IAAI,CAAC,sBAAsB,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;YAC3C,CAAC;QACH,CAAC;IACH,CAAC;CACF;AAED,SAAS,iBAAiB,CAAC,iBAAyB,EAAE,eAA+B;IACnF,OAAO,GAAG,iBAAiB,IAAI,eAAe,EAAE,CAAC;AACnD,CAAC;AAED,SAAS,qBAAqB,CAAC,YAAuC;IACpE,MAAM,EAAC,IAAI,EAAE,eAAe,EAAE,iBAAiB,EAAC,GAAG,YAAY,CAAC;IAChE,OAAO,GAAG,IAAI,MAAM,SAAS,CAAC,eAAe,CAAC,MAAM,iBAAiB,EAAE,CAAC;AAC1E,CAAC"}
+{"version":3,"file":"seenCommitteeContribution.js","sourceRoot":"","sources":["../../../src/chain/seenCache/seenCommitteeContribution.ts"],"names":[],"mappings":"AAEA,OAAO,EAAC,MAAM,EAAE,SAAS,EAAC,MAAM,iBAAiB,CAAC;AAElD,OAAO,EAAC,iBAAiB,EAAC,MAAM,wBAAwB,CAAC;AACzD,OAAO,EAAkB,UAAU,EAAC,MAAM,4BAA4B,CAAC;AAEvE;;GAEG;AACH,MAAM,kBAAkB,GAAG,CAAC,CAAC;AAQ7B;;;;GAIG;AACH,MAAM,OAAO,wBAAwB;IASN;IARZ,oBAAoB,GAAG,IAAI,MAAM,CAChD,GAAG,EAAE,CAAC,IAAI,GAAG,EAAuB,CACrC,CAAC;IAEe,sBAAsB,GAAG,IAAI,MAAM,CAClD,GAAG,EAAE,CAAC,IAAI,MAAM,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAC3B,CAAC;IAEF,YAA6B,OAAuB;QAAvB,YAAO,GAAP,OAAO,CAAgB;IAAG,CAAC;IAExD;;;OAGG;IACH,iBAAiB,CAAC,YAAuC;QACvD,MAAM,EAAC,eAAe,EAAE,IAAI,EAAC,GAAG,YAAY,CAAC;QAC7C,MAAM,eAAe,GAAG,IAAI,CAAC,sBAAsB,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;QACvE,MAAM,sBAAsB,GAAG,eAAe,CAAC,YAAY,CAAC,qBAAqB,CAAC,YAAY,CAAC,CAAC,CAAC;QACjG,IAAI,CAAC,OAAO,EAAE,SAAS,CAAC,sBAAsB,CAAC,YAAY,CAAC,GAAG,EAAE,CAAC;QAClE,yDAAyD;QAEzD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,sBAAsB,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACvD,IAAI,iBAAiB,CAAC,sBAAsB,CAAC,CAAC,CAAC,CAAC,eAAe,EAAE,eAAe,CAAC,EAAE,CAAC;gBAClF,IAAI,CAAC,OAAO,EAAE,SAAS,CAAC,sBAAsB,CAAC,WAAW,CAAC,GAAG,EAAE,CAAC;gBACjE,IAAI,CAAC,OAAO,EAAE,SAAS,CAAC,sBAAsB,CAAC,kBAAkB,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;gBACjF,OAAO,IAAI,CAAC;YACd,CAAC;QACH,CAAC;QAED,IAAI,CAAC,OAAO,EAAE,SAAS,CAAC,sBAAsB,CAAC,kBAAkB,CAAC,OAAO,CAAC,sBAAsB,CAAC,MAAM,CAAC,CAAC;QACzG,OAAO,KAAK,CAAC;IACf,CAAC;IAED;;;;OAIG;IACH,iBAAiB,CAAC,IAAU,EAAE,iBAAyB,EAAE,eAA+B;QACtF,OAAO,IAAI,CAAC,oBAAoB,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,GAAG,CAAC,iBAAiB,CAAC,iBAAiB,EAAE,eAAe,CAAC,CAAC,KAAK,IAAI,CAAC;IAClH,CAAC;IAED,yCAAyC;IACzC,GAAG,CAAC,oBAA0C,EAAE,YAAoB;QAClE,MAAM,EAAC,YAAY,EAAE,eAAe,EAAC,GAAG,oBAAoB,CAAC;QAC7D,MAAM,EAAC,iBAAiB,EAAE,IAAI,EAAE,eAAe,EAAC,GAAG,YAAY,CAAC;QAEhE,8BAA8B;QAC9B,IAAI,CAAC,oBAAoB,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,iBAAiB,CAAC,iBAAiB,EAAE,eAAe,CAAC,CAAC,CAAC;QAExG,gCAAgC;QAChC,MAAM,eAAe,GAAG,IAAI,CAAC,sBAAsB,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;QACvE,MAAM,sBAAsB,GAAG,eAAe,CAAC,YAAY,CAAC,qBAAqB,CAAC,YAAY,CAAC,CAAC,CAAC;QACjG,UAAU,CAAC,sBAAsB,EAAE,EAAC,eAAe,EAAE,YAAY,EAAC,CAAC,CAAC;IACtE,CAAC;IAED,0BAA0B;IAC1B,KAAK,CAAC,QAAc;QAClB,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,oBAAoB,CAAC,IAAI,EAAE,EAAE,CAAC;YACpD,IAAI,IAAI,GAAG,QAAQ,GAAG,kBAAkB,EAAE,CAAC;gBACzC,IAAI,CAAC,oBAAoB,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;YACzC,CAAC;QACH,CAAC;QAED,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,sBAAsB,CAAC,IAAI,EAAE,EAAE,CAAC;YACtD,IAAI,IAAI,GAAG,QAAQ,GAAG,kBAAkB,EAAE,CAAC;gBACzC,IAAI,CAAC,sBAAsB,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;YAC3C,CAAC;QACH,CAAC;IACH,CAAC;CACF;AAED,SAAS,iBAAiB,CAAC,iBAAyB,EAAE,eAA+B;IACnF,OAAO,GAAG,iBAAiB,IAAI,eAAe,EAAE,CAAC;AACnD,CAAC;AAED,SAAS,qBAAqB,CAAC,YAAuC;IACpE,MAAM,EAAC,IAAI,EAAE,eAAe,EAAE,iBAAiB,EAAC,GAAG,YAAY,CAAC;IAChE,OAAO,GAAG,IAAI,MAAM,SAAS,CAAC,eAAe,CAAC,MAAM,iBAAiB,EAAE,CAAC;AAC1E,CAAC"}

package/lib/chain/seenCache/seenGossipBlockInput.d.ts

@@ -1,94 +1,90 @@
 import { ChainForkConfig } from "@lodestar/config";
-import { ForkName } from "@lodestar/params";
-import { RootHex, SignedBeaconBlock, deneb, fulu } from "@lodestar/types";
+import { CheckpointWithHex } from "@lodestar/fork-choice";
+import { RootHex, deneb, fulu } from "@lodestar/types";
 import { Logger } from "@lodestar/utils";
-import { IExecutionEngine } from "../../execution/index.js";
-import { Metrics } from "../../metrics/index.js";
+import { Metrics } from "../../metrics/metrics.js";
 import { IClock } from "../../util/clock.js";
 import { CustodyConfig } from "../../util/dataColumns.js";
-import { BlockInput, CachedData, GossipedInputType, NullBlockInput } from "../blocks/types.js";
+import { BlockInput, BlockInputBlobs, BlockInputColumns, BlockWithSource, IBlockInput, SourceMeta } from "../blocks/blockInput/index.js";
 import { ChainEventEmitter } from "../emitter.js";
-export declare enum BlockInputAvailabilitySource {
-    GOSSIP = "gossip",
-    RECOVERED = "recovered",
-    UNKNOWN_SYNC = "unknown_sync"
-}
-type GossipedBlockInput = {
-    type: GossipedInputType.block;
-    signedBlock: SignedBeaconBlock;
-} | {
-    type: GossipedInputType.blob;
-    blobSidecar: deneb.BlobSidecar;
-} | {
-    type: GossipedInputType.dataColumn;
-    dataColumnSidecar: fulu.DataColumnSidecar;
-    dataColumnBytes: Uint8Array | null;
-};
-export type BlockInputCacheType = {
-    fork: ForkName;
-    block?: SignedBeaconBlock;
-    cachedData?: CachedData;
-    blockInputPromise: Promise<BlockInput>;
-    resolveBlockInput: (blockInput: BlockInput) => void;
-};
-type GossipBlockInputResponseWithBlock = {
-    blockInput: BlockInput;
-    blockInputMeta: {
-        pending: GossipedInputType.blob | null;
-        haveBlobs: number;
-        expectedBlobs: number;
-    } | {
-        pending: GossipedInputType.dataColumn | null;
-        haveColumns: number;
-        expectedColumns: number;
-    };
-};
-type BlockInputPendingBlock = {
-    pending: GossipedInputType.block;
-};
-export type BlockInputMetaPendingBlockWithBlobs = BlockInputPendingBlock & {
-    haveBlobs: number;
-    expectedBlobs: null;
-};
-type BlockInputMetaPendingBlockWithColumns = BlockInputPendingBlock & {
-    haveColumns: number;
-    expectedColumns: null;
+export type SeenBlockInputCacheModules = {
+    config: ChainForkConfig;
+    clock: IClock;
+    chainEvents: ChainEventEmitter;
+    signal: AbortSignal;
+    custodyConfig: CustodyConfig;
+    metrics: Metrics | null;
+    logger?: Logger;
 };
-type GossipBlockInputResponseWithNullBlock = {
-    blockInput: NullBlockInput;
-    blockInputMeta: BlockInputMetaPendingBlockWithBlobs | BlockInputMetaPendingBlockWithColumns;
+export type GetByBlobOptions = {
+    throwErrorIfAlreadyKnown?: boolean;
 };
-type GossipBlockInputResponse = GossipBlockInputResponseWithBlock | GossipBlockInputResponseWithNullBlock;
 /**
- * For predeneb, SeenGossipBlockInput only tracks and caches block so that we don't need to download known block
- * roots. From deneb, it serves same purpose plus tracks and caches the live blobs and blocks on the network to
- * solve data availability for the blockInput. If no block has been seen yet for some already seen blobs, it
- * responds will null, but on the first block or the consequent blobs it responds with blobs promise till all blobs
- * become available.
+ * Consumers that create BlockInputs or change types of old BlockInputs
  *
- * One can start processing block on blobs promise blockInput response and can await on the promise before
- * fully importing the block. The blobs promise is gets resolved as soon as all blobs corresponding to that
- * block are seen by SeenGossipBlockInput
+ * - gossipHandlers (block and blob)
+ * - beaconBlocksMaybeBlobsByRange
+ * - unavailableBeaconBlobsByRoot (beaconBlocksMaybeBlobsByRoot)
+ * - publishBlock in the beacon/blocks/index.ts API
+ *   https://github.com/ChainSafe/lodestar/blob/unstable/packages/beacon-node/src/api/impl/beacon/blocks/index.ts#L62
+ * - maybeValidateBlobs in verifyBlocksDataAvailability (is_data_available spec function)
+ *   https://github.com/ChainSafe/lodestar/blob/unstable/packages/beacon-node/src/chain/blocks/verifyBlocksDataAvailability.ts#L111
+ *
+ *
+ * Pruning management for SeenBlockInputCache
+ * ------------------------------------------
+ * There are four cases for how pruning needs to be handled
+ * - Normal operation following head via gossip (and/or reqresp). For this situation the consumer (process pipeline or
+ *   caller of processBlock) will call the `prune` method to remove any processed BlockInputs from the cache. This will
+ *   also remove any ancestors of the processed BlockInput as that will also need to have been successfully processed
+ *   for import to work correctly
+ * - onFinalized event handler will help to prune any non-canonical forks once the chain finalizes. Any block-slots that
+ *   are before the finalized checkpoint will be pruned.
+ * - Range-sync periods.  The range process uses this cache to store and sync blocks with DA data as the chain is pulled
+ *   from peers.  We pull batches, by epoch, so 32 slots are pulled at a time and several batches are pulled concurrently.
+ *   It is important to set the MAX_BLOCK_INPUT_CACHE_SIZE high enough to support range sync activities.  Currently the
+ *   value is set for 5 batches of 32 slots.  As process block is called (similar to following head) the BlockInput and
+ *   its ancestors will be pruned.
+ * - Non-Finality times.  This is a bit more tricky.  There can be long periods of non-finality and storing everything
+ *   will cause OOM.  The pruneToMax will help ensure a hard limit on the number of stored blocks (with DA) that are held
+ *   in memory at any one time.  The value for MAX_BLOCK_INPUT_CACHE_SIZE is set to accommodate range-sync but in
+ *   practice this value may need to be massaged in the future if we find issues when debugging non-finality
  */
-export declare class SeenGossipBlockInput {
-    private readonly blockInputCache;
+export declare class SeenBlockInput {
+    private readonly config;
     private readonly custodyConfig;
-    private readonly executionEngine;
     private readonly clock;
-    private readonly emitter;
-    private readonly logger;
-    constructor(custodyConfig: CustodyConfig, executionEngine: IExecutionEngine, emitter: ChainEventEmitter, clock: IClock, logger: Logger);
-    globalCacheId: number;
-    prune(): void;
-    hasBlock(blockRoot: RootHex): boolean;
+    private readonly chainEvents;
+    private readonly signal;
+    private readonly metrics;
+    private readonly logger?;
+    private blockInputs;
+    constructor({ config, custodyConfig, clock, chainEvents, signal, metrics, logger }: SeenBlockInputCacheModules);
+    has(rootHex: RootHex): boolean;
+    get(rootHex: RootHex): IBlockInput | undefined;
+    /**
+     * Removes the single BlockInput from the cache
+     */
+    remove(rootHex: RootHex): void;
+    /**
+     * Removes a processed BlockInput from the cache and also removes any ancestors of processed blocks
+     */
+    prune(rootHex: RootHex): void;
+    onFinalized: (checkpoint: CheckpointWithHex) => void;
+    getByBlock({ blockRootHex, block, source, seenTimestampSec, peerIdStr }: BlockWithSource): BlockInput;
+    getByBlob({ blockRootHex, blobSidecar, source, seenTimestampSec, peerIdStr, }: SourceMeta & {
+        blockRootHex: RootHex;
+        blobSidecar: deneb.BlobSidecar;
+    }, opts?: GetByBlobOptions): BlockInputBlobs;
+    getByColumn({ blockRootHex, columnSidecar, seenTimestampSec, source, peerIdStr, }: SourceMeta & {
+        blockRootHex: RootHex;
+        columnSidecar: fulu.DataColumnSidecar;
+    }, opts?: GetByBlobOptions): BlockInputColumns;
+    private buildCommonProps;
     /**
-     * Intended to be used for gossip validation, specifically this check:
-     * [IGNORE] The sidecar is the first sidecar for the tuple (block_header.slot, block_header.proposer_index,
-     *          sidecar.index) with valid header signature, sidecar inclusion proof, and kzg proof
+     * Use custom implementation of pruneSetToMax to allow for sorting by slot
+     * and deleting via key/rootHex
      */
-    hasDataColumnSidecar(sidecar: fulu.DataColumnSidecar): boolean;
-    getGossipBlockInput(config: ChainForkConfig, gossipedInput: GossipedBlockInput, metrics: Metrics | null): GossipBlockInputResponse;
+    private pruneToMaxSize;
 }
-export declare function getEmptyBlockInputCacheEntry(fork: ForkName, globalCacheId: number): BlockInputCacheType;
-export {};
 //# sourceMappingURL=seenGossipBlockInput.d.ts.map