@lodestar/beacon-node 1.35.0-dev.f80d2d52da → 1.35.0-dev.fd1dac853d

package/src/constants/constants.ts

@@ -0,0 +1,15 @@
+export const DEPOSIT_CONTRACT_TREE_DEPTH = 2 ** 5; // 32
+export const GENESIS_SLOT = 0;
+export const GENESIS_EPOCH = 0;
+export const GENESIS_START_SHARD = 0;
+export const FAR_FUTURE_EPOCH = Infinity;
+export const ZERO_HASH = Buffer.alloc(32, 0);
+export const ZERO_HASH_HEX = "0x" + "00".repeat(32);
+export const EMPTY_SIGNATURE = Buffer.alloc(96, 0);
+export const GRAFFITI_SIZE = 32;
+
+/**
+ * The maximum milliseconds of clock disparity assumed between honest nodes.
+ */
+export const MAXIMUM_GOSSIP_CLOCK_DISPARITY = 500;
+export const MAXIMUM_GOSSIP_CLOCK_DISPARITY_SEC = MAXIMUM_GOSSIP_CLOCK_DISPARITY / 1000;

package/src/constants/index.ts

@@ -0,0 +1,2 @@
+export * from "./constants.js";
+export * from "./network.js";

package/src/constants/network.ts

@@ -0,0 +1,52 @@
+/**
+ * For more info on some of these constants:
+ * https://github.com/ethereum/consensus-specs/blob/v1.1.10/specs/phase0/p2p-interface.md#configuration
+ */
+
+// Gossip constants
+
+/**
+ * The maximum number of slots during which an attestation can be propagated.
+ */
+export const ATTESTATION_PROPAGATION_SLOT_RANGE = 32;
+
+/** The maximum allowed size of uncompressed gossip messages. */
+export const GOSSIP_MAX_SIZE = 2 ** 20;
+export const GOSSIP_MAX_SIZE_BELLATRIX = 10 * GOSSIP_MAX_SIZE;
+/** The maximum allowed size of uncompressed req/resp chunked responses. */
+export const MAX_CHUNK_SIZE = 2 ** 20;
+export const MAX_CHUNK_SIZE_BELLATRIX = 10 * MAX_CHUNK_SIZE;
+
+export enum GoodByeReasonCode {
+  INBOUND_DISCONNECT = -1,
+  CLIENT_SHUTDOWN = 1,
+  IRRELEVANT_NETWORK = 2,
+  ERROR = 3,
+  TOO_MANY_PEERS = 129,
+  SCORE_TOO_LOW = 250,
+  BANNED = 251,
+}
+
+export const GOODBYE_KNOWN_CODES: Record<string, string> = {
+  "-1": "InboundDisconnect",
+  0: "Unknown",
+
+  // spec-defined codes
+  1: "Client shutdown",
+  2: "Irrelevant network",
+  3: "Internal fault/error",
+
+  // Teku-defined codes
+  128: "Unable to verify network",
+
+  // Lighthouse-defined codes
+  129: "Client has too many peers",
+  250: "Peer score too low",
+  251: "Peer banned this node",
+};
+
+/** Until js-libp2p exports an enum for its events */
+export enum Libp2pEvent {
+  connectionOpen = "connection:open",
+  connectionClose = "connection:close",
+}

package/src/db/beacon.ts

@@ -0,0 +1,113 @@
+import {ChainForkConfig} from "@lodestar/config";
+import {Db, LevelDbControllerMetrics} from "@lodestar/db";
+import {IBeaconDb} from "./interface.js";
+import {CheckpointStateRepository} from "./repositories/checkpointState.js";
+import {
+  AttesterSlashingRepository,
+  BLSToExecutionChangeRepository,
+  BackfilledRanges,
+  BestLightClientUpdateRepository,
+  BlobSidecarsArchiveRepository,
+  BlobSidecarsRepository,
+  BlockArchiveRepository,
+  BlockRepository,
+  CheckpointHeaderRepository,
+  DataColumnSidecarArchiveRepository,
+  DataColumnSidecarRepository,
+  DepositDataRootRepository,
+  DepositEventRepository,
+  Eth1DataRepository,
+  ProposerSlashingRepository,
+  StateArchiveRepository,
+  SyncCommitteeRepository,
+  SyncCommitteeWitnessRepository,
+  VoluntaryExitRepository,
+} from "./repositories/index.js";
+import {PreGenesisState, PreGenesisStateLastProcessedBlock} from "./single/index.js";
+
+export type BeaconDbModules = {
+  config: ChainForkConfig;
+  db: Db;
+};
+
+export class BeaconDb implements IBeaconDb {
+  block: BlockRepository;
+  blockArchive: BlockArchiveRepository;
+
+  blobSidecars: BlobSidecarsRepository;
+  blobSidecarsArchive: BlobSidecarsArchiveRepository;
+  dataColumnSidecar: DataColumnSidecarRepository;
+  dataColumnSidecarArchive: DataColumnSidecarArchiveRepository;
+
+  stateArchive: StateArchiveRepository;
+  checkpointState: CheckpointStateRepository;
+
+  voluntaryExit: VoluntaryExitRepository;
+  proposerSlashing: ProposerSlashingRepository;
+  attesterSlashing: AttesterSlashingRepository;
+  depositEvent: DepositEventRepository;
+  blsToExecutionChange: BLSToExecutionChangeRepository;
+
+  depositDataRoot: DepositDataRootRepository;
+  eth1Data: Eth1DataRepository;
+  preGenesisState: PreGenesisState;
+  preGenesisStateLastProcessedBlock: PreGenesisStateLastProcessedBlock;
+
+  // lightclient
+  bestLightClientUpdate: BestLightClientUpdateRepository;
+  checkpointHeader: CheckpointHeaderRepository;
+  syncCommittee: SyncCommitteeRepository;
+  syncCommitteeWitness: SyncCommitteeWitnessRepository;
+
+  backfilledRanges: BackfilledRanges;
+
+  constructor(
+    config: ChainForkConfig,
+    protected readonly db: Db
+  ) {
+    // Warning: If code is ever run in the constructor, must change this stub to not extend 'packages/beacon-node/test/utils/stub/beaconDb.ts' -
+    this.block = new BlockRepository(config, db);
+    this.blockArchive = new BlockArchiveRepository(config, db);
+
+    this.blobSidecars = new BlobSidecarsRepository(config, db);
+    this.blobSidecarsArchive = new BlobSidecarsArchiveRepository(config, db);
+    this.dataColumnSidecar = new DataColumnSidecarRepository(config, db);
+    this.dataColumnSidecarArchive = new DataColumnSidecarArchiveRepository(config, db);
+
+    this.stateArchive = new StateArchiveRepository(config, db);
+    this.checkpointState = new CheckpointStateRepository(config, db);
+    this.voluntaryExit = new VoluntaryExitRepository(config, db);
+    this.blsToExecutionChange = new BLSToExecutionChangeRepository(config, db);
+    this.proposerSlashing = new ProposerSlashingRepository(config, db);
+    this.attesterSlashing = new AttesterSlashingRepository(config, db);
+    this.depositEvent = new DepositEventRepository(config, db);
+    this.depositDataRoot = new DepositDataRootRepository(config, db);
+    this.eth1Data = new Eth1DataRepository(config, db);
+    this.preGenesisState = new PreGenesisState(config, db);
+    this.preGenesisStateLastProcessedBlock = new PreGenesisStateLastProcessedBlock(config, db);
+
+    // lightclient
+    this.bestLightClientUpdate = new BestLightClientUpdateRepository(config, db);
+    this.checkpointHeader = new CheckpointHeaderRepository(config, db);
+    this.syncCommittee = new SyncCommitteeRepository(config, db);
+    this.syncCommitteeWitness = new SyncCommitteeWitnessRepository(config, db);
+
+    this.backfilledRanges = new BackfilledRanges(config, db);
+  }
+
+  close(): Promise<void> {
+    return this.db.close();
+  }
+
+  setMetrics(metrics: LevelDbControllerMetrics): void {
+    this.db.setMetrics(metrics);
+  }
+
+  async pruneHotDb(): Promise<void> {
+    // Prune all hot blobs
+    await this.blobSidecars.batchDelete(await this.blobSidecars.keys());
+    // Prune all hot blocks
+    // TODO: Enable once it's deemed safe
+    // await this.block.batchDelete(await this.block.keys());
+  }
+}

package/src/db/buckets.ts

@@ -0,0 +1,80 @@
+// Buckets are separate database namespaces
+export enum Bucket {
+  // beacon chain
+  // finalized states
+  allForks_stateArchive = 0, // Root -> phase0.BeaconState
+  // unfinalized blocks
+  allForks_block = 1, // Root -> phase0.SignedBeaconBlock
+  // finalized blocks
+  allForks_blockArchive = 2, // Slot -> phase0.SignedBeaconBlock
+  // finalized block additional indices
+  index_blockArchiveParentRootIndex = 3, // parent Root -> Slot
+  index_blockArchiveRootIndex = 4, // Root -> Slot
+  // known bad block
+  // index_invalidBlock = 5, // DEPRECATED on v0.25.0
+  // finalized chain
+  index_mainChain = 6, // Slot -> Root<BeaconBlock>
+  // justified, finalized state and block hashes
+  index_chainInfo = 7, // Key -> Number64 | stateHash | blockHash
+  // eth1 processing
+  phase0_eth1Data = 8, // timestamp -> Eth1Data
+  index_depositDataRoot = 9, // depositIndex -> Root<DepositData>
+
+  // op pool
+  // phase0_attestation = 10, // DEPRECATED on v0.25.0
+  // phase0_aggregateAndProof = 11, // Root -> AggregateAndProof, DEPRECATED on v.27.0
+  phase0_depositData = 12, // [DEPRECATED] index -> DepositData
+  phase0_exit = 13, // ValidatorIndex -> VoluntaryExit
+  phase0_proposerSlashing = 14, // ValidatorIndex -> ProposerSlashing
+  allForks_attesterSlashing = 15, // Root -> AttesterSlashing
+  capella_blsToExecutionChange = 16, // ValidatorIndex -> SignedBLSToExecutionChange
+  // checkpoint states
+  allForks_checkpointState = 17, // Root -> BeaconState
+
+  // allForks_pendingBlock = 25, // Root -> SignedBeaconBlock // DEPRECATED on v0.30.0
+  phase0_depositEvent = 19, // depositIndex -> DepositEvent
+
+  index_stateArchiveRootIndex = 26, // State Root -> slot
+
+  deneb_blobSidecars = 27, // DENEB BeaconBlockRoot -> BlobSidecars
+  deneb_blobSidecarsArchive = 28, // DENEB BeaconBlockSlot -> BlobSidecars
+
+  phase0_preGenesisState = 30, // Single = phase0.BeaconState
+  phase0_preGenesisStateLastProcessedBlock = 31, // Single = Uint8
+
+  // Lightclient server
+  // altair_bestUpdatePerCommitteePeriod = 30, // DEPRECATED on v0.32.0
+  // altair_latestFinalizedUpdate = 31, // DEPRECATED on v0.32.0
+  // altair_latestNonFinalizedUpdate = 32, // DEPRECATED on v0.32.0
+  // altair_lightclientFinalizedCheckpoint = 33, // DEPRECATED on v0.32.0
+  // altair_lightClientInitProof = 34, // DEPRECATED on v0.32.0
+  // altair_lightClientSyncCommitteeProof = 35, // DEPRECATED on v0.32.0
+  // index_lightClientInitProof = 36, // DEPRECATED on v0.32.0
+
+  backfilled_ranges = 42, // Backfilled From to To, inclusive of both From, To
+
+  // Buckets to support LightClient server v2
+  lightClient_syncCommitteeWitness = 51, // BlockRoot -> SyncCommitteeWitness
+  lightClient_syncCommittee = 52, // Root(altair.SyncCommittee) -> altair.SyncCommittee
+  // TODO: May be redundant to block stores
+  lightClient_checkpointHeader = 53, // BlockRoot -> phase0.BeaconBlockHeader
+  // 54 was for bestPartialLightClientUpdate, allocate a fresh one
+  // lightClient_bestLightClientUpdate = 55, // SyncPeriod -> LightClientUpdate // DEPRECATED on v1.5.0
+  lightClient_bestLightClientUpdate = 56, // SyncPeriod -> [Slot, LightClientUpdate]
+
+  fulu_dataColumnSidecars = 57, // FULU BeaconBlockRoot -> DataColumnSidecars
+  fulu_dataColumnSidecarsArchive = 58, // FULU BeaconBlockSlot -> DataColumnSidecars
+}
+
+export function getBucketNameByValue<T extends Bucket>(enumValue: T): keyof typeof Bucket {
+  const keys = Object.keys(Bucket).filter((x) => {
+    if (Number.isNaN(parseInt(x))) {
+      return Bucket[x as keyof typeof Bucket] === enumValue;
+    }
+    return false;
+  }) as (keyof typeof Bucket)[];
+  if (keys.length > 0) {
+    return keys[0];
+  }
+  throw new Error("Missing bucket for value " + enumValue);
+}

package/src/db/index.ts

@@ -0,0 +1,2 @@
+export {BeaconDb} from "./beacon.js";
+export type {IBeaconDb} from "./interface.js";

package/src/db/interface.ts

@@ -0,0 +1,76 @@
+import {LevelDbControllerMetrics} from "@lodestar/db";
+import {CheckpointStateRepository} from "./repositories/checkpointState.js";
+import {
+  AttesterSlashingRepository,
+  BLSToExecutionChangeRepository,
+  BackfilledRanges,
+  BestLightClientUpdateRepository,
+  BlobSidecarsArchiveRepository,
+  BlobSidecarsRepository,
+  BlockArchiveRepository,
+  BlockRepository,
+  CheckpointHeaderRepository,
+  DataColumnSidecarArchiveRepository,
+  DataColumnSidecarRepository,
+  DepositDataRootRepository,
+  DepositEventRepository,
+  Eth1DataRepository,
+  ProposerSlashingRepository,
+  StateArchiveRepository,
+  SyncCommitteeRepository,
+  SyncCommitteeWitnessRepository,
+  VoluntaryExitRepository,
+} from "./repositories/index.js";
+import {PreGenesisState, PreGenesisStateLastProcessedBlock} from "./single/index.js";
+
+/**
+ * The DB service manages the data layer of the beacon chain
+ * The exposed methods do not refer to the underlying data engine,
+ * but instead expose relevant beacon chain objects
+ */
+export interface IBeaconDb {
+  // unfinalized blocks
+  block: BlockRepository;
+  // finalized blocks
+  blockArchive: BlockArchiveRepository;
+
+  blobSidecars: BlobSidecarsRepository;
+  blobSidecarsArchive: BlobSidecarsArchiveRepository;
+  dataColumnSidecar: DataColumnSidecarRepository;
+  dataColumnSidecarArchive: DataColumnSidecarArchiveRepository;
+
+  // finalized states
+  stateArchive: StateArchiveRepository;
+  // checkpoint states
+  checkpointState: CheckpointStateRepository;
+
+  // op pool
+  voluntaryExit: VoluntaryExitRepository;
+  proposerSlashing: ProposerSlashingRepository;
+  attesterSlashing: AttesterSlashingRepository;
+  depositEvent: DepositEventRepository;
+  blsToExecutionChange: BLSToExecutionChangeRepository;
+
+  // eth1 processing
+  preGenesisState: PreGenesisState;
+  preGenesisStateLastProcessedBlock: PreGenesisStateLastProcessedBlock;
+
+  // all deposit data roots and merkle tree
+  depositDataRoot: DepositDataRootRepository;
+  eth1Data: Eth1DataRepository;
+
+  // lightclient
+  bestLightClientUpdate: BestLightClientUpdateRepository;
+  checkpointHeader: CheckpointHeaderRepository;
+  syncCommittee: SyncCommitteeRepository;
+  syncCommitteeWitness: SyncCommitteeWitnessRepository;
+
+  backfilledRanges: BackfilledRanges;
+
+  pruneHotDb(): Promise<void>;
+
+  /**  Close the connection to the db instance and close the db store. */
+  close(): Promise<void>;
+  /** To inject metrics after CLI initialization */
+  setMetrics(metrics: LevelDbControllerMetrics): void;
+}

package/src/db/options.ts

@@ -0,0 +1,7 @@
+export type DatabaseOptions = {
+  name: string;
+};
+
+export const defaultDbOptions: DatabaseOptions = {
+  name: "./.tmp/lodestar-db",
+};

package/src/db/repositories/attesterSlashing.ts

@@ -0,0 +1,38 @@
+import {ChainForkConfig} from "@lodestar/config";
+import {Db, Repository} from "@lodestar/db";
+import {AttesterSlashing, ValidatorIndex, ssz} from "@lodestar/types";
+import {Bucket, getBucketNameByValue} from "../buckets.js";
+
+/**
+ * AttesterSlashing indexed by root
+ *
+ * Added via gossip or api
+ * Removed when included on chain or old
+ */
+export class AttesterSlashingRepository extends Repository<Uint8Array, AttesterSlashing> {
+  constructor(config: ChainForkConfig, db: Db) {
+    const bucket = Bucket.allForks_attesterSlashing;
+    /**
+     * We are using `ssz.electra.AttesterSlashing` type since it is backward compatible with `ssz.phase0.AttesterSlashing`
+     * But this also means the length of `attestingIndices` is not checked/enforced here. Need to make sure length
+     * is correct before writing to db.
+     */
+    const type = ssz.electra.AttesterSlashing;
+    super(config, db, bucket, type, getBucketNameByValue(bucket));
+  }
+
+  async hasAll(attesterIndices: ValidatorIndex[] = []): Promise<boolean> {
+    const attesterSlashings = (await this.values()) ?? [];
+    const indices = new Set<ValidatorIndex>();
+    for (const slashing of attesterSlashings) {
+      for (const index of slashing.attestation1.attestingIndices) indices.add(index);
+      for (const index of slashing.attestation2.attestingIndices) indices.add(index);
+    }
+    for (const attesterIndice of attesterIndices) {
+      if (!indices.has(attesterIndice)) {
+        return false;
+      }
+    }
+    return true;
+  }
+}

package/src/db/repositories/backfilledRanges.ts

@@ -0,0 +1,29 @@
+import {ChainForkConfig} from "@lodestar/config";
+import {DatabaseController, Repository} from "@lodestar/db";
+import {Slot, ssz} from "@lodestar/types";
+import {bytesToInt} from "@lodestar/utils";
+import {Bucket, getBucketNameByValue} from "../buckets.js";
+
+/**
+ * Slot to slot ranges that ensure that block range is fully backfilled
+ *
+ * If node starts backfilling at slots 1000, and backfills to 800, there will be an entry
+ * 1000 -> 800
+ *
+ * When the node is backfilling if it starts at 1200 and backfills to 1000, it will find this sequence and,
+ * jump directly to 800 and delete the key 1000.
+ */
+export class BackfilledRanges extends Repository<Slot, Slot> {
+  constructor(config: ChainForkConfig, db: DatabaseController<Uint8Array, Uint8Array>) {
+    const bucket = Bucket.backfilled_ranges;
+    super(config, db, bucket, ssz.Slot, getBucketNameByValue(bucket));
+  }
+
+  decodeKey(data: Buffer): number {
+    return bytesToInt(super.decodeKey(data) as unknown as Uint8Array, "be");
+  }
+
+  getId(_value: Slot): number {
+    throw new Error("Cannot get the db key from slot");
+  }
+}

package/src/db/repositories/blobSidecars.ts

@@ -0,0 +1,37 @@
+import {ContainerType, ValueOf} from "@chainsafe/ssz";
+import {ChainForkConfig} from "@lodestar/config";
+import {Db, Repository} from "@lodestar/db";
+import {ssz} from "@lodestar/types";
+import {Bucket, getBucketNameByValue} from "../buckets.js";
+
+export const blobSidecarsWrapperSsz = new ContainerType(
+  {
+    blockRoot: ssz.Root,
+    slot: ssz.Slot,
+    blobSidecars: ssz.deneb.BlobSidecars,
+  },
+  {typeName: "BlobSidecarsWrapper", jsonCase: "eth2"}
+);
+
+export type BlobSidecarsWrapper = ValueOf<typeof blobSidecarsWrapperSsz>;
+export const BLOB_SIDECARS_IN_WRAPPER_INDEX = 44;
+
+/**
+ * blobSidecarsWrapper by block root (= hash_tree_root(SignedBeaconBlock.message))
+ *
+ * Used to store unfinalized BlobSidecars
+ */
+export class BlobSidecarsRepository extends Repository<Uint8Array, BlobSidecarsWrapper> {
+  constructor(config: ChainForkConfig, db: Db) {
+    const bucket = Bucket.deneb_blobSidecars;
+    super(config, db, bucket, blobSidecarsWrapperSsz, getBucketNameByValue(bucket));
+  }
+
+  /**
+   * Id is hashTreeRoot of unsigned BeaconBlock
+   */
+  getId(value: BlobSidecarsWrapper): Uint8Array {
+    const {blockRoot} = value;
+    return blockRoot;
+  }
+}

package/src/db/repositories/blobSidecarsArchive.ts

@@ -0,0 +1,28 @@
+import {ChainForkConfig} from "@lodestar/config";
+import {Db, Repository} from "@lodestar/db";
+import {Slot} from "@lodestar/types";
+import {bytesToInt} from "@lodestar/utils";
+import {Bucket, getBucketNameByValue} from "../buckets.js";
+import {BlobSidecarsWrapper, blobSidecarsWrapperSsz} from "./blobSidecars.js";
+
+/**
+ * blobSidecarsWrapper by slot
+ *
+ * Used to store finalized BlobSidecars
+ */
+export class BlobSidecarsArchiveRepository extends Repository<Slot, BlobSidecarsWrapper> {
+  constructor(config: ChainForkConfig, db: Db) {
+    const bucket = Bucket.deneb_blobSidecarsArchive;
+    super(config, db, bucket, blobSidecarsWrapperSsz, getBucketNameByValue(bucket));
+  }
+
+  // Handle key as slot
+
+  getId(value: BlobSidecarsWrapper): Slot {
+    return value.slot;
+  }
+
+  decodeKey(data: Uint8Array): number {
+    return bytesToInt(super.decodeKey(data) as unknown as Uint8Array, "be");
+  }
+}

package/src/db/repositories/block.ts

@@ -0,0 +1,33 @@
+import {ChainForkConfig} from "@lodestar/config";
+import {Db, Repository} from "@lodestar/db";
+import {SignedBeaconBlock, ssz} from "@lodestar/types";
+import {getSignedBlockTypeFromBytes} from "../../util/multifork.js";
+import {Bucket, getBucketNameByValue} from "../buckets.js";
+
+/**
+ * Blocks by root
+ *
+ * Used to store unfinalized blocks
+ */
+export class BlockRepository extends Repository<Uint8Array, SignedBeaconBlock> {
+  constructor(config: ChainForkConfig, db: Db) {
+    const bucket = Bucket.allForks_block;
+    const type = ssz.phase0.SignedBeaconBlock; // Pick some type but won't be used
+    super(config, db, bucket, type, getBucketNameByValue(bucket));
+  }
+
+  /**
+   * Id is hashTreeRoot of unsigned BeaconBlock
+   */
+  getId(value: SignedBeaconBlock): Uint8Array {
+    return this.config.getForkTypes(value.message.slot).BeaconBlock.hashTreeRoot(value.message);
+  }
+
+  encodeValue(value: SignedBeaconBlock): Buffer {
+    return this.config.getForkTypes(value.message.slot).SignedBeaconBlock.serialize(value) as Buffer;
+  }
+
+  decodeValue(data: Buffer): SignedBeaconBlock {
+    return getSignedBlockTypeFromBytes(this.config, data).deserialize(data);
+  }
+}