@lodestar/beacon-node 1.35.0-dev.fcf8d024ea → 1.35.0-dev.feed916580

package/src/util/execution.ts

@@ -1,223 +0,0 @@
-import {routes} from "@lodestar/api";
-import {ChainForkConfig} from "@lodestar/config";
-import {ForkPostFulu, ForkPreFulu} from "@lodestar/params";
-import {signedBlockToSignedHeader} from "@lodestar/state-transition";
-import {deneb, fulu} from "@lodestar/types";
-import {toHex} from "@lodestar/utils";
-import {isBlockInputBlobs, isBlockInputColumns} from "../chain/blocks/blockInput/blockInput.js";
-import {BlockInputSource, IBlockInput} from "../chain/blocks/blockInput/types.js";
-import {ChainEvent, ChainEventEmitter} from "../chain/emitter.js";
-import {IExecutionEngine} from "../execution/index.js";
-import {Metrics} from "../metrics/index.js";
-import {computePreFuluKzgCommitmentsInclusionProof} from "./blobs.js";
-import {
-  getCellsAndProofs,
-  getDataColumnSidecarsFromBlock,
-  getDataColumnSidecarsFromColumnSidecar,
-} from "./dataColumns.js";
-
-export enum DataColumnEngineResult {
-  PreFulu = "pre_fulu",
-  // the recover is not attempted because it has full data columns
-  NotAttemptedFull = "not_attempted_full",
-  // block has no blob so no need to call EL
-  NotAttemptedNoBlobs = "not_attempted_no_blobs",
-  // EL call returned null, meaning it could not find the blobs
-  NullResponse = "null_response",
-  // the recover is a success and it helps resolve availability
-  SuccessResolved = "success_resolved",
-  // the recover is a success but it's late, availability is already resolved by either gossip or getBlobsV2
-  SuccessLate = "success_late",
-  Failed = "failed",
-}
-
-export async function getBlobSidecarsFromExecution(
-  config: ChainForkConfig,
-  executionEngine: IExecutionEngine,
-  metrics: Metrics | null,
-  emitter: ChainEventEmitter,
-  blockInput: IBlockInput
-) {
-  if (!isBlockInputBlobs(blockInput)) {
-    return;
-  }
-
-  if (blockInput.hasAllData()) {
-    return;
-  }
-
-  const forkName = blockInput.forkName as ForkPreFulu;
-  const blobMeta = blockInput.getMissingBlobMeta();
-
-  metrics?.blobs.getBlobsV1Requests.inc();
-  metrics?.blobs.getBlobsV1RequestedBlobCount.inc(blobMeta.length);
-  const enginedResponse = await executionEngine
-    .getBlobs(
-      forkName,
-      blobMeta.map(({versionedHash}) => versionedHash)
-    )
-    .catch((_e) => {
-      // TODO(fulu): this should only count as a single error? need to update the promql to reflect this
-      metrics?.blobs.getBlobsV1Error.inc(blobMeta.length);
-      return null;
-    });
-
-  if (enginedResponse === null) {
-    return;
-  }
-
-  const block = blockInput.getBlock();
-
-  const blobSidecars: deneb.BlobSidecars = [];
-  // response.length should always match blobMeta.length and they should be in the same order
-  for (let i = 0; i < blobMeta.length; i++) {
-    const blobAndProof = enginedResponse[i];
-
-    if (!blobAndProof) {
-      metrics?.blobs.getBlobsV1Miss.inc();
-    } else {
-      metrics?.blobs.getBlobsV1Hit.inc();
-
-      if (blockInput.hasBlob(blobMeta[i].index)) {
-        // blob arrived and was cached while waiting for API response
-        metrics?.blobs.getBlobsV1HitButArrivedWhileWaiting.inc();
-        continue;
-      }
-
-      metrics?.blobs.getBlobsV1HitUseful.inc();
-      const {blob, proof} = blobAndProof;
-      const index = blobMeta[i].index;
-      const kzgCommitment = block.message.body.blobKzgCommitments[index];
-      const blobSidecar: deneb.BlobSidecar = {
-        index,
-        blob,
-        kzgProof: proof,
-        kzgCommitment,
-        // TODO(fulu): refactor this to only calculate the root inside these following two functions once
-        kzgCommitmentInclusionProof: computePreFuluKzgCommitmentsInclusionProof(forkName, block.message.body, index),
-        signedBlockHeader: signedBlockToSignedHeader(config, block),
-      };
-
-      blockInput.addBlob({
-        blobSidecar,
-        blockRootHex: blockInput.blockRootHex,
-        seenTimestampSec: Date.now() / 1000,
-        source: BlockInputSource.engine,
-      });
-
-      if (emitter.listenerCount(routes.events.EventType.blobSidecar)) {
-        emitter.emit(routes.events.EventType.blobSidecar, {
-          blockRoot: blockInput.blockRootHex,
-          slot: blockInput.slot,
-          index,
-          kzgCommitment: toHex(kzgCommitment),
-          versionedHash: toHex(blobMeta[i].versionedHash),
-        });
-      }
-
-      blobSidecars.push(blobSidecar);
-    }
-  }
-
-  emitter.emit(ChainEvent.publishBlobSidecars, blobSidecars);
-  metrics?.gossipBlob.publishedFromEngine.inc(blobSidecars.length);
-}
-
-/**
- * Post fulu, call getBlobsV2 from execution engine once per slot whenever we see either beacon_block or data_column_sidecar gossip message
- */
-export async function getDataColumnSidecarsFromExecution(
-  config: ChainForkConfig,
-  executionEngine: IExecutionEngine,
-  emitter: ChainEventEmitter,
-  blockInput: IBlockInput,
-  metrics: Metrics | null,
-  blobAndProofBuffers?: Uint8Array[]
-): Promise<DataColumnEngineResult> {
-  // If its not a column block input, exit
-  if (!isBlockInputColumns(blockInput)) {
-    return DataColumnEngineResult.PreFulu;
-  }
-
-  // If already have all columns, exit
-  if (blockInput.hasAllData()) {
-    return DataColumnEngineResult.NotAttemptedFull;
-  }
-
-  const versionedHashes = blockInput.getVersionedHashes();
-
-  // If there are no blobs in this block, exit
-  if (versionedHashes.length === 0) {
-    return DataColumnEngineResult.NotAttemptedNoBlobs;
-  }
-
-  // Get blobs from execution engine
-  metrics?.peerDas.getBlobsV2Requests.inc();
-  const timer = metrics?.peerDas.getBlobsV2RequestDuration.startTimer();
-  const blobs = await executionEngine.getBlobs(
-    blockInput.forkName as ForkPostFulu,
-    versionedHashes,
-    blobAndProofBuffers
-  );
-  timer?.();
-
-  // Execution engine was unable to find one or more blobs
-  if (blobs === null) {
-    return DataColumnEngineResult.NullResponse;
-  }
-  metrics?.peerDas.getBlobsV2Responses.inc();
-
-  // Return if we received all data columns while waiting for getBlobs
-  if (blockInput.hasAllData()) {
-    return DataColumnEngineResult.SuccessLate;
-  }
-
-  let dataColumnSidecars: fulu.DataColumnSidecars;
-  const cellsAndProofs = await getCellsAndProofs(blobs);
-  if (blockInput.hasBlock()) {
-    dataColumnSidecars = getDataColumnSidecarsFromBlock(
-      config,
-      blockInput.getBlock() as fulu.SignedBeaconBlock,
-      cellsAndProofs
-    );
-  } else {
-    const firstSidecar = blockInput.getAllColumns()[0];
-    dataColumnSidecars = getDataColumnSidecarsFromColumnSidecar(firstSidecar, cellsAndProofs);
-  }
-
-  // Publish columns if and only if subscribed to them
-  const previouslyMissingColumns = blockInput.getMissingSampledColumnMeta().missing;
-  const sampledColumns = previouslyMissingColumns.map((columnIndex) => dataColumnSidecars[columnIndex]);
-
-  // for columns that we already seen, it will be ignored through `ignoreDuplicatePublishError` gossip option
-  emitter.emit(ChainEvent.publishDataColumns, sampledColumns);
-
-  // add all sampled columns to the block input, even if we didn't sample them
-  const seenTimestampSec = Date.now() / 1000;
-  for (const columnSidecar of sampledColumns) {
-    if (blockInput.hasColumn(columnSidecar.index)) {
-      // columns may have been added while waiting
-      // TODO(fulu): add metrics for this condition
-      continue;
-    }
-
-    blockInput.addColumn({
-      columnSidecar,
-      blockRootHex: blockInput.blockRootHex,
-      source: BlockInputSource.engine,
-      seenTimestampSec,
-    });
-
-    if (emitter.listenerCount(routes.events.EventType.dataColumnSidecar)) {
-      emitter.emit(routes.events.EventType.dataColumnSidecar, {
-        blockRoot: blockInput.blockRootHex,
-        slot: blockInput.slot,
-        index: columnSidecar.index,
-        kzgCommitments: columnSidecar.kzgCommitments.map(toHex),
-      });
-    }
-  }
-
-  metrics?.dataColumns.bySource.inc({source: BlockInputSource.engine}, previouslyMissingColumns.length);
-  return DataColumnEngineResult.SuccessResolved;
-}

package/src/util/file.ts

@@ -1,52 +0,0 @@
-import fs from "node:fs";
-import {promisify} from "node:util";
-
-/** Ensure a directory exists */
-export async function ensureDir(path: string): Promise<void> {
-  try {
-    await promisify(fs.stat)(path);
-  } catch (_) {
-    // not exists
-    await promisify(fs.mkdir)(path, {recursive: true});
-  }
-}
-
-/** Write data to a file if it does not exist */
-export async function writeIfNotExist(filepath: string, bytes: Uint8Array): Promise<boolean> {
-  try {
-    await promisify(fs.stat)(filepath);
-    return false;
-    // file exists, do nothing
-  } catch (_) {
-    // not exists
-    await promisify(fs.writeFile)(filepath, bytes);
-    return true;
-  }
-}
-
-/** Remove a file if it exists */
-export async function removeFile(path: string): Promise<boolean> {
-  try {
-    await promisify(fs.unlink)(path);
-    return true;
-  } catch (_) {
-    // may not exists
-    return false;
-  }
-}
-
-export async function readFile(path: string): Promise<Uint8Array | null> {
-  try {
-    return await fs.promises.readFile(path);
-  } catch (_) {
-    return null;
-  }
-}
-
-export async function readFileNames(folderPath: string): Promise<string[]> {
-  try {
-    return await fs.promises.readdir(folderPath);
-  } catch (_) {
-    return [];
-  }
-}

package/src/util/forkChoice.ts

@@ -1,5 +0,0 @@
-import {ExecutionStatus, ProtoBlock} from "@lodestar/fork-choice";
-
-export function isOptimisticBlock(block: ProtoBlock): boolean {
-  return block.executionStatus === ExecutionStatus.Syncing;
-}

package/src/util/forkName.ts

@@ -1,20 +0,0 @@
-import {BeaconConfig} from "@lodestar/config";
-import {ForkName} from "@lodestar/params";
-import {Slot} from "@lodestar/types";
-
-/**
- * Group an array of items by ForkName according to the slot associted to each item
- */
-export function groupByFork<T>(config: BeaconConfig, items: T[], getSlot: (item: T) => Slot): Map<ForkName, T[]> {
-  const itemsByFork = new Map<ForkName, T[]>();
-  for (const item of items) {
-    const forkName = config.getForkName(getSlot(item));
-    let itemsInFork = itemsByFork.get(forkName);
-    if (!itemsInFork) {
-      itemsInFork = [];
-      itemsByFork.set(forkName, itemsInFork);
-    }
-    itemsInFork.push(item);
-  }
-  return itemsByFork;
-}

package/src/util/graffiti.ts

@@ -1,39 +0,0 @@
-import {Bytes32} from "@lodestar/types";
-import {GRAFFITI_SIZE} from "../constants/index.js";
-import {ClientVersion} from "../execution/index.js";
-
-/**
- * Parses a graffiti UTF8 string and returns a 32 bytes buffer right padded with zeros
- */
-export function toGraffitiBytes(graffiti: string): Bytes32 {
-  return Buffer.concat([Buffer.from(graffiti, "utf8"), Buffer.alloc(GRAFFITI_SIZE, 0)], GRAFFITI_SIZE);
-}
-
-/**
- * Converts a graffiti from 32 bytes buffer back to a UTF-8 string
- */
-export function fromGraffitiBytes(graffiti: Bytes32): string {
-  return Buffer.from(graffiti.buffer, graffiti.byteOffset, graffiti.byteLength)
-    .toString("utf8")
-    .replaceAll("\u0000", "");
-}
-
-export function getDefaultGraffiti(
-  consensusClientVersion: ClientVersion,
-  executionClientVersion: ClientVersion | null | undefined,
-  opts: {private?: boolean}
-): string {
-  if (opts.private) {
-    return "";
-  }
-
-  if (executionClientVersion != null) {
-    const {code: executionCode, commit: executionCommit} = executionClientVersion;
-
-    // Follow the 2-byte commit format in https://github.com/ethereum/execution-apis/pull/517#issuecomment-1918512560
-    return `${executionCode}${executionCommit.slice(0, 4)}${consensusClientVersion.code}${consensusClientVersion.commit.slice(0, 4)}`;
-  }
-
-  // No EL client info available. We still want to include CL info albeit not spec compliant
-  return `${consensusClientVersion.code}${consensusClientVersion.commit.slice(0, 4)}`;
-}

package/src/util/hex.ts

@@ -1,9 +0,0 @@
-import {RootHex} from "@lodestar/types";
-
-export function bufferToHex(buffer: Buffer): RootHex {
-  return "0x" + buffer.toString("hex");
-}
-
-export function hexToBuffer(v: string): Buffer {
-  return Buffer.from(v.replace("0x", ""));
-}

package/src/util/index.ts

@@ -1,2 +0,0 @@
-//  Only export the functions which are useful for external packages and difficult to reproduce or mock
-export * from "./kzg.js";

package/src/util/ip.ts

@@ -1,6 +0,0 @@
-/**
- * Returns true if string represents a localhost IP (v4 or v6)
- */
-export function isLocalhostIP(ip: string): boolean {
-  return ip === "127.0.0.1" || ip === "::1" || ip === "0:0:0:0:0:0:0:1";
-}

package/src/util/itTrigger.ts

@@ -1,49 +0,0 @@
-/**
- * Same as [`it-pushable`](https://github.com/alanshaw/it-pushable/blob/76a67cbe92d1db940311ee07775dbc662697e09c/index.d.ts)
- * but it does not buffer values, and this.end() stops the AsyncGenerator immediately
- */
-export class ItTrigger {
-  private triggered = false;
-  private ended = false;
-  private error?: Error;
-  private onNext?: () => void;
-
-  trigger(): void {
-    this.triggered = true;
-    if (this.onNext) this.onNext();
-  }
-  end(err?: Error): void {
-    if (err) this.error = err;
-    else this.ended = true;
-    if (this.onNext) this.onNext();
-  }
-
-  // AsyncGenerator API
-
-  [Symbol.asyncIterator](): this {
-    return this;
-  }
-  async next(): ReturnType<AsyncGenerator["next"]> {
-    if (this.error) throw this.error;
-    if (this.ended) return {done: true, value: undefined};
-    if (this.triggered) {
-      this.triggered = false;
-      return {done: false, value: undefined};
-    }
-
-    return new Promise((resolve) => {
-      this.onNext = () => {
-        this.onNext = undefined;
-        resolve(this.next());
-      };
-    });
-  }
-  async return(): ReturnType<AsyncGenerator["return"]> {
-    this.end();
-    return {done: true, value: undefined};
-  }
-  async throw(err: Error): ReturnType<AsyncGenerator["throw"]> {
-    this.end(err);
-    return {done: true, value: undefined};
-  }
-}

package/src/util/kzg.ts

@@ -1,3 +0,0 @@
-import {DasContextJs} from "@crate-crypto/node-eth-kzg";
-
-export const kzg = DasContextJs.create({usePrecomp: true});

package/src/util/map.ts

@@ -1,77 +0,0 @@
-import {OrderedSet} from "./set.js";
-
-/**
- * An implementation of Map that support getting first/last key and value.
- */
-export class OrderedMap<K, V> {
-  private _set: OrderedSet<K>;
-  private map: Map<K, V>;
-
-  constructor() {
-    this._set = new OrderedSet<K>();
-    this.map = new Map<K, V>();
-  }
-
-  get(key: K): V | undefined {
-    return this.map.get(key);
-  }
-
-  set(key: K, value: V): this {
-    this._set.add(key);
-    this.map.set(key, value);
-    return this;
-  }
-
-  delete(key: K, searchFromHead: boolean): boolean {
-    if (this.map.has(key)) {
-      this._set.delete(key, searchFromHead);
-      return this.map.delete(key);
-    }
-    return false;
-  }
-
-  keys(): IterableIterator<K> {
-    return this._set.values();
-  }
-
-  lastKey(): K | null {
-    return this._set.last();
-  }
-
-  firstKey(): K | null {
-    return this._set.first();
-  }
-
-  values(): IterableIterator<V> {
-    const _self = this;
-    return (function* generateValues() {
-      for (const key of _self.keys()) {
-        yield _self.get(key) as V;
-      }
-    })();
-  }
-
-  lastValue(): V | null {
-    const lastKey = this._set.last();
-    if (lastKey === null) {
-      return null;
-    }
-    return this.get(lastKey) as V;
-  }
-
-  firstValue(): V | null {
-    const firstKey = this._set.first();
-    if (firstKey === null) {
-      return null;
-    }
-    return this.get(firstKey) as V;
-  }
-
-  size(): number {
-    return this._set.size;
-  }
-
-  has(key: K): boolean {
-    return this.map.has(key);
-  }
-}

package/src/util/metadata.ts

@@ -1,22 +0,0 @@
-import {bytesToInt, intToBytes} from "@lodestar/utils";
-import {ClientCode, ClientVersion} from "../execution/index.js";
-
-export function getLodestarClientVersion(info?: {version?: string; commit?: string}): ClientVersion {
-  return {
-    code: ClientCode.LS,
-    name: "Lodestar",
-    version: info?.version ?? "",
-    commit: info?.commit?.slice(0, 8) ?? "",
-  };
-}
-
-/**
- * Serializes a custody group count value into a Uint8Array suitable for the the ENR field `cgc`.
- */
-export function serializeCgc(cgc: number): Uint8Array {
-  return intToBytes(cgc, Math.ceil(Math.log2(cgc + 1) / 8), "be");
-}
-
-export function deserializeCgc(cgcBytes: Uint8Array): number {
-  return bytesToInt(cgcBytes, "be");
-}

package/src/util/multifork.ts

@@ -1,69 +0,0 @@
-import {ChainForkConfig} from "@lodestar/config";
-import {ForkAll, ForkPostAltair} from "@lodestar/params";
-import {SSZTypesFor, Slot} from "@lodestar/types";
-import {bytesToInt} from "@lodestar/utils";
-import {getSlotFromSignedBeaconBlockSerialized} from "./sszBytes.js";
-
-/**
- * Slot	uint64
- */
-const SLOT_BYTE_COUNT = 8;
-
-/**
- * 8 + 32 = 40
- * ```
- * class BeaconState(Container):
- *   genesis_time: uint64 [fixed - 8 bytes]
- *   genesis_validators_root: Root [fixed - 32 bytes]
- *   slot: Slot [fixed - 8 bytes]
- *   ...
- * ```
- */
-const SLOT_BYTES_POSITION_IN_STATE = 40;
-
-export function getSignedBlockTypeFromBytes(
-  config: ChainForkConfig,
-  bytes: Buffer | Uint8Array
-): SSZTypesFor<ForkAll, "SignedBeaconBlock"> {
-  const slot = getSlotFromSignedBeaconBlockSerialized(bytes);
-  if (slot === null) {
-    throw Error("getSignedBlockTypeFromBytes: invalid bytes");
-  }
-
-  return config.getForkTypes(slot).SignedBeaconBlock;
-}
-
-export function getStateTypeFromBytes(
-  config: ChainForkConfig,
-  bytes: Buffer | Uint8Array
-): SSZTypesFor<ForkAll, "BeaconState"> {
-  const slot = getStateSlotFromBytes(bytes);
-  return config.getForkTypes(slot).BeaconState;
-}
-
-export function getStateSlotFromBytes(bytes: Uint8Array): Slot {
-  return bytesToInt(bytes.subarray(SLOT_BYTES_POSITION_IN_STATE, SLOT_BYTES_POSITION_IN_STATE + SLOT_BYTE_COUNT));
-}
-
-/**
- * First field in update is beacon, first field in beacon is slot
- *
- * header = {
- *  beacon: {
- *   slot
- *   ...
- *  }
- *  ...
- * }
- *   ...
- */
-const SLOT_BYTES_POSITION_IN_LIGHTCLIENTHEADER = 0;
-export function getLightClientHeaderTypeFromBytes(
-  config: ChainForkConfig,
-  bytes: Buffer | Uint8Array
-): SSZTypesFor<ForkPostAltair, "LightClientHeader"> {
-  const slot = bytesToInt(
-    bytes.subarray(SLOT_BYTES_POSITION_IN_LIGHTCLIENTHEADER, SLOT_BYTES_POSITION_IN_LIGHTCLIENTHEADER + SLOT_BYTE_COUNT)
-  );
-  return config.getPostAltairForkTypes(slot).LightClientHeader;
-}

package/src/util/numpy.ts

@@ -1,8 +0,0 @@
-/**
- * Returns num evenly spaced samples, calculated over the interval [start, stop] inclusive.
- */
-export function linspace(start: number, stop: number): number[] {
-  const arr: number[] = [];
-  for (let i = start; i <= stop; i++) arr.push(i);
-  return arr;
-}

package/src/util/peerId.ts

@@ -1,16 +0,0 @@
-import {PeerId} from "@libp2p/interface";
-import {peerIdFromString} from "@libp2p/peer-id";
-import {base58btc} from "multiformats/bases/base58";
-
-// Ensure consistent serialization of PeerId to string
-
-/**
- * PeerId serialized with `peerId.toString`
- */
-export type PeerIdStr = string;
-
-export {peerIdFromString};
-
-export function peerIdToString(peerId: PeerId): string {
-  return base58btc.encode(peerId.toMultihash().bytes).slice(1);
-}

package/src/util/profile.ts

@@ -1,54 +0,0 @@
-import {sleep} from "@lodestar/utils";
-
-/**
- * Take 10m profile of the current thread without promise tracking.
- */
-export async function profileNodeJS(durationMs: number): Promise<string> {
-  const inspector = await import("node:inspector");
-
-  // due to some typing issues, not able to use promisify here
-  return new Promise<string>((resolve, reject) => {
-    // Start the inspector and connect to it
-    const session = new inspector.Session();
-    session.connect();
-
-    session.post("Profiler.enable", () => {
-      session.post("Profiler.start", async () => {
-        await sleep(durationMs);
-        session.post("Profiler.stop", (err, {profile}) => {
-          if (!err) {
-            resolve(JSON.stringify(profile));
-          } else {
-            reject(err);
-          }
-
-          // Detach from the inspector and close the session
-          session.post("Profiler.disable");
-          session.disconnect();
-        });
-      });
-    });
-  });
-}
-
-/**
- * Write heap snapshot of the current thread to the specified file.
- */
-export async function writeHeapSnapshot(prefix: string, dirpath: string): Promise<string> {
-  // Lazily import NodeJS only modules
-  const fs = await import("node:fs");
-  const v8 = await import("node:v8");
-  const snapshotStream = v8.getHeapSnapshot();
-  const filepath = `${dirpath}/${prefix}_${new Date().toISOString()}.heapsnapshot`;
-  const fileStream = fs.createWriteStream(filepath);
-  return new Promise<string>((resolve, reject) => {
-    fileStream.on("error", (err) => {
-      reject(err);
-    });
-
-    snapshotStream.pipe(fileStream);
-    snapshotStream.on("end", () => {
-      resolve(filepath);
-    });
-  });
-}

package/src/util/promises.ts

@@ -1,14 +0,0 @@
-/**
- * Promise.all() but allows all functions to run even if one throws syncronously
- */
-export function promiseAllMaybeAsync<T>(fns: Array<() => Promise<T>>): Promise<T[]> {
-  return Promise.all(
-    fns.map((fn) => {
-      try {
-        return fn();
-      } catch (e) {
-        return Promise.reject(e);
-      }
-    })
-  );
-}