@lodestar/beacon-node 1.35.0-dev.ba92bd8a88 → 1.35.0-dev.c0078a16b5

package/src/network/networkConfig.ts

@@ -1,12 +0,0 @@
-import {BeaconConfig} from "@lodestar/config";
-import {CustodyConfig} from "../util/dataColumns.js";
-import {NodeId} from "./subnets/interface.js";
-
-/**
- * Store shared data for different modules in the network stack.
- */
-export type NetworkConfig = {
-  readonly nodeId: NodeId;
-  readonly config: BeaconConfig;
-  readonly custodyConfig: CustodyConfig;
-};

package/src/network/options.ts

@@ -1,70 +0,0 @@
-import {Eth2GossipsubOpts} from "./gossip/gossipsub.js";
-import {PeerManagerOpts, PeerRpcScoreOpts} from "./peers/index.js";
-import {NetworkProcessorOpts} from "./processor/index.js";
-import {ReqRespBeaconNodeOpts} from "./reqresp/ReqRespBeaconNode.js";
-import {SubnetsServiceOpts} from "./subnets/interface.js";
-
-// Since Network is eventually intended to be run in a separate thread, ensure that all options are cloneable using structuredClone
-export interface NetworkOptions
-  extends PeerManagerOpts,
-    // remove all Functions
-    Omit<ReqRespBeaconNodeOpts, "getPeerLogMetadata" | "onRateLimit" | "disableLightClientServer">,
-    NetworkProcessorOpts,
-    PeerRpcScoreOpts,
-    SubnetsServiceOpts,
-    Omit<Eth2GossipsubOpts, "disableLightClientServer"> {
-  localMultiaddrs: string[];
-  bootMultiaddrs?: string[];
-  subscribeAllSubnets?: boolean;
-  mdns?: boolean;
-  connectToDiscv5Bootnodes?: boolean;
-  version?: string;
-  private?: boolean;
-  useWorker?: boolean;
-  maxYoungGenerationSizeMb?: number;
-  disableLightClientServer?: boolean;
-  supernode?: boolean;
-
-  /**
-   * During E2E tests observe a lot of following `missing stream`:
-   *
-   * > libp2p:mplex receiver stream with id 2 and protocol /eth2/beacon_chain/req/metadata/2/ssz_snappy ended
-   * > libp2p:mplex initiator stream with id 4 and protocol /eth2/beacon_chain/req/metadata/2/ssz_snappy ended
-   * > libp2p:mplex initiator stream with id 2 and protocol /eth2/beacon_chain/req/metadata/2/ssz_snappy ended
-   * > libp2p:mplex missing stream 2 for message type CLOSE_INITIATOR
-   * > libp2p:mplex missing stream 2 for message type CLOSE_RECEIVER
-   * > libp2p:mplex missing stream 4 for message type CLOSE_INITIATOR
-   *
-   * which results in following rate-limit error and cause the connection to close and fail the e2e tests
-   * > libp2p:mplex rate limit hit when receiving messages for streams that do not exist - closing remote connection
-   * > libp2p:mplex:stream:initiator:3 abort with error Error: Too many messages for missing streams
-   *
-   * The default value for `disconnectThreshold` in libp2p is set to `5`.
-   * We need to increase this only for the testing purpose
-   */
-  disconnectThreshold?: number;
-}
-
-export const defaultNetworkOptions: NetworkOptions = {
-  maxPeers: 210, // Allow some room above targetPeers for new inbound peers
-  targetPeers: 200,
-  localMultiaddrs: ["/ip4/0.0.0.0/tcp/9000", "/ip6/::/tcp/9000"],
-  bootMultiaddrs: [],
-  /** disabled by default */
-  discv5: null,
-  rateLimitMultiplier: 1,
-  useWorker: true,
-  // set after testing before 1.19.0, see https://github.com/ChainSafe/lodestar/issues/6596
-  disableFloodPublish: true,
-  // default set via research in https://github.com/ChainSafe/lodestar/issues/2115
-  maxYoungGenerationSizeMb: 152,
-  // subscribe 2 slots before aggregator dutied slot to get stable mesh peers as monitored on goerli
-  slotsToSubscribeBeforeAggregatorDuty: 2,
-  // This will enable the light client server by default
-  disableLightClientServer: false,
-  // specific option for fulu
-  //   - this is the same to TARGET_SUBNET_PEERS
-  //   - for fusaka-devnets, we have 25-30 peers per subnet
-  //   - for public testnets or mainnet, average number of peers per group is SAMPLES_PER_SLOT * targetPeers / NUMBER_OF_CUSTODY_GROUPS = 6.25 so this should not be an issue
-  targetGroupPeers: 6,
-};

package/src/network/peers/client.ts

@@ -1,29 +0,0 @@
-export enum ClientKind {
-  Lighthouse = "Lighthouse",
-  Nimbus = "Nimbus",
-  Teku = "Teku",
-  Prysm = "Prysm",
-  Lodestar = "Lodestar",
-  Grandine = "Grandine",
-  Unknown = "Unknown",
-}
-
-/**
- * Get known client from agent version.
- * If client is not known, don't return ClientKind.Unknown here.
- * For metrics it'll have fallback logic to use ClientKind.Unknown
- * For logs, we want to print out agentVersion instead for debugging purposes.
- */
-export function getKnownClientFromAgentVersion(agentVersion: string): ClientKind | null {
-  const slashIndex = agentVersion.indexOf("/");
-  const agent = slashIndex >= 0 ? agentVersion.slice(0, slashIndex) : agentVersion;
-  const agentLC = agent.toLowerCase();
-  if (agentLC === "lighthouse") return ClientKind.Lighthouse;
-  if (agentLC === "teku") return ClientKind.Teku;
-  if (agentLC === "prysm") return ClientKind.Prysm;
-  if (agentLC === "nimbus") return ClientKind.Nimbus;
-  if (agentLC === "grandine") return ClientKind.Grandine;
-  if (agentLC === "lodestar" || agentLC === "js-libp2p") return ClientKind.Lodestar;
-
-  return null;
-}

package/src/network/peers/datastore.ts

@@ -1,188 +0,0 @@
-import {AbortOptions} from "@libp2p/interface";
-import {BaseDatastore} from "datastore-core";
-import {FsDatastore} from "datastore-fs";
-import {Key, KeyQuery, Pair, Query} from "interface-datastore";
-
-type MemoryItem = {
-  lastAccessedMs: number;
-  data: Uint8Array;
-};
-
-/**
- * Before libp2p 0.35, peerstore stays in memory and periodically write to db after n dirty items
- * This has a memory issue because all peer data stays in memory and loaded at startup time
- * This is written for libp2p >=0.35, we maintain the same mechanism but with bounded data structure
- * This datastore includes a memory datastore and fallback to db datastore
- * Use an in-memory datastore with last accessed time and _maxMemoryItems, on start it's empty (lazy load)
- * - get: Search in-memory datastore first, if not found search from db.
- *     - If found from db, add back to the in-memory datastore
- *     - Update lastAccessedMs
- * - put: move oldest items from memory to db if there are more than _maxMemoryItems items in memory
- *     -  update memory datastore, only update db datastore if there are at least _threshold dirty items
- *     -  Update lastAccessedMs
- */
-export class Eth2PeerDataStore extends BaseDatastore {
-  private _dbDatastore: FsDatastore;
-  private _memoryDatastore: Map<string, MemoryItem>;
-  /** Same to PersistentPeerStore of the old libp2p implementation */
-  private _dirtyItems = new Set<string>();
-  /** If there are more dirty items than threshold, commit data to db */
-  private _threshold: number;
-  /** If there are more memory items than this, prune oldest ones from memory and move to db */
-  private _maxMemoryItems: number;
-
-  constructor(
-    dbDatastore: FsDatastore | string,
-    {threshold = 5, maxMemoryItems = 50}: {threshold?: number | undefined; maxMemoryItems?: number | undefined} = {}
-  ) {
-    super();
-
-    if (threshold <= 0 || maxMemoryItems <= 0) {
-      throw Error(`Invalid threshold ${threshold} or maxMemoryItems ${maxMemoryItems}`);
-    }
-    if (threshold > maxMemoryItems) {
-      throw Error(`Threshold ${threshold} should be at most maxMemoryItems ${maxMemoryItems}`);
-    }
-
-    this._dbDatastore = typeof dbDatastore === "string" ? new FsDatastore(dbDatastore) : dbDatastore;
-    this._memoryDatastore = new Map();
-    this._threshold = threshold;
-    this._maxMemoryItems = maxMemoryItems;
-  }
-
-  async open(): Promise<void> {
-    return this._dbDatastore.open();
-  }
-
-  async close(): Promise<void> {
-    return this._dbDatastore.close();
-  }
-
-  async put(key: Key, val: Uint8Array, _options?: AbortOptions): Promise<Key> {
-    return this._put(key, val, false);
-  }
-
-  /**
-   * Same interface to put with "fromDb" option, if this item is updated back from db
-   * Move oldest items from memory data store to db if it's over this._maxMemoryItems
-   */
-  async _put(key: Key, val: Uint8Array, fromDb = false): Promise<Key> {
-    while (this._memoryDatastore.size >= this._maxMemoryItems) {
-      // it's likely this is called only 1 time
-      await this.pruneMemoryDatastore();
-    }
-
-    const keyStr = key.toString();
-    const memoryItem = this._memoryDatastore.get(keyStr);
-    if (memoryItem) {
-      // update existing
-      memoryItem.lastAccessedMs = Date.now();
-      memoryItem.data = val;
-    } else {
-      // new
-      this._memoryDatastore.set(keyStr, {data: val, lastAccessedMs: Date.now()});
-    }
-
-    if (!fromDb) await this._addDirtyItem(keyStr);
-    return key;
-  }
-
-  /**
-   * Check memory datastore - update lastAccessedMs, then db datastore
-   * If found in db datastore then update back the memory datastore
-   * This throws error if not found
-   * see https://github.com/ipfs/js-datastore-level/blob/38f44058dd6be858e757a1c90b8edb31590ec0bc/src/index.js#L102
-   */
-  async get(key: Key): Promise<Uint8Array> {
-    const keyStr = key.toString();
-    const memoryItem = this._memoryDatastore.get(keyStr);
-    if (memoryItem) {
-      memoryItem.lastAccessedMs = Date.now();
-      return memoryItem.data;
-    }
-
-    // this throws error if not found
-    const dbValue = await this._dbDatastore.get(key);
-    // don't call this._memoryDatastore.set directly
-    // we want to get through prune() logic with fromDb as true
-    await this._put(key, dbValue, true);
-    return dbValue;
-  }
-
-  async has(key: Key): Promise<boolean> {
-    try {
-      await this.get(key);
-    } catch (err) {
-      // this is the same to how js-datastore-level handles notFound error
-      // https://github.com/ipfs/js-datastore-level/blob/38f44058dd6be858e757a1c90b8edb31590ec0bc/src/index.js#L121
-      if ((err as {notFound: boolean}).notFound) return false;
-      throw err;
-    }
-    return true;
-  }
-
-  async delete(key: Key): Promise<void> {
-    this._memoryDatastore.delete(key.toString());
-    await this._dbDatastore.delete(key);
-  }
-
-  async *_all(q: Query): AsyncIterable<Pair> {
-    for (const [key, value] of this._memoryDatastore.entries()) {
-      yield {
-        key: new Key(key),
-        value: value.data,
-      };
-    }
-    yield* this._dbDatastore.query(q);
-  }
-
-  async *_allKeys(q: KeyQuery): AsyncIterable<Key> {
-    for (const key of this._memoryDatastore.keys()) {
-      yield new Key(key);
-    }
-    yield* this._dbDatastore.queryKeys(q);
-  }
-
-  private async _addDirtyItem(keyStr: string): Promise<void> {
-    this._dirtyItems.add(keyStr);
-    if (this._dirtyItems.size >= this._threshold) {
-      try {
-        await this._commitData();
-      } catch (_e) {}
-    }
-  }
-
-  private async _commitData(): Promise<void> {
-    const batch = this._dbDatastore.batch();
-    for (const keyStr of this._dirtyItems) {
-      const memoryItem = this._memoryDatastore.get(keyStr);
-      if (memoryItem) {
-        batch.put(new Key(keyStr), memoryItem.data);
-      }
-    }
-    await batch.commit();
-    this._dirtyItems.clear();
-  }
-
-  /**
-   * Prune from memory and move to db
-   */
-  private async pruneMemoryDatastore(): Promise<void> {
-    let oldestAccessedMs = Date.now() + 1000;
-    let oldestKey: string | undefined = undefined;
-    let oldestValue: Uint8Array | undefined = undefined;
-
-    for (const [key, value] of this._memoryDatastore) {
-      if (value.lastAccessedMs < oldestAccessedMs) {
-        oldestAccessedMs = value.lastAccessedMs;
-        oldestKey = key;
-        oldestValue = value.data;
-      }
-    }
-
-    if (oldestKey && oldestValue) {
-      await this._dbDatastore.put(new Key(oldestKey), oldestValue);
-      this._memoryDatastore.delete(oldestKey);
-    }
-  }
-}