@lodestar/beacon-node 1.35.0-dev.e18102ed8c → 1.35.0-dev.f45a2be721

package/src/network/discv5/index.ts

@@ -0,0 +1,132 @@
+import EventEmitter from "node:events";
+import {privateKeyToProtobuf} from "@libp2p/crypto/keys";
+import {PrivateKey} from "@libp2p/interface";
+import {StrictEventEmitter} from "strict-event-emitter-types";
+import {ENR, ENRData, SignableENR} from "@chainsafe/enr";
+import {Thread, Worker, spawn} from "@chainsafe/threads";
+import {BeaconConfig, chainConfigFromJson, chainConfigToJson} from "@lodestar/config";
+import {LoggerNode} from "@lodestar/logger/node";
+import {NetworkCoreMetrics} from "../core/metrics.js";
+import {Discv5WorkerApi, Discv5WorkerData, LodestarDiscv5Opts} from "./types.js";
+
+export type Discv5Opts = {
+  privateKey: PrivateKey;
+  discv5: LodestarDiscv5Opts;
+  logger: LoggerNode;
+  config: BeaconConfig;
+  genesisTime: number;
+  metrics?: NetworkCoreMetrics;
+};
+
+export type Discv5Events = {
+  discovered: (enr: ENR) => void;
+};
+
+/**
+ * Wrapper class abstracting the details of discv5 worker instantiation and message-passing
+ */
+export class Discv5Worker extends (EventEmitter as {new (): StrictEventEmitter<EventEmitter, Discv5Events>}) {
+  private readonly subscription: {unsubscribe: () => void};
+  private closed = false;
+
+  constructor(
+    private readonly opts: Discv5Opts,
+    private readonly workerApi: Discv5WorkerApi
+  ) {
+    super();
+
+    this.subscription = workerApi.discovered().subscribe((enrObj) => this.onDiscovered(enrObj));
+  }
+
+  static async init(opts: Discv5Opts): Promise<Discv5Worker> {
+    const workerData: Discv5WorkerData = {
+      enr: opts.discv5.enr,
+      privateKeyProto: privateKeyToProtobuf(opts.privateKey),
+      bindAddrs: opts.discv5.bindAddrs,
+      config: opts.discv5.config ?? {},
+      bootEnrs: opts.discv5.bootEnrs,
+      metrics: Boolean(opts.metrics),
+      chainConfig: chainConfigFromJson(chainConfigToJson(opts.config)),
+      genesisValidatorsRoot: opts.config.genesisValidatorsRoot,
+      loggerOpts: opts.logger.toOpts(),
+      genesisTime: opts.genesisTime,
+    };
+    const worker = new Worker("./worker.js", {
+      suppressTranspileTS: Boolean(globalThis.Bun),
+      workerData,
+    } as ConstructorParameters<typeof Worker>[1]);
+
+    const workerApi = await spawn<Discv5WorkerApi>(worker, {
+      // A Lodestar Node may do very expensive task at start blocking the event loop and causing
+      // the initialization to timeout. The number below is big enough to almost disable the timeout
+      timeout: 5 * 60 * 1000,
+    });
+
+    return new Discv5Worker(opts, workerApi);
+  }
+
+  async close(): Promise<void> {
+    if (this.closed) return;
+    this.closed = true;
+
+    this.subscription.unsubscribe();
+    await this.workerApi.close();
+    await Thread.terminate(this.workerApi as unknown as Thread);
+  }
+
+  onDiscovered(obj: ENRData): void {
+    const enr = this.decodeEnr(obj);
+    if (enr) {
+      this.emit("discovered", enr);
+    }
+  }
+
+  async enr(): Promise<SignableENR> {
+    const obj = await this.workerApi.enr();
+    return new SignableENR(obj.kvs, obj.seq, this.opts.privateKey.raw);
+  }
+
+  setEnrValue(key: string, value: Uint8Array): Promise<void> {
+    return this.workerApi.setEnrValue(key, value);
+  }
+
+  async kadValues(): Promise<ENR[]> {
+    return this.decodeEnrs(await this.workerApi.kadValues());
+  }
+
+  discoverKadValues(): Promise<void> {
+    return this.workerApi.discoverKadValues();
+  }
+
+  async findRandomNode(): Promise<ENR[]> {
+    return this.decodeEnrs(await this.workerApi.findRandomNode());
+  }
+
+  scrapeMetrics(): Promise<string> {
+    return this.workerApi.scrapeMetrics();
+  }
+
+  async writeProfile(durationMs: number, dirpath: string): Promise<string> {
+    return this.workerApi.writeProfile(durationMs, dirpath);
+  }
+
+  async writeHeapSnapshot(prefix: string, dirpath: string): Promise<string> {
+    return this.workerApi.writeHeapSnapshot(prefix, dirpath);
+  }
+
+  private decodeEnrs(objs: ENRData[]): ENR[] {
+    const enrs: ENR[] = [];
+    for (const obj of objs) {
+      const enr = this.decodeEnr(obj);
+      if (enr) {
+        enrs.push(enr);
+      }
+    }
+    return enrs;
+  }
+
+  private decodeEnr(obj: ENRData): ENR | null {
+    this.opts.metrics?.discv5.decodeEnrAttemptCount.inc(1);
+    return new ENR(obj.kvs, obj.seq, obj.signature);
+  }
+}

package/src/network/discv5/types.ts

@@ -0,0 +1,74 @@
+import {Discv5} from "@chainsafe/discv5";
+import {ENRData, SignableENRData} from "@chainsafe/enr";
+import {Observable} from "@chainsafe/threads/observable";
+import {ChainConfig} from "@lodestar/config";
+import {LoggerNodeOpts} from "@lodestar/logger/node";
+
+// TODO export IDiscv5Config so we don't need this convoluted type
+type Discv5Config = Parameters<(typeof Discv5)["create"]>[0]["config"];
+
+type BindAddrs =
+  | {
+      ip4: string;
+      ip6?: string;
+    }
+  | {
+      ip4?: string;
+      ip6: string;
+    }
+  | {
+      ip4: string;
+      ip6: string;
+    };
+
+export type LodestarDiscv5Opts = {
+  config?: Discv5Config;
+  enr: string;
+  bindAddrs: BindAddrs;
+  bootEnrs: string[];
+};
+
+/** discv5 worker constructor data */
+export interface Discv5WorkerData {
+  enr: string;
+  privateKeyProto: Uint8Array;
+  bindAddrs: BindAddrs;
+  config: Discv5Config;
+  bootEnrs: string[];
+  metrics: boolean;
+  chainConfig: ChainConfig;
+  genesisValidatorsRoot: Uint8Array;
+  loggerOpts: LoggerNodeOpts;
+  genesisTime: number;
+}
+
+/**
+ * API exposed by the discv5 worker
+ *
+ * Note: ENRs are represented as bytes to facilitate message-passing
+ */
+export type Discv5WorkerApi = {
+  /** The current host ENR */
+  enr(): Promise<SignableENRData>;
+  /** Set a key-value of the current host ENR */
+  setEnrValue(key: string, value: Uint8Array): Promise<void>;
+
+  /** Return the ENRs currently in the kad table */
+  kadValues(): Promise<ENRData[]>;
+  /** emit the ENRs currently in the kad table */
+  discoverKadValues(): Promise<void>;
+  /** Begin a random search through the DHT, return discovered ENRs */
+  findRandomNode(): Promise<ENRData[]>;
+  /** Stream of discovered ENRs */
+  discovered(): Observable<ENRData>;
+
+  /** Prometheus metrics string */
+  scrapeMetrics(): Promise<string>;
+
+  /** write profile to disc */
+  writeProfile(durationMs: number, dirpath: string): Promise<string>;
+  /** write heap snapshot to disc */
+  writeHeapSnapshot(prefix: string, dirpath: string): Promise<string>;
+  /** tear down discv5 resources */
+  close(): Promise<void>;
+};

package/src/network/discv5/utils.ts

@@ -0,0 +1,50 @@
+import {ENR} from "@chainsafe/enr";
+import {BeaconConfig} from "@lodestar/config";
+import {MAXIMUM_GOSSIP_CLOCK_DISPARITY} from "../../constants/constants.js";
+import {IClock} from "../../util/clock.js";
+import {ENRKey} from "../metadata.js";
+
+export enum ENRRelevance {
+  no_tcp = "no_tcp",
+  no_eth2 = "no_eth2",
+  // biome-ignore lint/style/useNamingConvention: Need to use the this name for network convention
+  unknown_forkDigest = "unknown_forkDigest",
+  current_fork_mismatch = "current_fork_mismatch",
+  relevant = "relevant",
+}
+
+export function enrRelevance(enr: ENR, config: BeaconConfig, clock: IClock): ENRRelevance {
+  // We are not interested in peers that don't advertise their tcp addr
+  const multiaddrTCP = enr.getLocationMultiaddr(ENRKey.tcp);
+  if (!multiaddrTCP) {
+    return ENRRelevance.no_tcp;
+  }
+
+  // Check if the ENR.eth2 field matches and is of interest
+  const eth2 = enr.kvs.get(ENRKey.eth2);
+  if (!eth2) {
+    return ENRRelevance.no_eth2;
+  }
+
+  // Fast de-serialization without SSZ
+  const forkDigest = eth2.slice(0, 4);
+  // Check if forkDigest matches any of our known forks.
+  const {fork: forkName} = config.forkDigest2ForkBoundaryOption(forkDigest) ?? {};
+  if (forkName === undefined) {
+    return ENRRelevance.unknown_forkDigest;
+  }
+
+  // Check if fork digest's fork matches ours
+  const currentSlot = clock.slotWithFutureTolerance(MAXIMUM_GOSSIP_CLOCK_DISPARITY);
+  const localForkInfo = config.getForkInfo(currentSlot);
+  // We only connect if the ENR's fork matches our current fork.
+  // We also allow it to be the previous fork due to delay and infrequent update of DHT.
+  if (forkName !== localForkInfo.name && forkName !== localForkInfo.prevForkName) {
+    return ENRRelevance.current_fork_mismatch;
+  }
+
+  // TODO: If we have next fork scheduled, check if next fork info matches ours
+  // const enrForkId = ssz.phase0.ENRForkID.deserialize(eth2);
+
+  return ENRRelevance.relevant;
+}

package/src/network/discv5/worker.ts

@@ -0,0 +1,137 @@
+import fs from "node:fs";
+import path from "node:path";
+import worker from "node:worker_threads";
+import {privateKeyFromProtobuf} from "@libp2p/crypto/keys";
+import {peerIdFromPrivateKey} from "@libp2p/peer-id";
+import {Multiaddr, multiaddr} from "@multiformats/multiaddr";
+import {Discv5, Discv5EventEmitter} from "@chainsafe/discv5";
+import {ENR, ENRData, SignableENR, SignableENRData} from "@chainsafe/enr";
+import {Observable, Subject} from "@chainsafe/threads/observable";
+import {expose} from "@chainsafe/threads/worker";
+import {createBeaconConfig} from "@lodestar/config";
+import {getNodeLogger} from "@lodestar/logger/node";
+import {Gauge} from "@lodestar/utils";
+import {RegistryMetricCreator} from "../../metrics/index.js";
+import {collectNodeJSMetrics} from "../../metrics/nodeJsMetrics.js";
+import {Clock} from "../../util/clock.js";
+import {profileNodeJS, writeHeapSnapshot} from "../../util/profile.js";
+import {Discv5WorkerApi, Discv5WorkerData} from "./types.js";
+import {ENRRelevance, enrRelevance} from "./utils.js";
+
+// This discv5 worker will start discv5 on initialization (there is no `start` function to call)
+// A consumer _should_ call `close` before terminating the worker to cleanly exit discv5 before destroying the thread
+// A `setEnrValue` function is also provided to update the host ENR key-values shared in the discv5 network.
+
+// Cloned data from instatiation
+const workerData = worker.workerData as Discv5WorkerData;
+if (!workerData) throw Error("workerData must be defined");
+
+const logger = getNodeLogger(workerData.loggerOpts);
+
+// Set up metrics, nodejs and discv5-specific
+let metricsRegistry: RegistryMetricCreator | undefined;
+let enrRelevanceMetric: Gauge<{status: string}> | undefined;
+let closeMetrics: () => void | undefined;
+if (workerData.metrics) {
+  metricsRegistry = new RegistryMetricCreator();
+  closeMetrics = collectNodeJSMetrics(metricsRegistry, "discv5_worker_");
+
+  // add enr relevance metric
+  enrRelevanceMetric = metricsRegistry.gauge<{status: string}>({
+    name: "lodestar_discv5_discovered_status_total_count",
+    help: "Total count of status results of enrRelevance() function",
+    labelNames: ["status"],
+  });
+}
+
+const privateKey = privateKeyFromProtobuf(workerData.privateKeyProto);
+const peerId = peerIdFromPrivateKey(privateKey);
+
+const config = createBeaconConfig(workerData.chainConfig, workerData.genesisValidatorsRoot);
+
+// Initialize discv5
+const discv5 = Discv5.create({
+  enr: SignableENR.decodeTxt(workerData.enr, privateKey.raw),
+  privateKey,
+  bindAddrs: {
+    ip4: (workerData.bindAddrs.ip4 ? multiaddr(workerData.bindAddrs.ip4) : undefined) as Multiaddr,
+    ip6: workerData.bindAddrs.ip6 ? multiaddr(workerData.bindAddrs.ip6) : undefined,
+  },
+  config: workerData.config,
+  metricsRegistry,
+}) as Discv5 & Discv5EventEmitter;
+
+// Load boot enrs
+for (const bootEnr of workerData.bootEnrs) {
+  discv5.addEnr(bootEnr);
+}
+
+/** Used to push discovered ENRs */
+const subject = new Subject<ENRData>();
+
+/** Define a new clock */
+const abortController = new AbortController();
+const clock = new Clock({config, genesisTime: workerData.genesisTime, signal: abortController.signal});
+
+const onDiscovered = (enr: ENR): void => {
+  const status = enrRelevance(enr, config, clock);
+  enrRelevanceMetric?.inc({status});
+  if (status === ENRRelevance.relevant) {
+    subject.next(enr.toObject());
+  }
+};
+discv5.addListener("discovered", onDiscovered);
+
+// Discv5 will now begin accepting request/responses
+await discv5.start();
+
+const module: Discv5WorkerApi = {
+  async enr(): Promise<SignableENRData> {
+    return discv5.enr.toObject();
+  },
+  async setEnrValue(key: string, value: Uint8Array): Promise<void> {
+    discv5.enr.set(key, value);
+  },
+  async kadValues(): Promise<ENRData[]> {
+    return discv5.kadValues().map((enr: ENR) => enr.toObject());
+  },
+  async discoverKadValues(): Promise<void> {
+    discv5.kadValues().map(onDiscovered);
+  },
+  async findRandomNode(): Promise<ENRData[]> {
+    return (await discv5.findRandomNode()).map((enr: ENR) => enr.toObject());
+  },
+  discovered() {
+    return Observable.from(subject);
+  },
+  async scrapeMetrics(): Promise<string> {
+    return (await metricsRegistry?.metrics()) ?? "";
+  },
+  writeProfile: async (durationMs: number, dirpath: string) => {
+    const profile = await profileNodeJS(durationMs);
+    const filePath = path.join(dirpath, `discv5_thread_${new Date().toISOString()}.cpuprofile`);
+    fs.writeFileSync(filePath, profile);
+    return filePath;
+  },
+  writeHeapSnapshot: async (prefix: string, dirpath: string) => {
+    return writeHeapSnapshot(prefix, dirpath);
+  },
+  async close() {
+    closeMetrics?.();
+    discv5.removeListener("discovered", onDiscovered);
+    subject.complete();
+    await discv5.stop();
+  },
+};
+
+expose(module);
+
+const logData: Record<string, string> = {
+  peerId: peerId.toString(),
+  initialENR: workerData.enr,
+};
+
+if (workerData.bindAddrs.ip4) logData.bindAddr4 = workerData.bindAddrs.ip4;
+if (workerData.bindAddrs.ip6) logData.bindAddr6 = workerData.bindAddrs.ip6;
+
+logger.info("discv5 worker started", logData);

package/src/network/events.ts

@@ -0,0 +1,51 @@
+import {EventEmitter} from "node:events";
+import {PeerId, TopicValidatorResult} from "@libp2p/interface";
+import {CustodyIndex, Status} from "@lodestar/types";
+import {PeerIdStr} from "../util/peerId.js";
+import {StrictEventEmitterSingleArg} from "../util/strictEvents.js";
+import {EventDirection} from "../util/workerEvents.js";
+import {PendingGossipsubMessage} from "./processor/types.js";
+import {RequestTypedContainer} from "./reqresp/ReqRespBeaconNode.js";
+
+export enum NetworkEvent {
+  /** A relevant peer has connected or has been re-STATUS'd */
+  peerConnected = "peer-manager.peer-connected",
+  /** A peer has been disconnected */
+  peerDisconnected = "peer-manager.peer-disconnected",
+  reqRespRequest = "req-resp.request",
+
+  // Network processor events
+  /** (Network -> App) A gossip message is ready for validation */
+  pendingGossipsubMessage = "gossip.pendingGossipsubMessage",
+  /** (App -> Network) A gossip message has been validated */
+  gossipMessageValidationResult = "gossip.messageValidationResult",
+}
+
+export type NetworkEventData = {
+  [NetworkEvent.peerConnected]: {
+    peer: PeerIdStr;
+    status: Status;
+    custodyColumns: CustodyIndex[];
+    clientAgent: string;
+  };
+  [NetworkEvent.peerDisconnected]: {peer: PeerIdStr};
+  [NetworkEvent.reqRespRequest]: {request: RequestTypedContainer; peer: PeerId};
+  [NetworkEvent.pendingGossipsubMessage]: PendingGossipsubMessage;
+  [NetworkEvent.gossipMessageValidationResult]: {
+    msgId: string;
+    propagationSource: PeerIdStr;
+    acceptance: TopicValidatorResult;
+  };
+};
+
+export const networkEventDirection: Record<NetworkEvent, EventDirection> = {
+  [NetworkEvent.peerConnected]: EventDirection.workerToMain,
+  [NetworkEvent.peerDisconnected]: EventDirection.workerToMain,
+  [NetworkEvent.reqRespRequest]: EventDirection.none, // Only used internally in NetworkCore
+  [NetworkEvent.pendingGossipsubMessage]: EventDirection.workerToMain,
+  [NetworkEvent.gossipMessageValidationResult]: EventDirection.mainToWorker,
+};
+
+export type INetworkEventBus = StrictEventEmitterSingleArg<NetworkEventData>;
+
+export class NetworkEventBus extends (EventEmitter as {new (): INetworkEventBus}) {}

package/src/network/forks.ts

@@ -0,0 +1,94 @@
+import {ChainForkConfig, ForkBoundary} from "@lodestar/config";
+import {Epoch} from "@lodestar/types";
+
+/**
+ * Subscribe topics to the new fork N epochs before the fork. Remove all subscriptions N epochs after the fork
+ *
+ * This lookahead ensures a smooth fork transition. During `FORK_EPOCH_LOOKAHEAD` both forks will be active.
+ *
+ * ```
+ *    phase0     phase0     phase0       -
+ *      -        altair     altair     altair
+ * |----------|----------|----------|----------|
+ * 0        fork-2      fork      fork+2       oo
+ * ```
+ *
+ * It the fork epochs are very close to each other there may more than two active at once
+ *
+ * ```
+ *   f0    f0   f0    f0   f0    -
+ *   -     fa   fa    fa   fa    fa   -
+ *   -     -    fb    fb   fb    fb   fb
+ *
+ *     forka-2    forka      forka+2
+ * |     |          |          |
+ * |----------|----------|----------|----------|
+ * 0        forkb-2    forkb      forkb+2      oo
+ * ```
+ */
+export const FORK_EPOCH_LOOKAHEAD = 2;
+
+/**
+ * Return the list of `ForkBoundary`s meant to be active at `epoch`
+ * @see FORK_EPOCH_LOOKAHEAD for details on when fork boundaries are considered 'active'
+ */
+export function getActiveForkBoundaries(config: ChainForkConfig, epoch: Epoch): ForkBoundary[] {
+  const activeBoundaries: ForkBoundary[] = [];
+  const {forkBoundariesAscendingEpochOrder} = config;
+
+  for (let i = 0; i < forkBoundariesAscendingEpochOrder.length; i++) {
+    const currentForkBoundary = forkBoundariesAscendingEpochOrder[i];
+    const nextForkBoundary = forkBoundariesAscendingEpochOrder[i + 1];
+
+    const currentEpoch = currentForkBoundary.epoch;
+    const nextEpoch = nextForkBoundary !== undefined ? nextForkBoundary.epoch : Infinity;
+
+    // Edge case: If multiple fork boundaries start at the same epoch, only consider the latest one
+    if (currentEpoch === nextEpoch) {
+      continue;
+    }
+
+    if (epoch >= currentEpoch - FORK_EPOCH_LOOKAHEAD && epoch <= nextEpoch + FORK_EPOCH_LOOKAHEAD) {
+      activeBoundaries.push(currentForkBoundary);
+    }
+  }
+
+  return activeBoundaries;
+}
+
+/**
+ * Return the currentBoundary and nextBoundary given a fork/BPO schedule and `epoch`
+ */
+export function getCurrentAndNextForkBoundary(
+  config: ChainForkConfig,
+  epoch: Epoch
+): {currentBoundary: ForkBoundary; nextBoundary: ForkBoundary | undefined} {
+  if (epoch < 0) {
+    epoch = 0;
+  }
+
+  // NOTE: fork boundaries are sorted by ascending epoch
+  const boundaries = config.forkBoundariesAscendingEpochOrder;
+  let currentBoundaryIdx = -1;
+  // findLastIndex
+  for (let i = 0; i < boundaries.length; i++) {
+    if (epoch >= boundaries[i].epoch) currentBoundaryIdx = i;
+  }
+
+  let nextBoundaryIdx = currentBoundaryIdx + 1;
+  const hasNextBoundary = boundaries[nextBoundaryIdx] !== undefined && boundaries[nextBoundaryIdx].epoch !== Infinity;
+  // Keep moving the needle of nextBoundaryIdx if there the higher boundary also exists on same epoch
+  // for e.g. altair and bellatrix are on same epoch 6, next boundary should be bellatrix
+  if (hasNextBoundary) {
+    for (let i = nextBoundaryIdx + 1; i < boundaries.length; i++) {
+      // If the boundary's epoch is same as nextBoundaryIdx (which is not equal to infinity),
+      // update nextBoundaryIdx to the same
+      if (boundaries[i].epoch === boundaries[nextBoundaryIdx].epoch) nextBoundaryIdx = i;
+    }
+  }
+
+  return {
+    currentBoundary: boundaries[currentBoundaryIdx] || boundaries[0],
+    nextBoundary: hasNextBoundary ? boundaries[nextBoundaryIdx] : undefined,
+  };
+}

package/src/network/gossip/constants.ts

@@ -0,0 +1,15 @@
+import {GossipEncoding} from "./interface.js";
+
+export const GOSSIP_MSGID_LENGTH = 20;
+
+/**
+ * 4-byte domain for gossip message-id isolation of *valid* snappy messages
+ */
+export const MESSAGE_DOMAIN_VALID_SNAPPY = Buffer.from("01000000", "hex");
+
+/**
+ * 4-byte domain for gossip message-id isolation of *invalid* snappy messages
+ */
+export const MESSAGE_DOMAIN_INVALID_SNAPPY = Buffer.from("00000000", "hex");
+
+export const DEFAULT_ENCODING = GossipEncoding.ssz_snappy;

package/src/network/gossip/encoding.ts

@@ -0,0 +1,111 @@
+import {Message} from "@libp2p/interface";
+import {compress, uncompress} from "snappyjs";
+import xxhashFactory from "xxhash-wasm";
+import {digest} from "@chainsafe/as-sha256";
+import {RPC} from "@chainsafe/libp2p-gossipsub/message";
+import {DataTransform} from "@chainsafe/libp2p-gossipsub/types";
+import {ForkName} from "@lodestar/params";
+import {intToBytes} from "@lodestar/utils";
+import {MESSAGE_DOMAIN_VALID_SNAPPY} from "./constants.js";
+import {GossipTopicCache, getGossipSSZType} from "./topic.js";
+
+// Load WASM
+const xxhash = await xxhashFactory();
+
+// Use salt to prevent msgId from being mined for collisions
+const h64Seed = BigInt(Math.floor(Math.random() * 1e9));
+
+// Shared buffer to convert msgId to string
+const sharedMsgIdBuf = Buffer.alloc(20);
+
+/**
+ * The function used to generate a gossipsub message id
+ * We use the first 8 bytes of SHA256(data) for content addressing
+ */
+export function fastMsgIdFn(rpcMsg: RPC.Message): string {
+  if (rpcMsg.data) {
+    return xxhash.h64Raw(rpcMsg.data, h64Seed).toString(16);
+  }
+  return "0000000000000000";
+}
+
+export function msgIdToStrFn(msgId: Uint8Array): string {
+  // this is the same logic to `toHex(msgId)` with better performance
+  sharedMsgIdBuf.set(msgId);
+  return `0x${sharedMsgIdBuf.toString("hex")}`;
+}
+
+/**
+ * Only valid msgId. Messages that fail to snappy_decompress() are not tracked
+ */
+export function msgIdFn(gossipTopicCache: GossipTopicCache, msg: Message): Uint8Array {
+  const topic = gossipTopicCache.getTopic(msg.topic);
+
+  let vec: Uint8Array[];
+
+  if (topic.boundary.fork === ForkName.phase0) {
+    // message id for phase0.
+    // ```
+    // SHA256(MESSAGE_DOMAIN_VALID_SNAPPY + snappy_decompress(message.data))[:20]
+    // ```
+    vec = [MESSAGE_DOMAIN_VALID_SNAPPY, msg.data];
+  } else {
+    // message id for altair and subsequent future forks.
+    // ```
+    // SHA256(
+    //   MESSAGE_DOMAIN_VALID_SNAPPY +
+    //   uint_to_bytes(uint64(len(message.topic))) +
+    //   message.topic +
+    //   snappy_decompress(message.data)
+    // )[:20]
+    // ```
+    // https://github.com/ethereum/eth2.0-specs/blob/v1.1.0-alpha.7/specs/altair/p2p-interface.md#topics-and-messages
+    vec = [MESSAGE_DOMAIN_VALID_SNAPPY, intToBytes(msg.topic.length, 8), Buffer.from(msg.topic), msg.data];
+  }
+
+  return digest(Buffer.concat(vec)).subarray(0, 20);
+}
+
+export class DataTransformSnappy implements DataTransform {
+  constructor(
+    private readonly gossipTopicCache: GossipTopicCache,
+    private readonly maxSizePerMessage: number
+  ) {}
+
+  /**
+   * Takes the data published by peers on a topic and transforms the data.
+   * Should be the reverse of outboundTransform(). Example:
+   * - `inboundTransform()`: decompress snappy payload
+   * - `outboundTransform()`: compress snappy payload
+   */
+  inboundTransform(topicStr: string, data: Uint8Array): Uint8Array {
+    const uncompressedData = uncompress(data, this.maxSizePerMessage);
+
+    // check uncompressed data length before we extract beacon block root, slot or
+    // attestation data at later steps
+    const uncompressedDataLength = uncompressedData.length;
+    const topic = this.gossipTopicCache.getTopic(topicStr);
+    const sszType = getGossipSSZType(topic);
+
+    if (uncompressedDataLength < sszType.minSize) {
+      throw Error(`ssz_snappy decoded data length ${uncompressedDataLength} < ${sszType.minSize}`);
+    }
+    if (uncompressedDataLength > sszType.maxSize) {
+      throw Error(`ssz_snappy decoded data length ${uncompressedDataLength} > ${sszType.maxSize}`);
+    }
+
+    return uncompressedData;
+  }
+
+  /**
+   * Takes the data to be published (a topic and associated data) transforms the data. The
+   * transformed data will then be used to create a `RawGossipsubMessage` to be sent to peers.
+   */
+  outboundTransform(_topicStr: string, data: Uint8Array): Uint8Array {
+    if (data.length > this.maxSizePerMessage) {
+      throw Error(`ssz_snappy encoded data length ${data.length} > ${this.maxSizePerMessage}`);
+    }
+    // No need to parse topic, everything is snappy compressed
+    return compress(data);
+  }
+}

package/src/network/gossip/errors.ts

@@ -0,0 +1,7 @@
+export class GossipValidationError extends Error {
+  code: string;
+  constructor(code: string, message?: string) {
+    super(message);
+    this.code = code;
+  }
+}