@aztec/p2p 0.1.0-alpha11

package/src/bootstrap/bootstrap.ts

@@ -0,0 +1,107 @@
+import { Libp2p, Libp2pOptions, ServiceFactoryMap, createLibp2p } from 'libp2p';
+import { tcp } from '@libp2p/tcp';
+import { noise } from '@chainsafe/libp2p-noise';
+import { mplex } from '@libp2p/mplex';
+import { kadDHT } from '@libp2p/kad-dht';
+import { createFromProtobuf } from '@libp2p/peer-id-factory';
+import { createDebugLogger } from '@aztec/foundation/log';
+import { P2PConfig } from '../config.js';
+import { yamux } from '@chainsafe/libp2p-yamux';
+import { identifyService } from 'libp2p/identify';
+import type { ServiceMap } from '@libp2p/interface-libp2p';
+import { createLibP2PPeerId } from '../index.js';
+
+/**
+ * Encapsulates a 'Bootstrap' node, used for the purpose of assisting new joiners in acquiring peers.
+ */
+export class BootstrapNode {
+  private node?: Libp2p = undefined;
+
+  constructor(private logger = createDebugLogger('aztec:p2p_bootstrap')) {}
+
+  /**
+   * Starts the bootstrap node.
+   * @param config - The P2P configuration.
+   * @returns An empty promise.
+   */
+  public async start(config: P2PConfig) {
+    const { peerIdPrivateKey, tcpListenIp, tcpListenPort, announceHostname, announcePort, minPeerCount, maxPeerCount } =
+      config;
+    const peerId = peerIdPrivateKey
+      ? await createFromProtobuf(Buffer.from(peerIdPrivateKey, 'hex'))
+      : await createLibP2PPeerId();
+    this.logger(
+      `Starting bootstrap node ${peerId} on ${tcpListenIp}:${tcpListenPort} announced at ${announceHostname}:${announcePort}`,
+    );
+
+    const opts: Libp2pOptions<ServiceMap> = {
+      start: false,
+      peerId,
+      addresses: {
+        listen: [`/ip4/${tcpListenIp}/tcp/${tcpListenPort}`],
+        announce: [`/ip4/${announceHostname}/tcp/${announcePort ?? tcpListenPort}`],
+      },
+      transports: [tcp()],
+      streamMuxers: [yamux(), mplex()],
+      connectionEncryption: [noise()],
+      connectionManager: {
+        minConnections: minPeerCount,
+        maxConnections: maxPeerCount,
+      },
+    };
+
+    const services: ServiceFactoryMap = {
+      identify: identifyService({
+        protocolPrefix: 'aztec',
+      }),
+      kadDHT: kadDHT({
+        protocolPrefix: 'aztec',
+        clientMode: false,
+      }),
+    };
+
+    this.node = await createLibp2p({
+      ...opts,
+      services,
+    });
+
+    await this.node.start();
+    this.logger(`lib p2p has started`);
+
+    // print out listening addresses
+    this.logger('listening on addresses:');
+    this.node.getMultiaddrs().forEach(addr => {
+      this.logger(addr.toString());
+    });
+
+    this.node.addEventListener('peer:discovery', evt => {
+      this.logger('Discovered %s', evt.detail.id.toString()); // Log discovered peer
+    });
+
+    this.node.addEventListener('peer:connect', evt => {
+      this.logger('Connected to %s', evt.detail.toString()); // Log connected peer
+    });
+
+    this.node.addEventListener('peer:disconnect', evt => {
+      this.logger('Disconnected from %s', evt.detail.toString()); // Log connected peer
+    });
+  }
+
+  /**
+   * Stops the bootstrap node.
+   * @returns And empty promise.
+   */
+  public async stop() {
+    // stop libp2p
+    await this.node?.stop();
+    this.logger('libp2p has stopped');
+  }
+
+  /**
+   * Returns the peerId of this node.
+   * @returns The node's peer Id
+   */
+  public getPeerId() {
+    return this.node?.peerId;
+  }
+}

package/src/client/index.ts

@@ -0,0 +1,10 @@
+import { L2BlockSource } from '@aztec/types';
+import { LibP2PService, P2PClient, P2PConfig, TxPool } from '../index.js';
+import { DummyP2PService } from '../service/dummy_service.js';
+
+export * from './p2p_client.js';
+
+export const createP2PClient = async (config: P2PConfig, txPool: TxPool, l2BlockSource: L2BlockSource) => {
+  const p2pService = config.p2pEnabled ? await LibP2PService.new(config, txPool) : new DummyP2PService();
+  return new P2PClient(l2BlockSource, txPool, p2pService);
+};

package/src/client/mocks.ts

@@ -0,0 +1,63 @@
+import { KERNEL_PUBLIC_CALL_STACK_LENGTH, Proof } from '@aztec/circuits.js';
+import { makeKernelPublicInputs, makePublicCallRequest } from '@aztec/circuits.js/factories';
+import { Tx, TxL2Logs, L2BlockSource, L2Block } from '@aztec/types';
+import times from 'lodash.times';
+
+export const MockTx = () => {
+  return new Tx(
+    makeKernelPublicInputs(),
+    new Proof(Buffer.alloc(0)),
+    TxL2Logs.random(8, 3), // 8 priv function invocations creating 3 encrypted logs each
+    TxL2Logs.random(11, 2), // 8 priv + 3 pub function invocations creating 2 unencrypted logs each
+    [],
+    times(KERNEL_PUBLIC_CALL_STACK_LENGTH, makePublicCallRequest),
+  );
+};
+
+/**
+ * A mocked implementation of L2BlockSource to be used in p2p tests.
+ */
+export class MockBlockSource implements L2BlockSource {
+  private l2Blocks: L2Block[];
+
+  constructor(private numBlocks = 100) {
+    this.l2Blocks = [];
+    for (let i = 0; i < this.numBlocks; i++) {
+      this.l2Blocks.push(L2Block.random(i));
+    }
+  }
+
+  /**
+   * Gets the number of the latest L2 block processed by the block source implementation.
+   * @returns In this mock instance, returns the number of L2 blocks that we've mocked.
+   */
+  public getBlockHeight() {
+    return Promise.resolve(this.l2Blocks.length - 1);
+  }
+
+  /**
+   * Gets the `take` amount of L2 blocks starting from `from`.
+   * @param from - Number of the first block to return (inclusive).
+   * @param take - The number of blocks to return.
+   * @returns The requested mocked L2 blocks.
+   */
+  public getL2Blocks(from: number, take: number) {
+    return Promise.resolve(this.l2Blocks.slice(from, from + take));
+  }
+
+  /**
+   * Starts the block source. In this mock implementation, this is a noop.
+   * @returns A promise that signals the initialization of the l2 block source on compmletion.
+   */
+  public start(): Promise<void> {
+    return Promise.resolve();
+  }
+
+  /**
+   * Stops the block source. In this mock implementation, this is a noop.
+   * @returns A promise that signals the l2 block source is now stopped.
+   */
+  public stop(): Promise<void> {
+    return Promise.resolve();
+  }
+}

package/src/client/p2p_client.test.ts

@@ -0,0 +1,79 @@
+import { expect, jest } from '@jest/globals';
+import { L2BlockSource } from '@aztec/types';
+
+import { P2PClient } from './p2p_client.js';
+import { TxPool } from '../tx_pool/index.js';
+import { MockBlockSource } from './mocks.js';
+import { MockTx } from './mocks.js';
+import { P2PService } from '../index.js';
+
+/**
+ * Mockify helper for testing purposes.
+ */
+type Mockify<T> = {
+  [P in keyof T]: ReturnType<typeof jest.fn>;
+};
+
+describe('In-Memory P2P Client', () => {
+  let txPool: Mockify<TxPool>;
+  let blockSource: L2BlockSource;
+  let p2pService: Mockify<P2PService>;
+
+  beforeEach(() => {
+    txPool = {
+      addTxs: jest.fn(),
+      getTxByHash: jest.fn().mockReturnValue(undefined),
+      deleteTxs: jest.fn(),
+      getAllTxs: jest.fn().mockReturnValue([]),
+      getAllTxHashes: jest.fn().mockReturnValue([]),
+      hasTx: jest.fn().mockReturnValue(false),
+    };
+
+    p2pService = {
+      start: jest.fn(),
+      stop: jest.fn(),
+      propagateTx: jest.fn(),
+      settledTxs: jest.fn(),
+    };
+
+    blockSource = new MockBlockSource();
+  });
+
+  it('can start & stop', async () => {
+    const client = new P2PClient(blockSource, txPool, p2pService);
+    expect(await client.isReady()).toEqual(false);
+
+    await client.start();
+    expect(await client.isReady()).toEqual(true);
+
+    await client.stop();
+    expect(await client.isReady()).toEqual(false);
+  });
+
+  it('adds txs to pool', async () => {
+    const client = new P2PClient(blockSource, txPool, p2pService);
+    await client.start();
+    const tx1 = MockTx();
+    const tx2 = MockTx();
+    await client.sendTx(tx1);
+    await client.sendTx(tx2);
+
+    expect(txPool.addTxs).toHaveBeenCalledTimes(2);
+    await client.stop();
+  });
+
+  it('rejects txs after being stopped', async () => {
+    const client = new P2PClient(blockSource, txPool, p2pService);
+    await client.start();
+    const tx1 = MockTx();
+    const tx2 = MockTx();
+    await client.sendTx(tx1);
+    await client.sendTx(tx2);
+
+    expect(txPool.addTxs).toHaveBeenCalledTimes(2);
+    await client.stop();
+    const tx3 = MockTx();
+    await expect(client.sendTx(tx3)).rejects.toThrow();
+    expect(txPool.addTxs).toHaveBeenCalledTimes(2);
+  });
+});

package/src/client/p2p_client.ts

@@ -0,0 +1,304 @@
+import { L2Block, L2BlockContext, L2BlockDownloader, L2BlockSource } from '@aztec/types';
+import { Tx, TxHash } from '@aztec/types';
+
+import { TxPool } from '../tx_pool/index.js';
+import { getP2PConfigEnvVars } from '../config.js';
+import { createDebugLogger } from '@aztec/foundation/log';
+import { P2PService } from '../service/service.js';
+
+/**
+ * Enum defining the possible states of the p2p client.
+ */
+export enum P2PClientState {
+  IDLE,
+  SYNCHING,
+  RUNNING,
+  STOPPED,
+}
+
+/**
+ * The synchronisation status of the P2P client.
+ */
+export interface P2PSyncState {
+  /**
+   * The current state of the p2p client.
+   */
+  state: P2PClientState;
+  /**
+   * The block number that the p2p client is synced to.
+   */
+  syncedToL2Block: number;
+}
+
+/**
+ * Interface of a P2P client.
+ **/
+export interface P2P {
+  /**
+   * Verifies the 'tx' and, if valid, adds it to local tx pool and forwards it to other peers.
+   **/
+  sendTx(tx: Tx): Promise<void>;
+
+  /**
+   * Deletes 'txs' from the pool, given hashes.
+   * NOT used if we use sendTx as reconcileTxPool will handle this.
+   **/
+  deleteTxs(txHashes: TxHash[]): Promise<void>;
+
+  /**
+   * Returns all transactions in the transaction pool.
+   * @returns An array of Txs.
+   */
+  getTxs(): Promise<Tx[]>;
+
+  /**
+   * Returns a transaction in the transaction pool by its hash.
+   * @returns A single tx or undefined.
+   */
+  getTxByhash(txHash: TxHash): Promise<Tx | undefined>;
+
+  /**
+   * Starts the p2p client.
+   * @returns A promise signalling the completion of the block sync.
+   */
+  start(): Promise<void>;
+
+  /**
+   * Stops the p2p client.
+   * @returns A promise signalling the completion of the stop process.
+   */
+  stop(): Promise<void>;
+
+  /**
+   * Indicates if the p2p client is ready for transaction submission.
+   * @returns A boolean flag indicating readiness.
+   */
+  isReady(): Promise<boolean>;
+
+  /*
+   * Returns the current status of the p2p client.
+   */
+  getStatus(): Promise<P2PSyncState>;
+}
+
+/**
+ * The P2P client implementation.
+ */
+export class P2PClient implements P2P {
+  /**
+   * L2 Block download that p2p client uses to stay in sync with latest blocks.
+   */
+  private blockDownloader: L2BlockDownloader;
+
+  /**
+   * Property that indicates whether the client is running.
+   */
+  private stopping = false;
+
+  /**
+   * The JS promise that will be running to keep the client's data in sync. Can be interrupted if the client is stopped.
+   */
+  private runningPromise!: Promise<void>;
+
+  /**
+   * Store the ID of the latest block the client has synced to.
+   */
+  private currentL2BlockNum = 0;
+
+  private currentState = P2PClientState.IDLE;
+  private syncPromise = Promise.resolve();
+  private latestBlockNumberAtStart = -1;
+  private syncResolve?: () => void = undefined;
+
+  /**
+   * In-memory P2P client constructor.
+   * @param l2BlockSource - P2P client's source for fetching existing block data.
+   * @param txPool - The client's instance of a transaction pool. Defaults to in-memory implementation.
+   * @param p2pService - The concrete instance of p2p networking to use.
+   * @param log - A logger.
+   */
+  constructor(
+    private l2BlockSource: L2BlockSource,
+    private txPool: TxPool,
+    private p2pService: P2PService,
+    private log = createDebugLogger('aztec:p2p'),
+  ) {
+    const { checkInterval, l2QueueSize } = getP2PConfigEnvVars();
+    this.blockDownloader = new L2BlockDownloader(l2BlockSource, l2QueueSize, checkInterval);
+  }
+
+  /**
+   * Starts the P2P client.
+   * @returns An empty promise signalling the synching process.
+   */
+  public async start() {
+    if (this.currentState === P2PClientState.STOPPED) {
+      throw new Error('P2P client already stopped');
+    }
+    if (this.currentState !== P2PClientState.IDLE) {
+      return this.syncPromise;
+    }
+
+    // get the current latest block number
+    this.latestBlockNumberAtStart = await this.l2BlockSource.getBlockHeight();
+
+    const blockToDownloadFrom = this.currentL2BlockNum + 1;
+
+    // if there are blocks to be retrieved, go to a synching state
+    if (blockToDownloadFrom <= this.latestBlockNumberAtStart) {
+      this.setCurrentState(P2PClientState.SYNCHING);
+      this.syncPromise = new Promise(resolve => {
+        this.syncResolve = resolve;
+      });
+      this.log(`Starting sync from ${blockToDownloadFrom}, latest block ${this.latestBlockNumberAtStart}`);
+    } else {
+      // if no blocks to be retrieved, go straight to running
+      this.setCurrentState(P2PClientState.RUNNING);
+      this.syncPromise = Promise.resolve();
+      await this.p2pService.start();
+      this.log(`Next block ${blockToDownloadFrom} already beyond latest block at ${this.latestBlockNumberAtStart}`);
+    }
+
+    // start looking for further blocks
+    const blockProcess = async () => {
+      while (!this.stopping) {
+        const blocks = await this.blockDownloader.getL2Blocks();
+        await this.handleL2Blocks(blocks);
+      }
+    };
+    this.runningPromise = blockProcess();
+    this.blockDownloader.start(blockToDownloadFrom);
+    this.log(`Started block downloader from block ${blockToDownloadFrom}`);
+    return this.syncPromise;
+  }
+
+  /**
+   * Allows consumers to stop the instance of the P2P client.
+   * 'ready' will now return 'false' and the running promise that keeps the client synced is interrupted.
+   */
+  public async stop() {
+    this.log('Stopping p2p client...');
+    this.stopping = true;
+    await this.p2pService.stop();
+    await this.blockDownloader.stop();
+    await this.runningPromise;
+    this.setCurrentState(P2PClientState.STOPPED);
+  }
+
+  /**
+   * Returns all transactions in the transaction pool.
+   * @returns An array of Txs.
+   */
+  public getTxs(): Promise<Tx[]> {
+    return Promise.resolve(this.txPool.getAllTxs());
+  }
+
+  /**
+   * Returns a transaction in the transaction pool by its hash.
+   * @param txHash - Hash of the transaction to look for in the pool.
+   * @returns A single tx or undefined.
+   */
+  getTxByhash(txHash: TxHash): Promise<Tx | undefined> {
+    return Promise.resolve(this.txPool.getTxByHash(txHash));
+  }
+
+  /**
+   * Verifies the 'tx' and, if valid, adds it to local tx pool and forwards it to other peers.
+   * @param tx - The tx to verify.
+   * @returns Empty promise.
+   **/
+  public async sendTx(tx: Tx): Promise<void> {
+    const ready = await this.isReady();
+    if (!ready) {
+      throw new Error('P2P client not ready');
+    }
+    await this.txPool.addTxs([tx]);
+    this.p2pService.propagateTx(tx);
+  }
+
+  /**
+   * Deletes the 'txs' from the pool.
+   * NOT used if we use sendTx as reconcileTxPool will handle this.
+   * @param txHashes - Hashes of the transactions to delete.
+   * @returns Empty promise.
+   **/
+  public async deleteTxs(txHashes: TxHash[]): Promise<void> {
+    const ready = await this.isReady();
+    if (!ready) {
+      throw new Error('P2P client not ready');
+    }
+    this.txPool.deleteTxs(txHashes);
+  }
+
+  /**
+   * Public function to check if the p2p client is fully synced and ready to receive txs.
+   * @returns True if the P2P client is ready to receive txs.
+   */
+  public isReady() {
+    return Promise.resolve(this.currentState === P2PClientState.RUNNING);
+  }
+
+  /**
+   * Public function to check the latest block number that the P2P client is synced to.
+   * @returns Block number of latest L2 Block we've synced with.
+   */
+  public getSyncedBlockNum() {
+    return this.currentL2BlockNum;
+  }
+
+  /**
+   * Method to check the status the p2p client.
+   * @returns Information about p2p client status: state & syncedToBlockNum.
+   */
+  public getStatus(): Promise<P2PSyncState> {
+    return Promise.resolve({
+      state: this.currentState,
+      syncedToL2Block: this.currentL2BlockNum,
+    } as P2PSyncState);
+  }
+
+  /**
+   * Internal method that uses the provided blocks to check against the client's tx pool.
+   * @param blocks - A list of existing blocks with txs that the P2P client needs to ensure the tx pool is reconciled with.
+   * @returns Empty promise.
+   */
+  private reconcileTxPool(blocks: L2Block[]): Promise<void> {
+    for (let i = 0; i < blocks.length; i++) {
+      const blockContext = new L2BlockContext(blocks[i]);
+      const txHashes = blockContext.getTxHashes();
+      this.txPool.deleteTxs(txHashes);
+      this.p2pService.settledTxs(txHashes);
+    }
+    return Promise.resolve();
+  }
+
+  /**
+   * Method for processing new blocks.
+   * @param blocks - A list of existing blocks with txs that the P2P client needs to ensure the tx pool is reconciled with.
+   * @returns Empty promise.
+   */
+  private async handleL2Blocks(blocks: L2Block[]): Promise<void> {
+    if (!blocks.length) {
+      return Promise.resolve();
+    }
+    await this.reconcileTxPool(blocks);
+    this.currentL2BlockNum = blocks[blocks.length - 1].number;
+    this.log(`Synched to block ${this.currentL2BlockNum}`);
+    if (this.currentState === P2PClientState.SYNCHING && this.currentL2BlockNum >= this.latestBlockNumberAtStart) {
+      this.setCurrentState(P2PClientState.RUNNING);
+      if (this.syncResolve !== undefined) {
+        this.syncResolve();
+        await this.p2pService.start();
+      }
+    }
+  }
+
+  /**
+   * Method to set the value of the current state.
+   * @param newState - New state value.
+   */
+  private setCurrentState(newState: P2PClientState) {
+    this.currentState = newState;
+    this.log(`Moved to state ${P2PClientState[this.currentState]}`);
+  }
+}

package/src/config.ts

@@ -0,0 +1,113 @@
+/**
+ * P2P client configuration values.
+ */
+export interface P2PConfig {
+  /**
+   * A flag dictating whether the P2P subsystem should be enabled.
+   */
+  p2pEnabled: boolean;
+
+  /**
+   * The frequency in which to check.
+   */
+  checkInterval: number;
+
+  /**
+   * Size of queue of L2 blocks to store.
+   */
+  l2QueueSize: number;
+
+  /**
+   * The tcp port on which the P2P service should listen for connections.
+   */
+  tcpListenPort: number;
+
+  /**
+   * The tcp IP on which the P2P service should listen for connections.
+   */
+  tcpListenIp: string;
+
+  /**
+   * An optional peer id private key. If blank, will generate a random key.
+   */
+  peerIdPrivateKey?: string;
+
+  /**
+   * A list of bootstrap peers to connect to.
+   */
+  bootstrapNodes: string[];
+
+  /**
+   * Protocol identifier for transaction gossiping.
+   */
+  transactionProtocol: string;
+
+  /**
+   * Hostname to announce.
+   */
+  announceHostname?: string;
+
+  /**
+   * Port to announce.
+   */
+  announcePort?: number;
+
+  /**
+   * Optional specification to run as a server node.
+   */
+  serverMode: boolean;
+
+  /**
+   * Whether to enable NAT from libp2p (ignored for bootstrap node).
+   */
+  enableNat?: boolean;
+
+  /**
+   * The minimum number of peers (a peer count below this will cause the node to look for more peers)
+   */
+  minPeerCount: number;
+
+  /**
+   * The maximum number of peers (a peer count above this will cause the node to refuse connection attempts)
+   */
+  maxPeerCount: number;
+}
+
+/**
+ * Gets the config values for p2p client from environment variables.
+ * @returns The config values for p2p client.
+ */
+export function getP2PConfigEnvVars(): P2PConfig {
+  const {
+    P2P_ENABLED,
+    P2P_CHECK_INTERVAL,
+    P2P_L2_BLOCK_QUEUE_SIZE,
+    P2P_TCP_LISTEN_PORT,
+    P2P_TCP_LISTEN_IP,
+    PEER_ID_PRIVATE_KEY,
+    BOOTSTRAP_NODES,
+    P2P_ANNOUNCE_HOSTNAME,
+    P2P_ANNOUNCE_PORT,
+    P2P_SERVER,
+    P2P_NAT_ENABLED,
+    P2P_MIN_PEERS,
+    P2P_MAX_PEERS,
+  } = process.env;
+  const envVars: P2PConfig = {
+    p2pEnabled: P2P_ENABLED === 'true',
+    checkInterval: P2P_CHECK_INTERVAL ? +P2P_CHECK_INTERVAL : 100,
+    l2QueueSize: P2P_L2_BLOCK_QUEUE_SIZE ? +P2P_L2_BLOCK_QUEUE_SIZE : 1000,
+    tcpListenPort: P2P_TCP_LISTEN_PORT ? +P2P_TCP_LISTEN_PORT : 0,
+    tcpListenIp: P2P_TCP_LISTEN_IP ? P2P_TCP_LISTEN_IP : '0.0.0.0',
+    peerIdPrivateKey: PEER_ID_PRIVATE_KEY,
+    bootstrapNodes: BOOTSTRAP_NODES ? BOOTSTRAP_NODES.split(',') : [],
+    transactionProtocol: '',
+    announceHostname: P2P_ANNOUNCE_HOSTNAME,
+    announcePort: P2P_ANNOUNCE_PORT ? +P2P_ANNOUNCE_PORT : undefined,
+    serverMode: P2P_SERVER === 'true',
+    enableNat: P2P_NAT_ENABLED === 'true',
+    minPeerCount: P2P_MIN_PEERS ? +P2P_MIN_PEERS : 10,
+    maxPeerCount: P2P_MAX_PEERS ? +P2P_MAX_PEERS : 100,
+  };
+  return envVars;
+}

package/src/index.ts

@@ -0,0 +1,5 @@
+export * from './client/index.js';
+export * from './config.js';
+export * from './tx_pool/index.js';
+export * from './service/index.js';
+export * from './bootstrap/bootstrap.js';