@aztec/p2p 0.0.1-commit.f2ce05ee → 0.0.1-commit.f5a9928

package/src/msg_validators/tx_validator/tx_validation_cache.ts

@@ -0,0 +1,101 @@
+import { LruMap } from '@aztec/foundation/collection';
+import { type Logger, createLogger } from '@aztec/foundation/log';
+import type { Tx, TxValidationResult } from '@aztec/stdlib/tx';
+
+import { createHash } from 'node:crypto';
+
+/**
+ * Minimal interface consumed by {@link CachedTxValidator}.
+ * Keeping the dependency on an interface lets callers (and tests) substitute any cache implementation.
+ */
+export interface ITxValidationCache {
+  /** Returns the cached promise if present, otherwise calls `validate`, caches its promise, and returns it. */
+  getOrValidate(
+    validatorSymbol: symbol,
+    tx: Tx,
+    validate: () => Promise<TxValidationResult>,
+  ): Promise<TxValidationResult>;
+}
+
+/**
+ * Caches per-validator tx validation results to avoid redundant work across repeated validation calls.
+ *
+ * The cache key is composed from the validator symbol and tx hash, ensuring results are
+ * scoped to the specific validator that produced them.
+ *
+ * Promises are stored before they are awaited, so concurrent calls for the same pair share
+ * a single in-flight validation rather than launching duplicate work.
+ *
+ * Entries are evicted in least-recently-used order once the cache reaches `maxSize`.
+ */
+export class TxValidationCache implements ITxValidationCache {
+  #log: Logger;
+
+  private readonly entries: LruMap<string, Promise<TxValidationResult>>;
+  // Remember hashes for known Tx object references to skip rehashing on subsequent lookups.
+  // WeakMap holds keys weakly, so an entry doesn't keep the Tx alive once nothing else references it.
+  private readonly txHashesCache: WeakMap<Tx, string> = new WeakMap();
+
+  constructor(maxSize: number) {
+    this.entries = new LruMap(maxSize);
+    this.#log = createLogger('p2p:tx_validation_cache');
+  }
+
+  /**
+   * Computes the cache key scoping a validation result to a specific validator and tx.
+   *
+   * @param validatorSymbol - The symbol of the validator.
+   * @param tx - The tx to compute the key for.
+   * @returns The cache key.
+   *
+   * Note: the key should NOT use the tx.hash because it can't be trusted at this point.
+   */
+  public key(validatorSymbol: symbol, tx: Tx): string {
+    // Serializing the tx dominates cost (~5 ms). If we have already hashed this object, skip toBuffer + hash.
+    let hash = this.txHashesCache.get(tx);
+    if (hash === undefined) {
+      hash = createHash('sha256').update(tx.toBuffer()).digest('hex');
+      this.txHashesCache.set(tx, hash);
+    }
+    return `${Symbol.keyFor(validatorSymbol) ?? validatorSymbol.toString()}:${hash}`;
+  }
+
+  /** Returns the cached promise for the given key, or undefined if not cached. Refreshes recency. */
+  public get(key: string): Promise<TxValidationResult> | undefined {
+    return this.entries.get(key);
+  }
+
+  /** Stores a validation promise under the given key, evicting the LRU entry if at capacity. */
+  public set(key: string, result: Promise<TxValidationResult>): void {
+    this.entries.set(key, result);
+  }
+
+  /** Removes the cached validation promise for the given key. */
+  public delete(key: string): void {
+    this.entries.delete(key);
+  }
+
+  /**
+   * Returns the cached promise if present, otherwise calls `validate`, stores its promise
+   * immediately (before awaiting), and returns it.
+   */
+  public getOrValidate(
+    validatorSymbol: symbol,
+    tx: Tx,
+    validate: () => Promise<TxValidationResult>,
+  ): Promise<TxValidationResult> {
+    const key = this.key(validatorSymbol, tx);
+    const cached = this.get(key);
+    if (cached !== undefined) {
+      this.#log.debug('Returning cached tx validation result', {
+        validator: validatorSymbol.toString(),
+        txHash: tx.txHash.toString(),
+        key: key,
+      });
+      return cached;
+    }
+    const promise = validate();
+    this.set(key, promise);
+    return promise;
+  }
+}

package/src/services/data_store.ts

@@ -28,8 +28,6 @@ export class AztecDatastore implements Datastore {
   #memoryDatastore: Map<string, MemoryItem>;
   #dbDatastore: AztecAsyncMap<string, Uint8Array>;
 
-  #batchOps: BatchOp[] = [];
-
   private maxMemoryItems: number;
 
   constructor(db: AztecAsyncKVStore, { maxMemoryItems } = { maxMemoryItems: 50 }) {
@@ -92,23 +90,17 @@ export class AztecDatastore implements Datastore {
   }
 
   batch(): Batch {
+    const ops: BatchOp[] = [];
     return {
       put: (key, value) => {
-        this.#batchOps.push({
-          type: 'put',
-          key,
-          value,
-        });
+        ops.push({ type: 'put', key, value });
       },
       delete: key => {
-        this.#batchOps.push({
-          type: 'del',
-          key,
-        });
+        ops.push({ type: 'del', key });
       },
       commit: async () => {
         await this.#db.transactionAsync(async () => {
-          for (const op of this.#batchOps) {
+          for (const op of ops) {
             if (op.type === 'put' && op.value) {
               await this.put(op.key, op.value);
             } else if (op.type === 'del') {
@@ -116,7 +108,7 @@ export class AztecDatastore implements Datastore {
             }
           }
         });
-        this.#batchOps = []; // Clear operations after commit
+        ops.length = 0;
       },
     };
   }

package/src/services/discv5/discV5_service.ts

@@ -1,5 +1,7 @@
 import { createLogger } from '@aztec/foundation/log';
 import { sleep } from '@aztec/foundation/sleep';
+import { DateProvider } from '@aztec/foundation/timer';
+import type { AztecAsyncKVStore } from '@aztec/kv-store';
 import { type ComponentsVersions, checkCompressedComponentVersion } from '@aztec/stdlib/versioning';
 import { OtelMetricsAdapter, type TelemetryClient, getTelemetryClient } from '@aztec/telemetry-client';
 
@@ -14,9 +16,13 @@ import { createNodeENR } from '../../enr/generate-enr.js';
 import { AZTEC_ENR_KEY, Discv5Event, PeerEvent } from '../../types/index.js';
 import { convertToMultiaddr } from '../../util.js';
 import { type PeerDiscoveryService, PeerDiscoveryState } from '../service.js';
+import { PersistedEnrStore } from './persisted_enr_store.js';
 
 const delayBeforeStart = 2000; // 2sec
 
+/** Upper bound on persisted peer ENRs, to keep the store bounded as peers churn. */
+const MAX_PERSISTED_PEER_ENRS = 100;
+
 /**
  * Peer discovery service using Discv5.
  */
@@ -35,12 +41,19 @@ export class DiscV5Service extends EventEmitter implements PeerDiscoveryService
   private bootstrapNodePeerIds: PeerId[] = [];
   public bootstrapNodeEnrs: ENR[] = [];
   private trustedPeerEnrs: ENR[] = [];
+  /** Node IDs of all config-provided peers (bootstrap, trusted, private); these are never persisted. */
+  private readonly configProvidedNodeIds: Set<string>;
 
   private startTime = 0;
 
+  private currentIp: string | undefined;
+
+  /** Persistent store of discovered peer ENRs, used to re-seed discovery after a restart. */
+  private readonly persistedEnrs?: PersistedEnrStore;
+
   private handlers = {
     onMultiaddrUpdated: this.onMultiaddrUpdated.bind(this),
-    onDiscovered: this.onDiscovered.bind(this),
+    onPeerDiscovered: this.onPeerDiscovered.bind(this),
     onEnrAdded: this.onEnrAdded.bind(this),
   };
 
@@ -50,14 +63,27 @@ export class DiscV5Service extends EventEmitter implements PeerDiscoveryService
     private readonly packageVersion: string,
     telemetry: TelemetryClient = getTelemetryClient(),
     private logger = createLogger('p2p:discv5_service'),
+    store?: AztecAsyncKVStore,
     configOverrides: Partial<IDiscv5CreateOptions> = {},
+    private readonly dateProvider: DateProvider = new DateProvider(),
   ) {
     super();
+
+    this.persistedEnrs = store
+      ? new PersistedEnrStore(store, MAX_PERSISTED_PEER_ENRS, createLogger(`${this.logger.module}:enr-store`))
+      : undefined;
+
     const { p2pIp, p2pPort, p2pBroadcastPort, bootstrapNodes, trustedPeers, privatePeers } = config;
 
+    this.currentIp = p2pIp;
     this.bootstrapNodeEnrs = bootstrapNodes.map(x => ENR.decodeTxt(x));
     const privatePeerEnrs = new Set(privatePeers);
     this.trustedPeerEnrs = trustedPeers.filter(x => !privatePeerEnrs.has(x)).map(x => ENR.decodeTxt(x));
+    // Peers supplied via config are re-injected from config on every start, so they must never end up
+    // in the persisted (discovered-peer) store — private peers especially must not be written there.
+    this.configProvidedNodeIds = new Set(
+      [...bootstrapNodes, ...trustedPeers, ...privatePeers].map(x => ENR.decodeTxt(x).nodeId),
+    );
 
     // If no overridden broadcast port is provided, use the p2p port as the broadcast port
     if (!p2pBroadcastPort) {
@@ -96,7 +122,8 @@ export class DiscV5Service extends EventEmitter implements PeerDiscoveryService
         lookupTimeout: 2000,
         requestTimeout: 2000,
         allowUnverifiedSessions: true,
-        enrUpdate: !p2pIp ? true : false, // If no p2p IP is set, enrUpdate can automatically resolve it
+        enrUpdate: config.queryForIp || !p2pIp,
+        pingInterval: config.queryForIp ? 10_000 : 300_000,
         ...configOverrides.config,
       },
       metricsRegistry,
@@ -121,17 +148,40 @@ export class DiscV5Service extends EventEmitter implements PeerDiscoveryService
       }
     };
 
-    this.discv5.on(Discv5Event.DISCOVERED, this.handlers.onDiscovered);
+    this.discv5.on(Discv5Event.DISCOVERED, this.handlers.onPeerDiscovered);
     this.discv5.on(Discv5Event.ENR_ADDED, this.handlers.onEnrAdded);
     this.discv5.on(Discv5Event.MULTIADDR_UPDATED, this.handlers.onMultiaddrUpdated);
   }
 
   private onMultiaddrUpdated(m: Multiaddr) {
-    // We want to update our tcp port to match the udp port
-    // p2pBroadcastPort is optional on config, however it is set to default within the p2p client factory
-    const multiAddrTcp = multiaddr(convertToMultiaddr(m.nodeAddress().address, this.config.p2pBroadcastPort!, 'tcp'));
+    const newIp = m.nodeAddress().address;
+    const previousIp = this.currentIp;
+
+    if (newIp === previousIp) {
+      this.logger.debug('Discv5 confirmed current IP (no change)', { ip: newIp });
+      return;
+    }
+
+    const multiAddrTcp = multiaddr(convertToMultiaddr(newIp, this.config.p2pBroadcastPort!, 'tcp'));
     this.enr.setLocationMultiaddr(multiAddrTcp);
-    this.logger.info('Multiaddr updated', { multiaddr: multiAddrTcp.toString() });
+    this.currentIp = newIp;
+
+    if (previousIp) {
+      this.logger.info('IP address changed, ENR updated', {
+        previousIp,
+        newIp,
+        multiaddr: multiAddrTcp.toString(),
+        enr: this.enr.encodeTxt(),
+      });
+    } else {
+      this.logger.info('Initial IP discovered via discv5, ENR updated', {
+        ip: newIp,
+        multiaddr: multiAddrTcp.toString(),
+        enr: this.enr.encodeTxt(),
+      });
+    }
+
+    this.emit('ip:changed', newIp);
   }
 
   public async start(): Promise<void> {
@@ -140,14 +190,19 @@ export class DiscV5Service extends EventEmitter implements PeerDiscoveryService
     }
     this.logger.debug('Starting DiscV5');
     await this.discv5.start();
-    this.startTime = Date.now();
+    this.startTime = this.dateProvider.now();
 
+    const enrUpdateEnabled = this.config.queryForIp || !this.config.p2pIp;
     this.logger.info(`DiscV5 service started`, {
       nodeId: this.enr.nodeId,
       peerId: this.peerId,
       enrUdp: await this.enr.getFullMultiaddr('udp'),
       enrTcp: await this.enr.getFullMultiaddr('tcp'),
       versions: this.versions,
+      enrUpdateEnabled,
+      queryForIp: this.config.queryForIp,
+      configuredIp: this.config.p2pIp ?? 'none',
+      pingIntervalMs: this.config.queryForIp ? 10_000 : 300_000,
     });
     this.currentState = PeerDiscoveryState.RUNNING;
 
@@ -187,6 +242,26 @@ export class DiscV5Service extends EventEmitter implements PeerDiscoveryService
         this.discv5.addEnr(enr);
       }
     }
+
+    // Re-seed discovery from peers persisted on previous runs. This lets a node rejoin the network
+    // after a restart even when no bootstrap node is reachable: discv5 has no on-disk routing table,
+    // so without this its DHT would start empty and rediscovery would depend on inbound dials alone.
+    if (this.persistedEnrs) {
+      // Drop peers that are now config-provided (re-injected above) or fail version checks.
+      const enrs = await this.persistedEnrs.load(
+        enr => this.validateEnr(enr) && !this.configProvidedNodeIds.has(enr.nodeId),
+      );
+      for (const enr of enrs) {
+        try {
+          this.discv5.addEnr(enr);
+        } catch (err) {
+          this.logger.debug(`Error re-seeding persisted ENR ${enr.nodeId}`, err);
+        }
+      }
+      if (enrs.length > 0) {
+        this.logger.info(`Re-seeded discovery with ${enrs.length} persisted peer ENRs`);
+      }
+    }
   }
 
   public async runRandomNodesQuery(): Promise<void> {
@@ -196,8 +271,8 @@ export class DiscV5Service extends EventEmitter implements PeerDiscoveryService
 
     // First, wait some time before starting the peer discovery
     // reference: https://github.com/ChainSafe/lodestar/issues/3423
-    const msSinceStart = Date.now() - this.startTime;
-    if (Date.now() - this.startTime <= delayBeforeStart) {
+    const msSinceStart = this.dateProvider.now() - this.startTime;
+    if (this.dateProvider.now() - this.startTime <= delayBeforeStart) {
       await sleep(delayBeforeStart - msSinceStart);
     }
 
@@ -232,7 +307,7 @@ export class DiscV5Service extends EventEmitter implements PeerDiscoveryService
     if (this.currentState !== PeerDiscoveryState.RUNNING) {
       return;
     }
-    await this.discv5.off(Discv5Event.DISCOVERED, this.handlers.onDiscovered);
+    await this.discv5.off(Discv5Event.DISCOVERED, this.handlers.onPeerDiscovered);
     await this.discv5.off(Discv5Event.ENR_ADDED, this.handlers.onEnrAdded);
     await this.discv5.off(Discv5Event.MULTIADDR_UPDATED, this.handlers.onMultiaddrUpdated);
 
@@ -242,12 +317,38 @@ export class DiscV5Service extends EventEmitter implements PeerDiscoveryService
   }
 
   private async onEnrAdded(enr: ENR) {
-    const multiAddrTcp = await enr.getFullMultiaddr('tcp');
-    const multiAddrUdp = await enr.getFullMultiaddr('udp');
-    this.logger.debug(`Added ENR ${enr.encodeTxt()}`, { multiAddrTcp, multiAddrUdp, nodeId: enr.nodeId });
+    try {
+      const multiAddrTcp = await enr.getFullMultiaddr('tcp');
+      const multiAddrUdp = await enr.getFullMultiaddr('udp');
+      this.logger.debug(`Added ENR ${enr.encodeTxt()}`, { multiAddrTcp, multiAddrUdp, nodeId: enr.nodeId });
+      // A peer just entered the routing table — persist it so we can re-seed discovery after a restart.
+      if (this.persistedEnrs && this.shouldPersist(enr)) {
+        await this.persistedEnrs.persist(enr);
+      }
+      this.onDiscovered(enr);
+    } catch (err) {
+      // A malformed ENR address field (e.g. a 3-byte TCP port) makes @nethermindeth/enr throw while
+      // parsing the multiaddr. This is an async event listener, so an unhandled rejection would crash
+      // the process — catch, log, and drop the ENR instead.
+      this.logger.warn(`Dropping ENR with unparseable address`, { nodeId: enr.nodeId, err });
+    }
+  }
+
+  private async onPeerDiscovered(enr: ENR) {
+    // discv5 updates an existing routing-table entry on an ENR sequence bump without emitting enrAdded,
+    // so DISCOVERED is our only signal that a peer we already track may have changed address. Refresh
+    // the persisted copy (a no-op for peers we don't already store) so we don't keep dialing a stale ENR.
+    if (this.persistedEnrs && this.shouldPersist(enr)) {
+      await this.persistedEnrs.refresh(enr);
+    }
     this.onDiscovered(enr);
   }
 
+  /** Whether an ENR belongs in the persisted store: a validated, non-config peer that isn't ourselves. */
+  private shouldPersist(enr: ENR): boolean {
+    return !this.configProvidedNodeIds.has(enr.nodeId) && enr.nodeId !== this.enr.nodeId && this.validateEnr(enr);
+  }
+
   private isOurBootnode(enr: ENR) {
     return this.bootstrapNodeEnrs.some(x => x.nodeId === enr.nodeId);
   }
@@ -281,6 +382,17 @@ export class DiscV5Service extends EventEmitter implements PeerDiscoveryService
       return false;
     }
 
+    // A malformed TCP field (e.g. a non-2-byte port) makes @nethermindeth/enr throw while building the
+    // multiaddr. Reject such an ENR here so it's never emitted as a full peer or re-parsed by an
+    // unguarded listener (which would crash the process). A missing TCP addr is allowed — those peers
+    // are simply skipped at dial time.
+    try {
+      enr.getLocationMultiaddr('tcp');
+    } catch (err) {
+      this.logger.debug(`Peer node ${enr.nodeId} has an unparseable TCP address`, { err });
+      return false;
+    }
+
     // And check it has the correct version
     let compressedVersion;
     try {

package/src/services/discv5/persisted_enr_store.ts

@@ -0,0 +1,158 @@
+import { type Logger, createLogger } from '@aztec/foundation/log';
+import type { AztecAsyncKVStore, AztecAsyncMap } from '@aztec/kv-store';
+
+import { ENR } from '@nethermindeth/enr';
+
+/** Map name under which discovered peer ENRs are persisted so discovery can be re-seeded after a restart. */
+const PERSISTED_ENRS_MAP_NAME = 'discovered_peer_enrs';
+
+/**
+ * A persisted peer entry: the ENR text plus a monotonic sequence number recording when the peer was
+ * last seen. The sequence orders the store as a FIFO so eviction drops the oldest-seen peers first.
+ */
+interface PersistedEnr {
+  enr: string;
+  seq: number;
+}
+
+function parsePersistedEnr(value: string): PersistedEnr | undefined {
+  try {
+    const parsed = JSON.parse(value);
+    if (typeof parsed?.enr === 'string' && typeof parsed?.seq === 'number') {
+      return parsed;
+    }
+  } catch {
+    // Unparseable entry; caller treats it as stale.
+  }
+  return undefined;
+}
+
+/**
+ * A bounded, transactional store of discovered peer ENRs used to re-seed discovery after a restart.
+ *
+ * Entries are keyed by nodeId (so re-seeing a peer dedups in place) and carry a monotonic sequence so
+ * the store behaves as a FIFO: when it exceeds its cap, the oldest-seen peers are evicted first. All
+ * writes run in a transaction so concurrent discovery events can't race the size check.
+ */
+export class PersistedEnrStore {
+  private readonly enrs: AztecAsyncMap<string, string>;
+  /** Monotonic sequence stamped on each entry; seeded from the store's max in {@link load}. */
+  private seq = 0;
+
+  constructor(
+    private readonly store: AztecAsyncKVStore,
+    private readonly maxEntries: number,
+    private readonly logger: Logger = createLogger('p2p:discv5:enr-store'),
+  ) {
+    this.enrs = store.openMap(PERSISTED_ENRS_MAP_NAME);
+  }
+
+  /** Adds or replaces a peer's ENR, stamping it as most-recently-seen and evicting the oldest if over cap. */
+  public async persist(enr: ENR): Promise<void> {
+    const nodeId = enr.nodeId;
+    const enrTxt = enr.encodeTxt();
+    try {
+      await this.store.transactionAsync(async () => {
+        const seq = ++this.seq;
+        const others: { nodeId: string; seq: number }[] = [];
+        for await (const [id, value] of this.enrs.entriesAsync()) {
+          if (id !== nodeId) {
+            others.push({ nodeId: id, seq: parsePersistedEnr(value)?.seq ?? 0 });
+          }
+        }
+        const overflow = others.length + 1 - this.maxEntries;
+        if (overflow > 0) {
+          others.sort((a, b) => a.seq - b.seq); // oldest-seen (lowest sequence) first
+          for (let i = 0; i < overflow; i++) {
+            await this.enrs.delete(others[i].nodeId);
+          }
+        }
+        await this.enrs.set(nodeId, JSON.stringify({ enr: enrTxt, seq }));
+      });
+    } catch (err) {
+      this.logger.warn(`Failed to persist discovered ENR ${nodeId}`, err);
+    }
+  }
+
+  /**
+   * Refreshes an already-persisted peer's stored ENR when a newer one (higher ENR sequence) is seen.
+   * A no-op for peers we don't already track — we never add new peers via this path, only update.
+   *
+   * This exists because discv5 updates an existing routing-table entry on an ENR sequence bump without
+   * emitting an `enrAdded` event, so a peer that changes address would otherwise keep its stale ENR
+   * persisted until evicted.
+   */
+  public async refresh(enr: ENR): Promise<void> {
+    const nodeId = enr.nodeId;
+    try {
+      await this.store.transactionAsync(async () => {
+        const existing = await this.enrs.getAsync(nodeId);
+        if (!existing) {
+          return;
+        }
+        // Two unrelated sequence numbers are in play:
+        //  - the ENR sequence (`enr.seq` / `storedEnrSeq`) is set by the *peer* and bumped whenever it
+        //    revises its record (e.g. changes address); we compare these to decide whether the incoming
+        //    ENR is genuinely newer and worth keeping.
+        //  - our storage sequence (`this.seq`) is an internal counter used only for FIFO eviction; we
+        //    stamp a fresh one so this entry counts as most-recently-seen.
+        const storedEnrSeq = this.decodeStoredEnr(existing)?.seq;
+        if (storedEnrSeq !== undefined && enr.seq <= storedEnrSeq) {
+          return;
+        }
+        await this.enrs.set(nodeId, JSON.stringify({ enr: enr.encodeTxt(), seq: ++this.seq }));
+      });
+    } catch (err) {
+      this.logger.warn(`Failed to refresh persisted ENR ${nodeId}`, err);
+    }
+  }
+
+  /**
+   * Loads all persisted ENRs, decoding each and keeping those the `accept` predicate allows. Entries
+   * that fail to decode or are rejected are removed from the store. The FIFO sequence resumes from the
+   * highest persisted value so eviction order survives restarts.
+   */
+  public async load(accept: (enr: ENR) => boolean): Promise<ENR[]> {
+    const valid: ENR[] = [];
+    const stale: string[] = [];
+    let maxSeq = 0;
+    for await (const [nodeId, value] of this.enrs.entriesAsync()) {
+      const parsed = parsePersistedEnr(value);
+      let enr: ENR | undefined;
+      if (parsed) {
+        try {
+          enr = ENR.decodeTxt(parsed.enr);
+        } catch (err) {
+          this.logger.debug(`Dropping undecodable persisted ENR ${nodeId}`, err);
+        }
+      }
+      if (!parsed || !enr || !accept(enr)) {
+        stale.push(nodeId);
+        continue;
+      }
+      maxSeq = Math.max(maxSeq, parsed.seq);
+      valid.push(enr);
+    }
+    this.seq = maxSeq;
+    if (stale.length > 0) {
+      await this.store.transactionAsync(async () => {
+        for (const nodeId of stale) {
+          await this.enrs.delete(nodeId);
+        }
+      });
+    }
+    return valid;
+  }
+
+  private decodeStoredEnr(value: string): ENR | undefined {
+    const parsed = parsePersistedEnr(value);
+    if (!parsed) {
+      return undefined;
+    }
+    try {
+      return ENR.decodeTxt(parsed.enr);
+    } catch {
+      return undefined;
+    }
+  }
+}