@aztec/end-to-end 3.0.0-rc.5 → 4.0.0-nightly.20260107

package/src/fixtures/e2e_prover_test.ts

@@ -11,8 +11,7 @@ import {
   TestCircuitVerifier,
 } from '@aztec/bb-prover';
 import { BackendType, Barretenberg } from '@aztec/bb.js';
-import { createBlobSinkClient } from '@aztec/blob-sink/client';
-import type { BlobSinkServer } from '@aztec/blob-sink/server';
+import { createBlobClientWithFileStores } from '@aztec/blob-client/client';
 import type { DeployAztecL1ContractsReturnType } from '@aztec/ethereum/deploy-aztec-l1-contracts';
 import { Buffer32 } from '@aztec/foundation/buffer';
 import { SecretValue } from '@aztec/foundation/config';
@@ -69,7 +68,6 @@ export class FullProverTest {
   aztecNode!: AztecNode;
   aztecNodeAdmin!: AztecNodeAdmin;
   cheatCodes!: CheatCodes;
-  blobSink!: BlobSinkServer;
   private provenComponents: ProvenSetup[] = [];
   private bbConfigCleanup?: () => Promise<void>;
   private acvmConfigCleanup?: () => Promise<void>;
@@ -169,11 +167,11 @@ export class FullProverTest {
       aztecNode: this.aztecNode,
       deployL1ContractsValues: this.l1Contracts,
       cheatCodes: this.cheatCodes,
-      blobSink: this.blobSink,
     } = this.context);
     this.aztecNodeAdmin = this.context.aztecNode;
 
-    const blobSinkClient = createBlobSinkClient({ blobSinkUrl: `http://localhost:${this.blobSink.port}` });
+    const config = this.context.aztecNodeConfig;
+    const blobClient = await createBlobClientWithFileStores(config, this.logger);
 
     // Configure a full prover PXE
     let acvmConfig: Awaited<ReturnType<typeof getACVMConfig>> | undefined;
@@ -243,7 +241,7 @@ export class FullProverTest {
     this.logger.verbose('Starting archiver for new prover node');
     const archiver = await createArchiver(
       { ...this.context.aztecNodeConfig, dataDirectory: undefined },
-      { blobSinkClient, dateProvider: this.context.dateProvider },
+      { blobClient, dateProvider: this.context.dateProvider },
       { blockUntilSync: true },
     );
 
@@ -283,7 +281,6 @@ export class FullProverTest {
       {
         aztecNodeTxProvider: this.aztecNode,
         archiver: archiver as Archiver,
-        blobSinkClient,
       },
       { prefilledPublicData },
     );

package/src/fixtures/fixtures.ts

@@ -8,7 +8,7 @@ export const shouldCollectMetrics = () => {
 };
 
 export const TEST_PEER_CHECK_INTERVAL_MS = 1000;
-export const TEST_MAX_TX_POOL_SIZE = 2 * 1024 * 1024 * 1024; // 2GB
+export const TEST_MAX_PENDING_TX_POOL_COUNT = 10_000; // Number of max pending TXs ~ 1.56GB
 
 export const MNEMONIC = 'test test test test test test test test test test test junk';
 export const privateKey = Buffer.from('ac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80', 'hex');
@@ -24,5 +24,3 @@ export const NO_L1_TO_L2_MSG_ERROR =
   /No non-nullified L1 to L2 message found for message hash|Tried to consume nonexistent L1-to-L2 message/;
 export const STATIC_CALL_STATE_MODIFICATION_ERROR = /Static call cannot update the state.*/;
 export const STATIC_CONTEXT_ASSERTION_ERROR = /Assertion failed: Function .* can only be called statically.*/;
-
-export const DEFAULT_BLOB_SINK_PORT = '5052';

package/src/fixtures/snapshot_manager.ts

@@ -9,7 +9,6 @@ import type { Logger } from '@aztec/aztec.js/log';
 import type { AztecNode } from '@aztec/aztec.js/node';
 import type { Wallet } from '@aztec/aztec.js/wallet';
 import { AnvilTestWatcher, CheatCodes } from '@aztec/aztec/testing';
-import { type BlobSinkServer, createBlobSinkServer } from '@aztec/blob-sink/server';
 import { createExtendedL1Client } from '@aztec/ethereum/client';
 import { getL1ContractsConfigEnvVars } from '@aztec/ethereum/config';
 import { deployMulticall3 } from '@aztec/ethereum/contracts';
@@ -40,14 +39,13 @@ import type { Anvil } from '@viem/anvil';
 import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'fs';
 import { copySync, removeSync } from 'fs-extra/esm';
 import fs from 'fs/promises';
-import getPort from 'get-port';
 import { tmpdir } from 'os';
 import path, { join } from 'path';
 import type { Hex } from 'viem';
 import { mnemonicToAccount } from 'viem/accounts';
 import { foundry } from 'viem/chains';
 
-import { MNEMONIC, TEST_MAX_TX_POOL_SIZE, TEST_PEER_CHECK_INTERVAL_MS } from './fixtures.js';
+import { MNEMONIC, TEST_MAX_PENDING_TX_POOL_COUNT, TEST_PEER_CHECK_INTERVAL_MS } from './fixtures.js';
 import { getACVMConfig } from './get_acvm_config.js';
 import { getBBConfig } from './get_bb_config.js';
 import {
@@ -56,6 +54,7 @@ import {
   getLogger,
   getPrivateKeyFromIndex,
   getSponsoredFPCAddress,
+  setupSharedBlobStorage,
 } from './utils.js';
 import { getEndToEndTestTelemetryClient } from './with_telemetry_utils.js';
 
@@ -72,7 +71,6 @@ export type SubsystemsContext = {
   cheatCodes: CheatCodes;
   sequencer: SequencerClient;
   dateProvider: TestDateProvider;
-  blobSink: BlobSinkServer;
   initialFundedAccounts: InitialAccountData[];
   directoryToCleanup?: string;
 };
@@ -275,7 +273,6 @@ async function teardown(context: SubsystemsContext | undefined) {
     await context.bbConfig?.cleanup();
     await tryStop(context.anvil);
     await tryStop(context.watcher);
-    await tryStop(context.blobSink);
     await tryRmDir(context.directoryToCleanup, logger);
   } catch (err) {
     logger.error('Error during teardown', err);
@@ -297,8 +294,6 @@ async function setupFromFresh(
 ): Promise<SubsystemsContext> {
   logger.verbose(`Initializing state...`);
 
-  const blobSinkPort = await getPort();
-
   // Default to no slashing
   opts.slasherFlavor ??= 'none';
   deployL1ContractsArgs.slasherFlavor ??= opts.slasherFlavor;
@@ -307,7 +302,7 @@ async function setupFromFresh(
   // TODO: For some reason this is currently the union of a bunch of subsystems. That needs fixing.
   const aztecNodeConfig: AztecNodeConfig & SetupOptions = { ...getConfigEnvVars(), ...opts };
   aztecNodeConfig.peerCheckIntervalMS = TEST_PEER_CHECK_INTERVAL_MS;
-  aztecNodeConfig.maxTxPoolSize = opts.maxTxPoolSize ?? TEST_MAX_TX_POOL_SIZE;
+  aztecNodeConfig.maxPendingTxCount = opts.maxPendingTxCount ?? TEST_MAX_PENDING_TX_POOL_COUNT;
   // Only enable proving if specifically requested.
   aztecNodeConfig.realProofs = !!opts.realProofs;
   // Only enforce the time table if requested
@@ -326,7 +321,8 @@ async function setupFromFresh(
   } else {
     aztecNodeConfig.dataDirectory = statePath;
   }
-  aztecNodeConfig.blobSinkUrl = `http://127.0.0.1:${blobSinkPort}`;
+
+  await setupSharedBlobStorage(aztecNodeConfig);
 
   const hdAccount = mnemonicToAccount(MNEMONIC, { addressIndex: 0 });
   const publisherPrivKeyRaw = hdAccount.getHdKey().privateKey;
@@ -413,20 +409,6 @@ async function setupFromFresh(
 
   const telemetry = await getEndToEndTestTelemetryClient(opts.metricsPort);
 
-  // Setup blob sink service
-  const blobSink = await createBlobSinkServer(
-    {
-      l1ChainId: aztecNodeConfig.l1ChainId,
-      l1RpcUrls: aztecNodeConfig.l1RpcUrls,
-      l1Contracts: aztecNodeConfig.l1Contracts,
-      port: blobSinkPort,
-      dataDirectory: aztecNodeConfig.dataDirectory,
-      dataStoreMapSizeKb: aztecNodeConfig.dataStoreMapSizeKb,
-    },
-    telemetry,
-  );
-  await blobSink.start();
-
   logger.info('Creating and synching an aztec node...');
   const aztecNode = await AztecNodeService.createAndSync(
     aztecNodeConfig,
@@ -476,7 +458,6 @@ async function setupFromFresh(
     watcher,
     cheatCodes,
     dateProvider,
-    blobSink,
     initialFundedAccounts,
     directoryToCleanup,
   };
@@ -491,18 +472,16 @@ async function setupFromState(statePath: string, logger: Logger): Promise<Subsys
   const directoryToCleanup = path.join(tmpdir(), randomBytes(8).toString('hex'));
   await fs.mkdir(directoryToCleanup, { recursive: true });
 
-  // Run the blob sink on a random port
-  const blobSinkPort = await getPort();
-
   // TODO: For some reason this is currently the union of a bunch of subsystems. That needs fixing.
   const aztecNodeConfig: AztecNodeConfig & SetupOptions = JSON.parse(
     readFileSync(`${statePath}/aztec_node_config.json`, 'utf-8'),
     reviver,
   );
   aztecNodeConfig.dataDirectory = statePath;
-  aztecNodeConfig.blobSinkUrl = `http://127.0.0.1:${blobSinkPort}`;
   aztecNodeConfig.listenAddress = '127.0.0.1';
 
+  await setupSharedBlobStorage(aztecNodeConfig);
+
   const initialFundedAccounts: InitialAccountData[] =
     JSON.parse(readFileSync(`${statePath}/accounts.json`, 'utf-8'), reviver) || [];
   const { prefilledPublicData } = await getGenesisValues(initialFundedAccounts.map(a => a.address));
@@ -542,18 +521,6 @@ async function setupFromState(statePath: string, logger: Logger): Promise<Subsys
   await watcher.start();
 
   const telemetry = await initTelemetryClient(getTelemetryConfig());
-  const blobSink = await createBlobSinkServer(
-    {
-      l1ChainId: aztecNodeConfig.l1ChainId,
-      l1RpcUrls: aztecNodeConfig.l1RpcUrls,
-      l1Contracts: aztecNodeConfig.l1Contracts,
-      port: blobSinkPort,
-      dataDirectory: statePath,
-      dataStoreMapSizeKb: aztecNodeConfig.dataStoreMapSizeKb,
-    },
-    telemetry,
-  );
-  await blobSink.start();
 
   logger.verbose('Creating aztec node...');
   const aztecNode = await AztecNodeService.createAndSync(
@@ -603,7 +570,6 @@ async function setupFromState(statePath: string, logger: Logger): Promise<Subsys
     watcher,
     cheatCodes,
     dateProvider,
-    blobSink,
     initialFundedAccounts,
     directoryToCleanup,
   };

package/src/fixtures/utils.ts

@@ -10,8 +10,7 @@ import { type Logger, createLogger } from '@aztec/aztec.js/log';
 import { type AztecNode, createAztecNodeClient, waitForNode } from '@aztec/aztec.js/node';
 import type { Wallet } from '@aztec/aztec.js/wallet';
 import { AnvilTestWatcher, CheatCodes } from '@aztec/aztec/testing';
-import { createBlobSinkClient } from '@aztec/blob-sink/client';
-import { type BlobSinkServer, createBlobSinkServer } from '@aztec/blob-sink/server';
+import { createBlobClientWithFileStores } from '@aztec/blob-client/client';
 import { SPONSORED_FPC_SALT } from '@aztec/constants';
 import { isAnvilTestChain } from '@aztec/ethereum/chain';
 import { createExtendedL1Client } from '@aztec/ethereum/client';
@@ -70,7 +69,6 @@ import { getGenesisValues } from '@aztec/world-state/testing';
 import type { Anvil } from '@viem/anvil';
 import { randomBytes } from 'crypto';
 import fs from 'fs/promises';
-import getPort from 'get-port';
 import { tmpdir } from 'os';
 import * as path from 'path';
 import type { Hex } from 'viem';
@@ -91,6 +89,16 @@ import { isMetricsLoggingRequested, setupMetricsLogger } from './logging.js';
 export { deployAndInitializeTokenAndBridgeContracts } from '../shared/cross_chain_test_harness.js';
 export { startAnvil };
 
+/**
+ * Sets up shared blob storage using FileStore in the data directory.
+ */
+export async function setupSharedBlobStorage(config: { dataDirectory?: string } & Record<string, any>): Promise<void> {
+  const sharedBlobPath = path.join(config.dataDirectory!, 'shared-blobs');
+  await fs.mkdir(sharedBlobPath, { recursive: true });
+  config.blobFileStoreUrls = [`file://${sharedBlobPath}`];
+  config.blobFileStoreUploadUrl = `file://${sharedBlobPath}`;
+}
+
 const { AZTEC_NODE_URL = '' } = process.env;
 const getAztecUrl = () => AZTEC_NODE_URL;
 
@@ -232,7 +240,6 @@ async function setupWithRemoteEnvironment(
     mockGossipSubNetwork: undefined,
     watcher: undefined,
     dateProvider: undefined,
-    blobSink: undefined,
     telemetryClient: undefined,
     teardown,
   };
@@ -312,8 +319,6 @@ export type EndToEndContext = {
   watcher: AnvilTestWatcher | undefined;
   /** Allows tweaking current system time, used by the epoch cache only (undefined if connected to remote environment) */
   dateProvider: TestDateProvider | undefined;
-  /** The blob sink (undefined if connected to remote environment) */
-  blobSink: BlobSinkServer | undefined;
   /** Telemetry client */
   telemetryClient: TelemetryClient | undefined;
   /** Mock gossip sub network used for gossipping messages (only if mockGossipSubNetwork was set to true in opts) */
@@ -489,21 +494,7 @@ export async function setup(
 
     const telemetry = await getTelemetryClient(opts.telemetryConfig);
 
-    // Blob sink service - blobs get posted here and served from here
-    const blobSinkPort = await getPort();
-    const blobSink = await createBlobSinkServer(
-      {
-        l1ChainId: config.l1ChainId,
-        l1RpcUrls: config.l1RpcUrls,
-        l1Contracts: config.l1Contracts,
-        port: blobSinkPort,
-        dataDirectory: config.dataDirectory,
-        dataStoreMapSizeKb: config.dataStoreMapSizeKb,
-      },
-      telemetry,
-    );
-    await blobSink.start();
-    config.blobSinkUrl = `http://localhost:${blobSinkPort}`;
+    await setupSharedBlobStorage(config);
 
     logger.verbose('Creating and synching an aztec node', config);
 
@@ -519,8 +510,6 @@ export async function setup(
       config.bbWorkingDirectory = bbConfig.bbWorkingDirectory;
     }
 
-    const blobSinkClient = createBlobSinkClient(config, { logger: createLogger('node:blob-sink:client') });
-
     let mockGossipSubNetwork: MockGossipSubNetwork | undefined;
     let p2pClientDeps: P2PClientDeps<P2PClientType.Full> | undefined = undefined;
 
@@ -559,7 +548,7 @@ export async function setup(
 
     const aztecNode = await AztecNodeService.createAndSync(
       config, // REFACTOR: createAndSync mutates this config
-      { dateProvider, blobSinkClient, telemetry, p2pClientDeps, logger: createLogger('node:MAIN-aztec-node') },
+      { dateProvider, telemetry, p2pClientDeps, logger: createLogger('node:MAIN-aztec-node') },
       { prefilledPublicData },
     );
     const sequencerClient = aztecNode.getSequencer();
@@ -647,7 +636,6 @@ export async function setup(
         await tryStop(watcher, logger);
         await tryStop(anvil, logger);
 
-        await tryStop(blobSink, logger);
         await tryRmDir(directoryToCleanup, logger);
       } catch (err) {
         logger.error(`Error during e2e test teardown`, err);
@@ -657,7 +645,6 @@ export async function setup(
     return {
       aztecNode,
       aztecNodeAdmin: aztecNode,
-      blobSink,
       cheatCodes,
       ethCheatCodes,
       config,
@@ -852,13 +839,13 @@ export function createAndSyncProverNode(
       stop: () => Promise.resolve(),
     };
 
-    const blobSinkClient = createBlobSinkClient(aztecNodeConfig);
+    const blobClient = await createBlobClientWithFileStores(aztecNodeConfig, createLogger('blob-client:prover-node'));
 
     // Creating temp store and archiver for simulated prover node
     const archiverConfig = { ...aztecNodeConfig, dataDirectory: proverNodeConfig.dataDirectory };
     const archiver = await createArchiver(
       archiverConfig,
-      { blobSinkClient, dateProvider: proverNodeDeps.dateProvider },
+      { blobClient, dateProvider: proverNodeDeps.dateProvider },
       { blockUntilSync: true },
     );