@aztec/simulator 0.0.1-commit.87a0206 → 0.0.1-commit.88c5703d4

package/src/public/public_processor/public_processor.ts

@@ -25,6 +25,7 @@ import type {
   PublicProcessorValidator,
   SequencerConfig,
 } from '@aztec/stdlib/interfaces/server';
+import { type DebugLog, type DebugLogStore, NullDebugLogStore } from '@aztec/stdlib/logs';
 import { ProvingRequestType } from '@aztec/stdlib/proofs';
 import { MerkleTreeId } from '@aztec/stdlib/trees';
 import {
@@ -130,7 +131,6 @@ class PublicProcessorTimeoutError extends Error {
  */
 export class PublicProcessor implements Traceable {
   private metrics: PublicProcessorMetrics;
-
   constructor(
     protected globalVariables: GlobalVariables,
     private guardedMerkleTree: GuardedMerkleTreeOperations,
@@ -140,6 +140,7 @@ export class PublicProcessor implements Traceable {
     telemetryClient: TelemetryClient = getTelemetryClient(),
     private log: Logger,
     private opts: Pick<SequencerConfig, 'fakeProcessingDelayPerTxMs' | 'fakeThrowAfterProcessingTxCount'> = {},
+    private debugLogStore: DebugLogStore = new NullDebugLogStore(),
   ) {
     this.metrics = new PublicProcessorMetrics(telemetryClient, 'PublicProcessor');
   }
@@ -159,12 +160,13 @@ export class PublicProcessor implements Traceable {
     txs: Iterable<Tx> | AsyncIterable<Tx>,
     limits: PublicProcessorLimits = {},
     validator: PublicProcessorValidator = {},
-  ): Promise<[ProcessedTx[], FailedTx[], Tx[], NestedProcessReturnValues[], number]> {
-    const { maxTransactions, maxBlockSize, deadline, maxBlockGas, maxBlobFields } = limits;
+  ): Promise<[ProcessedTx[], FailedTx[], Tx[], NestedProcessReturnValues[], DebugLog[]]> {
+    const { maxTransactions, deadline, maxBlockGas, maxBlobFields, isBuildingProposal } = limits;
     const { preprocessValidator, nullifierCache } = validator;
     const result: ProcessedTx[] = [];
     const usedTxs: Tx[] = [];
     const failed: FailedTx[] = [];
+    const debugLogs: DebugLog[] = [];
     const timer = new Timer();
 
     let totalSizeInBytes = 0;
@@ -186,22 +188,23 @@ export class PublicProcessor implements Traceable {
         break;
       }
 
-      // Skip this tx if it'd exceed max block size
       const txHash = tx.getTxHash().toString();
-      const preTxSizeInBytes = tx.getEstimatedPrivateTxEffectsSize();
-      if (maxBlockSize !== undefined && totalSizeInBytes + preTxSizeInBytes > maxBlockSize) {
-        this.log.warn(`Skipping processing of tx ${txHash} sized ${preTxSizeInBytes} bytes due to block size limit`, {
-          txHash,
-          sizeInBytes: preTxSizeInBytes,
-          totalSizeInBytes,
-          maxBlockSize,
-        });
+
+      // Skip this tx if its estimated blob fields would exceed the limit.
+      // Only done during proposal building: during re-execution we must process the exact txs from the proposal.
+      const txBlobFields = tx.getPrivateTxEffectsSizeInFields();
+      if (isBuildingProposal && maxBlobFields !== undefined && totalBlobFields + txBlobFields > maxBlobFields) {
+        this.log.warn(
+          `Skipping tx ${txHash} with ${txBlobFields} fields from private side effects due to blob fields limit`,
+          { txHash, txBlobFields, totalBlobFields, maxBlobFields },
+        );
         continue;
       }
 
-      // Skip this tx if its gas limit would exceed the block gas limit
+      // Skip this tx if its gas limit would exceed the block gas limit (either da or l2).
+      // Only done during proposal building: during re-execution we must process the exact txs from the proposal.
       const txGasLimit = tx.data.constants.txContext.gasSettings.gasLimits;
-      if (maxBlockGas !== undefined && totalBlockGas.add(txGasLimit).gtAny(maxBlockGas)) {
+      if (isBuildingProposal && maxBlockGas !== undefined && totalBlockGas.add(txGasLimit).gtAny(maxBlockGas)) {
         this.log.warn(`Skipping processing of tx ${txHash} due to block gas limit`, {
           txHash,
           txGasLimit,
@@ -241,7 +244,7 @@ export class PublicProcessor implements Traceable {
       this.contractsDB.createCheckpoint();
 
       try {
-        const [processedTx, returnValues] = await this.processTx(tx, deadline);
+        const [processedTx, returnValues, txDebugLogs] = await this.processTx(tx, deadline);
 
         // Inject a fake processing failure after N txs if requested
         const fakeThrowAfter = this.opts.fakeThrowAfterProcessingTxCount;
@@ -250,23 +253,9 @@ export class PublicProcessor implements Traceable {
         }
 
         const txBlobFields = processedTx.txEffect.getNumBlobFields();
-
-        // If the actual size of this tx would exceed block size, skip it
         const txSize = txBlobFields * Fr.SIZE_IN_BYTES;
-        if (maxBlockSize !== undefined && totalSizeInBytes + txSize > maxBlockSize) {
-          this.log.debug(`Skipping processed tx ${txHash} sized ${txSize} due to max block size.`, {
-            txHash,
-            sizeInBytes: txSize,
-            totalSizeInBytes,
-            maxBlockSize,
-          });
-          // Need to revert the checkpoint here and don't go any further
-          await checkpoint.revert();
-          this.contractsDB.revertCheckpoint();
-          continue;
-        }
 
-        // If the actual blob fields of this tx would exceed the limit, skip it
+        // If the actual blob fields of this tx would exceed the limit, skip it.
         // Note: maxBlobFields already accounts for block end blob fields and previous blocks in checkpoint.
         if (maxBlobFields !== undefined && totalBlobFields + txBlobFields > maxBlobFields) {
           this.log.debug(
@@ -284,12 +273,34 @@ export class PublicProcessor implements Traceable {
           continue;
         }
 
+        // During re-execution, check if the actual gas used by this tx would push the block over the gas limit.
+        // Unlike the proposal-building check (which uses declared gas limits pessimistically before processing),
+        // this uses actual gas and stops processing when the limit is exceeded.
+        if (
+          !isBuildingProposal &&
+          maxBlockGas !== undefined &&
+          totalBlockGas.add(processedTx.gasUsed.totalGas).gtAny(maxBlockGas)
+        ) {
+          this.log.warn(`Stopping re-execution since tx ${txHash} would push block gas over limit`, {
+            txHash,
+            txGas: processedTx.gasUsed.totalGas,
+            totalBlockGas,
+            maxBlockGas,
+          });
+          await checkpoint.revert();
+          this.contractsDB.revertCheckpoint();
+          break;
+        }
+
         // FIXME(fcarreiro): it's ugly to have to notify the validator of nullifiers.
         // I'd rather pass the validators the processedTx as well and let them deal with it.
         nullifierCache?.addNullifiers(processedTx.txEffect.nullifiers.map(n => n.toBuffer()));
         result.push(processedTx);
         usedTxs.push(tx);
         returns = returns.concat(returnValues);
+        debugLogs.push(...txDebugLogs);
+
+        this.debugLogStore.storeLogs(processedTx.hash.toString(), txDebugLogs);
 
         totalPublicGas = totalPublicGas.add(processedTx.gasUsed.publicGas);
         totalBlockGas = totalBlockGas.add(processedTx.gasUsed.totalGas);
@@ -314,14 +325,10 @@ export class PublicProcessor implements Traceable {
           // 1. At least one outstanding checkpoint that has not been committed (the one created before we processed the tx).
           // 2. Possible state updates on that checkpoint or any others created during execution.
 
-          // First we revert a checkpoint as managed by the ForkCheckpoint. This will revert whatever is the current checkpoint
-          // which may not be the one originally created by this object. But that is ok, we do this to fulfil the ForkCheckpoint
-          // lifecycle expectations and ensure it doesn't attempt to commit later on.
-          await checkpoint.revert();
-
-          // Now we want to revert any/all remaining checkpoints, destroying any outstanding state updates.
-          // This needs to be done directly on the underlying fork as the guarded fork has been stopped.
-          await this.guardedMerkleTree.getUnderlyingFork().revertAllCheckpoints();
+          // Revert all checkpoints at or above this checkpoint's depth (inclusive), destroying any outstanding state
+          // updates from this tx and any nested checkpoints created during execution. This preserves any checkpoints
+          // created by callers below our depth.
+          await checkpoint.revertToCheckpoint();
 
           // Revert any contracts added to the DB for the tx.
           this.contractsDB.revertCheckpoint();
@@ -333,9 +340,9 @@ export class PublicProcessor implements Traceable {
           break;
         }
 
-        // Roll back state to start of TX before proceeding to next TX
-        await checkpoint.revert();
-        await this.guardedMerkleTree.getUnderlyingFork().revertAllCheckpoints();
+        // Roll back state to start of TX before proceeding to next TX.
+        // Reverts all checkpoints at or above this checkpoint's depth, preserving any caller checkpoints below.
+        await checkpoint.revertToCheckpoint();
         this.contractsDB.revertCheckpoint();
         const errorMessage = err instanceof Error || err instanceof AssertionError ? err.message : 'Unknown error';
         this.log.warn(`Failed to process tx ${txHash.toString()}: ${errorMessage} ${err?.stack}`);
@@ -363,7 +370,7 @@ export class PublicProcessor implements Traceable {
       totalSizeInBytes,
     });
 
-    return [result, failed, usedTxs, returns, totalBlobFields];
+    return [result, failed, usedTxs, returns, debugLogs];
   }
 
   private async checkWorldStateUnchanged(
@@ -383,8 +390,13 @@ export class PublicProcessor implements Traceable {
   }
 
   @trackSpan('PublicProcessor.processTx', tx => ({ [Attributes.TX_HASH]: tx.getTxHash().toString() }))
-  private async processTx(tx: Tx, deadline: Date | undefined): Promise<[ProcessedTx, NestedProcessReturnValues[]]> {
-    const [time, [processedTx, returnValues]] = await elapsed(() => this.processTxWithinDeadline(tx, deadline));
+  private async processTx(
+    tx: Tx,
+    deadline: Date | undefined,
+  ): Promise<[ProcessedTx, NestedProcessReturnValues[], DebugLog[]]> {
+    const [time, [processedTx, returnValues, debugLogs]] = await elapsed(() =>
+      this.processTxWithinDeadline(tx, deadline),
+    );
 
     this.log.verbose(
       !tx.hasPublicCalls()
@@ -407,7 +419,7 @@ export class PublicProcessor implements Traceable {
       },
     );
 
-    return [processedTx, returnValues ?? []];
+    return [processedTx, returnValues ?? [], debugLogs];
   }
 
   private async doTreeInsertionsForPrivateOnlyTx(processedTx: ProcessedTx): Promise<void> {
@@ -441,10 +453,9 @@ export class PublicProcessor implements Traceable {
   private async processTxWithinDeadline(
     tx: Tx,
     deadline: Date | undefined,
-  ): Promise<[ProcessedTx, NestedProcessReturnValues[] | undefined]> {
-    const innerProcessFn: () => Promise<[ProcessedTx, NestedProcessReturnValues[] | undefined]> = tx.hasPublicCalls()
-      ? () => this.processTxWithPublicCalls(tx)
-      : () => this.processPrivateOnlyTx(tx);
+  ): Promise<[ProcessedTx, NestedProcessReturnValues[] | undefined, DebugLog[]]> {
+    const innerProcessFn: () => Promise<[ProcessedTx, NestedProcessReturnValues[] | undefined, DebugLog[]]> =
+      tx.hasPublicCalls() ? () => this.processTxWithPublicCalls(tx) : () => this.processPrivateOnlyTx(tx);
 
     // Fake a delay per tx if instructed (used for tests)
     const fakeDelayPerTxMs = this.opts.fakeProcessingDelayPerTxMs;
@@ -512,7 +523,7 @@ export class PublicProcessor implements Traceable {
   @trackSpan('PublicProcessor.processPrivateOnlyTx', (tx: Tx) => ({
     [Attributes.TX_HASH]: tx.getTxHash().toString(),
   }))
-  private async processPrivateOnlyTx(tx: Tx): Promise<[ProcessedTx, undefined]> {
+  private async processPrivateOnlyTx(tx: Tx): Promise<[ProcessedTx, undefined, DebugLog[]]> {
     const gasFees = this.globalVariables.gasFees;
     const transactionFee = computeTransactionFee(gasFees, tx.data.constants.txContext.gasSettings, tx.data.gasUsed);
 
@@ -537,13 +548,13 @@ export class PublicProcessor implements Traceable {
 
     await this.contractsDB.addNewContracts(tx);
 
-    return [processedTx, undefined];
+    return [processedTx, undefined, []];
   }
 
   @trackSpan('PublicProcessor.processTxWithPublicCalls', tx => ({
     [Attributes.TX_HASH]: tx.getTxHash().toString(),
   }))
-  private async processTxWithPublicCalls(tx: Tx): Promise<[ProcessedTx, NestedProcessReturnValues[]]> {
+  private async processTxWithPublicCalls(tx: Tx): Promise<[ProcessedTx, NestedProcessReturnValues[], DebugLog[]]> {
     const timer = new Timer();
 
     const result = await this.publicTxSimulator.simulate(tx);
@@ -581,7 +592,7 @@ export class PublicProcessor implements Traceable {
       revertReason,
     );
 
-    return [processedTx, appLogicReturnValues];
+    return [processedTx, appLogicReturnValues, result.logs ?? []];
   }
 
   /**

package/src/public/public_tx_simulator/cpp_public_tx_simulator.ts

@@ -1,4 +1,4 @@
-import { type Logger, type LoggerBindings, createLogger, logLevel } from '@aztec/foundation/log';
+import { type Logger, type LoggerBindings, createLogger } from '@aztec/foundation/log';
 import { sleep } from '@aztec/foundation/sleep';
 import { type CancellationToken, avmSimulate, cancelSimulation, createCancellationToken } from '@aztec/native';
 import { ProtocolContractsList } from '@aztec/protocol-contracts';
@@ -100,8 +100,7 @@ export class CppPublicTxSimulator extends PublicTxSimulator implements PublicTxS
       inputBuffer,
       contractProvider,
       wsCppHandle,
-      logLevel,
-      // TODO: re-enable logging
+      this.log.level,
       undefined,
       this.cancellationToken,
     );

package/src/public/public_tx_simulator/cpp_public_tx_simulator_with_hinted_dbs.ts

@@ -1,4 +1,4 @@
-import { type Logger, type LoggerBindings, createLogger, logLevel } from '@aztec/foundation/log';
+import { type Logger, type LoggerBindings, createLogger } from '@aztec/foundation/log';
 import { avmSimulateWithHintedDbs } from '@aztec/native';
 import {
   AvmCircuitInputs,
@@ -75,7 +75,7 @@ export class CppPublicTxSimulatorHintedDbs extends PublicTxSimulator implements
 
     let resultBuffer: Buffer;
     try {
-      resultBuffer = await avmSimulateWithHintedDbs(inputBuffer, logLevel);
+      resultBuffer = await avmSimulateWithHintedDbs(inputBuffer, this.log.level);
     } catch (error: any) {
       throw new SimulationError(`C++ hinted simulation failed: ${error.message}`, []);
     }

package/src/public/public_tx_simulator/factories.ts

@@ -19,10 +19,11 @@ export function createPublicTxSimulatorForBlockBuilding(
   globalVariables: GlobalVariables,
   telemetryClient: TelemetryClient,
   bindings?: LoggerBindings,
+  collectDebugLogs = false,
 ) {
   const config = PublicSimulatorConfig.from({
     skipFeeEnforcement: false,
-    collectDebugLogs: false,
+    collectDebugLogs,
     collectHints: false,
     collectPublicInputs: false,
     collectStatistics: false,

package/src/public/public_tx_simulator/public_tx_simulator.ts

@@ -1,4 +1,4 @@
-import { AVM_MAX_PROCESSABLE_L2_GAS } from '@aztec/constants';
+import { MAX_PROCESSABLE_L2_GAS } from '@aztec/constants';
 import { Fr } from '@aztec/foundation/curves/bn254';
 import { type Logger, type LoggerBindings, createLogger } from '@aztec/foundation/log';
 import { ProtocolContractAddress, ProtocolContractsList } from '@aztec/protocol-contracts';
@@ -199,8 +199,8 @@ export class PublicTxSimulator implements PublicTxSimulatorInterface {
 
     // Such transactions should be filtered by GasTxValidator.
     assert(
-      context.getActualGasUsed().l2Gas <= AVM_MAX_PROCESSABLE_L2_GAS,
-      `Transaction consumes ${context.getActualGasUsed().l2Gas} L2 gas, which exceeds the AVM maximum processable gas of ${AVM_MAX_PROCESSABLE_L2_GAS}`,
+      context.getActualGasUsed().l2Gas <= MAX_PROCESSABLE_L2_GAS,
+      `Transaction consumes ${context.getActualGasUsed().l2Gas} L2 gas, which exceeds the maximum processable gas of ${MAX_PROCESSABLE_L2_GAS}`,
     );
     await this.payFee(context);
 

package/src/public/test_executor_metrics.ts

@@ -21,6 +21,7 @@ export interface PublicTxMetrics {
   totalDurationMs: number;
   manaUsed: number | undefined;
   totalInstructionsExecuted: number;
+  bytecodeSizes: { contractName: string; sizeBytes: number }[];
   nonRevertiblePrivateInsertionsUs: number | undefined;
   revertiblePrivateInsertionsUs: number | undefined;
   enqueuedCalls: PublicEnqueuedCallMetrics[];
@@ -62,6 +63,7 @@ function createEmptyTxMetrics(): PublicTxMetrics {
     totalDurationMs: 0,
     manaUsed: 0,
     totalInstructionsExecuted: 0,
+    bytecodeSizes: [],
     nonRevertiblePrivateInsertionsUs: undefined,
     revertiblePrivateInsertionsUs: undefined,
     enqueuedCalls: [],
@@ -172,6 +174,12 @@ export class TestExecutorMetrics implements ExecutorMetricsInterface {
     }
   }
 
+  recordBytecodeSize(txLabel: string, contractName: string, sizeBytes: number) {
+    const txMetrics = this.txMetrics.get(txLabel);
+    assert(txMetrics, `Cannot record bytecode size for unknown tx label: ${txLabel}`);
+    txMetrics.bytecodeSizes.push({ contractName, sizeBytes });
+  }
+
   recordProverMetrics(txLabel: string, metrics: Partial<PublicTxMetrics>) {
     if (!this.txMetrics.has(txLabel)) {
       this.txMetrics.set(txLabel, createEmptyTxMetrics());
@@ -216,6 +224,15 @@ export class TestExecutorMetrics implements ExecutorMetricsInterface {
       ) {
         pretty += `${INDENT0}Total instructions executed: ${fmtNum(txMetrics.totalInstructionsExecuted)}\n`;
       }
+      if (
+        (filter === PublicTxMetricsFilter.TOTALS || filter === PublicTxMetricsFilter.ALL) &&
+        txMetrics.bytecodeSizes.length > 0
+      ) {
+        pretty += `${INDENT0}Bytecode sizes:\n`;
+        for (const { contractName, sizeBytes } of txMetrics.bytecodeSizes) {
+          pretty += `${INDENT1}${contractName}: ${fmtNum(sizeBytes, 'bytes')}\n`;
+        }
+      }
       if (filter === PublicTxMetricsFilter.DURATIONS || filter === PublicTxMetricsFilter.ALL) {
         pretty += `${INDENT0}Private insertions:\n`;
         pretty += `${INDENT1}Non-revertible: ${fmtNum(txMetrics.nonRevertiblePrivateInsertionsUs! / 1_000, 'ms')}\n`;
@@ -387,6 +404,13 @@ export class TestExecutorMetrics implements ExecutorMetricsInterface {
           });
         }
       }
+      for (const { contractName, sizeBytes } of txMetrics.bytecodeSizes) {
+        data.push({
+          name: `${txLabel}/bytecodeSizeBytes/${contractName}`,
+          value: sizeBytes,
+          unit: 'bytes',
+        });
+      }
     }
     return JSON.stringify(data, null, indent);
   }