@aztec/prover-client 0.0.1-commit.b655e406 → 0.0.1-commit.b8a057fa

package/src/orchestrator/chonk-cache.ts

@@ -0,0 +1,99 @@
+import type { NESTED_RECURSIVE_ROLLUP_HONK_PROOF_LENGTH } from '@aztec/constants';
+import { type Logger, type LoggerBindings, createLogger } from '@aztec/foundation/log';
+import type { L2Block } from '@aztec/stdlib/block';
+import type { PublicInputsAndRecursiveProof } from '@aztec/stdlib/interfaces/server';
+import type { PublicChonkVerifierPublicInputs } from '@aztec/stdlib/rollup';
+
+/** Result of a chonk-verifier proof, cached by tx hash. */
+export type ChonkVerifierProofResult = PublicInputsAndRecursiveProof<
+  PublicChonkVerifierPublicInputs,
+  typeof NESTED_RECURSIVE_ROLLUP_HONK_PROOF_LENGTH
+>;
+
+/**
+ * Shared cache of `getPublicChonkVerifierProof` results, keyed by tx hash.
+ *
+ * Owned by the prover-node and shared across every epoch / session: a tx remined into
+ * a different block (e.g. after a brief L1 reorg) reuses the already-computed chonk
+ * proof rather than redoing it. Entries are released via `releaseForBlocks` once the
+ * containing block is no longer interesting to the caller (e.g. its epoch's proof
+ * submission window has expired).
+ */
+export class ChonkCache {
+  private readonly cache = new Map<string, Promise<ChonkVerifierProofResult>>();
+  private readonly pending = new Map<string, AbortController>();
+  private readonly log: Logger;
+  private stopped = false;
+
+  constructor(bindings?: LoggerBindings) {
+    this.log = createLogger('prover-client:chonk-cache', bindings);
+  }
+
+  /** Returns the cached promise for `txHash`, or `undefined` if none is registered. */
+  public get(txHash: string): Promise<ChonkVerifierProofResult> | undefined {
+    return this.cache.get(txHash);
+  }
+
+  /**
+   * Atomic get-or-compute: returns the cached promise for `txHash`, or runs `factory`
+   * (passing an AbortSignal the cache controls) and caches its result. Concurrent
+   * callers for the same `txHash` share the same promise.
+   *
+   * Rejected promises are evicted so a future caller can retry. The factory's
+   * AbortSignal fires when the cache is stopped.
+   */
+  public getOrCompute(
+    txHash: string,
+    factory: (signal: AbortSignal) => Promise<ChonkVerifierProofResult>,
+  ): Promise<ChonkVerifierProofResult> {
+    if (this.stopped) {
+      return Promise.reject(new Error('ChonkCache is stopped'));
+    }
+    const existing = this.cache.get(txHash);
+    if (existing) {
+      return existing;
+    }
+    const controller = new AbortController();
+    this.pending.set(txHash, controller);
+    this.log.debug(`Enqueueing chonk-verifier circuit`, { txHash });
+    const promise = factory(controller.signal).finally(() => this.pending.delete(txHash));
+    // Silently observe the rejection branch and evict so a retry is possible.
+    promise.catch(err => {
+      this.cache.delete(txHash);
+      this.log.debug(`Chonk-verifier proof failed; evicted from cache`, { txHash, error: `${err}` });
+    });
+    this.cache.set(txHash, promise);
+    return promise;
+  }
+
+  /** Drops cache entries for every tx in the supplied blocks. */
+  public releaseForBlocks(blocks: L2Block[]): void {
+    let released = 0;
+    for (const block of blocks) {
+      for (const txEffect of block.body.txEffects) {
+        if (this.cache.delete(txEffect.txHash.toString())) {
+          released++;
+        }
+      }
+    }
+    if (released > 0) {
+      this.log.debug(`Released ${released} chonk-verifier cache entries`, {
+        blockCount: blocks.length,
+        releasedCount: released,
+      });
+    }
+  }
+
+  /** Aborts every in-flight chonk-verifier job and clears the cache. */
+  public stop(): void {
+    if (this.stopped) {
+      return;
+    }
+    this.stopped = true;
+    for (const controller of this.pending.values()) {
+      controller.abort();
+    }
+    this.pending.clear();
+    this.cache.clear();
+  }
+}

package/src/orchestrator/index.ts

@@ -1 +1,8 @@
-export { ProvingOrchestrator } from './orchestrator.js';
+export { CheckpointSubTreeOrchestrator, type SubTreeResult } from './checkpoint-sub-tree-orchestrator.js';
+export { ChonkCache, type ChonkVerifierProofResult } from './chonk-cache.js';
+export {
+  TopTreeOrchestrator,
+  TopTreeCancelledError,
+  type CheckpointTopTreeData,
+  type TopTreeResult,
+} from './top-tree-orchestrator.js';

package/src/orchestrator/orchestrator_metrics.ts

@@ -1,41 +1,18 @@
-import {
-  type Histogram,
-  Metrics,
-  type TelemetryClient,
-  type Tracer,
-  type UpDownCounter,
-  ValueType,
-} from '@aztec/telemetry-client';
+import { type Histogram, Metrics, type TelemetryClient, type Tracer } from '@aztec/telemetry-client';
 
 export class ProvingOrchestratorMetrics {
   public readonly tracer: Tracer;
 
   private baseRollupInputsDuration: Histogram;
-  private avmFallbackCount: UpDownCounter;
 
   constructor(client: TelemetryClient, name = 'ProvingOrchestrator') {
     this.tracer = client.getTracer(name);
     const meter = client.getMeter(name);
 
-    this.baseRollupInputsDuration = meter.createHistogram(Metrics.PROVING_ORCHESTRATOR_BASE_ROLLUP_INPUTS_DURATION, {
-      unit: 'ms',
-      description: 'Duration to build base rollup inputs',
-      valueType: ValueType.INT,
-    });
-
-    this.avmFallbackCount = meter.createUpDownCounter(Metrics.PROVING_ORCHESTRATOR_AVM_FALLBACK_COUNT, {
-      description: 'How many times the AVM fallback was used',
-      valueType: ValueType.INT,
-    });
-
-    this.avmFallbackCount.add(0);
+    this.baseRollupInputsDuration = meter.createHistogram(Metrics.PROVING_ORCHESTRATOR_BASE_ROLLUP_INPUTS_DURATION);
   }
 
   recordBaseRollupInputs(durationMs: number) {
     this.baseRollupInputsDuration.record(Math.ceil(durationMs));
   }
-
-  incAvmFallback() {
-    this.avmFallbackCount.add(1);
-  }
 }

package/src/orchestrator/proving-scheduler.ts

@@ -0,0 +1,160 @@
+import { AbortError } from '@aztec/foundation/error';
+import { type Logger, type LoggerBindings, createLogger } from '@aztec/foundation/log';
+import { SerialQueue } from '@aztec/foundation/queue';
+import { sleep } from '@aztec/foundation/sleep';
+
+/**
+ * Minimal surface a deferred-proving state must expose. Both `CheckpointProvingState` /
+ * `BlockProvingState` (used by `CheckpointSubTreeOrchestrator`) and `TopTreeProvingState`
+ * (used by `TopTreeOrchestrator`) satisfy it.
+ */
+export interface ProvingStateLike {
+  /** Returns false once the state has been cancelled or otherwise invalidated. */
+  verifyState(): boolean;
+  /** Surfaces a proving error to the state's owner. */
+  reject(reason: string): void;
+}
+
+/**
+ * Common scheduling infrastructure shared by every orchestrator that drives broker
+ * proving jobs:
+ *
+ *   - A shared `SerialQueue` (`deferredJobQueue`) that serialises the *act of handing a
+ *     job to the broker*, one initiation per event-loop tick. Each queue task kicks off
+ *     `safeJob` (which submits to the broker) without awaiting it, then yields with
+ *     `sleep(0)`; the next task therefore runs on the following macrotask. This does NOT
+ *     cap how many broker jobs are concurrently in flight — the broker's own queue absorbs
+ *     that. What it bounds is the burst rate: a sub-tree that synchronously discovers
+ *     thousands of ready jobs can't flood the broker (and monopolise the event loop) in a
+ *     single tick. The queue is owned by the `ProverClient` and shared across every
+ *     orchestrator (every sub-tree and top-tree across every concurrent epoch session), so
+ *     this pacing is applied once globally rather than once-per-orchestrator.
+ *   - A list of `AbortController`s (`pendingProvingJobs`) so a `cancel()` can abort
+ *     in-flight broker jobs when needed.
+ *   - A `deferredProving<T>(state, request, callback, isCancelled?)` method that wraps
+ *     a broker request in the standard "submit, drop result if state invalidated, push
+ *     errors to state.reject" envelope.
+ *
+ * Subclasses own their own concrete proving state and define `cancelInternal()` for
+ * the rest of the cleanup work (closing world-state forks, marking sub-trees
+ * cancelled, etc.). Because the queue is shared, neither `cancel()` nor `stop()` touch
+ * it — they only abort this orchestrator's in-flight broker jobs. Queued-but-unrun jobs
+ * for a cancelled orchestrator no-op via the guards in `deferredProving`.
+ */
+export abstract class ProvingScheduler {
+  protected pendingProvingJobs: AbortController[] = [];
+  protected logger: Logger;
+
+  constructor(
+    private readonly deferredJobQueue: SerialQueue,
+    loggerName = 'prover-client:proving-scheduler',
+    bindings?: LoggerBindings,
+  ) {
+    this.logger = createLogger(loggerName, bindings);
+  }
+
+  /** Number of broker jobs currently in flight. */
+  public getNumPendingProvingJobs(): number {
+    return this.pendingProvingJobs.length;
+  }
+
+  /**
+   * Optionally aborts every in-flight broker job. Aborting is the right choice on
+   * reorg-driven cancel (where the in-flight inputs are no longer valid) and the
+   * wrong choice on shutdown (where leaving jobs in the broker queue lets a restart
+   * pick them up). The shared queue is not touched — queued-but-unrun jobs belonging
+   * to this orchestrator no-op once their controller is aborted (or once their state
+   * is marked invalid by the subclass).
+   */
+  protected resetSchedulerState(abortJobs: boolean): void {
+    if (abortJobs) {
+      for (const controller of this.pendingProvingJobs) {
+        controller.abort();
+      }
+    }
+  }
+
+  /**
+   * Subclass-defined cancellation. Implementations call `resetSchedulerState(...)`
+   * and then do their own cleanup (close world-state forks, propagate cancel into
+   * the proving state, etc.).
+   */
+  protected abstract cancelInternal(): void;
+
+  /**
+   * Standard stop: cancel this orchestrator's work. The shared queue is owned by the
+   * `ProverClient` and outlives every orchestrator, so it is not drained here.
+   */
+  public stop(): Promise<void> {
+    this.cancelInternal();
+    return Promise.resolve();
+  }
+
+  /**
+   * Submits a broker request. The returned-via-callback result is dropped if the
+   * state has become invalid (re-org, cancellation) by the time it lands. Errors
+   * are routed to `state.reject` unless they are abort-driven or the state is
+   * already invalid (in which case they're a stale echo of the cancel).
+   *
+   * @param state - Object exposing `verifyState()` and `reject()`.
+   * @param request - The broker call. Receives the controller's signal.
+   * @param callback - Runs on success, after `verifyState()` is checked.
+   * @param isCancelled - Optional extra cancellation predicate (e.g. a `cancelled`
+   *   flag the subclass maintains independently of the state). Defaults to never.
+   */
+  protected deferredProving<S extends ProvingStateLike, T>(
+    state: S,
+    request: (signal: AbortSignal) => Promise<T>,
+    callback: (result: T) => void | Promise<void>,
+    isCancelled: () => boolean = () => false,
+  ): void {
+    if (!state.verifyState()) {
+      this.logger.debug(`Not enqueuing job, state no longer valid`);
+      return;
+    }
+
+    const controller = new AbortController();
+    this.pendingProvingJobs.push(controller);
+
+    // We use a 'safeJob'. We don't want promise rejections in the proving pool — we
+    // want to capture the error here and reject the proving state while keeping the
+    // event loop free of rejections.
+    const safeJob = async () => {
+      try {
+        if (controller.signal.aborted) {
+          return;
+        }
+        const result = await request(controller.signal);
+        if (controller.signal.aborted || !state.verifyState() || isCancelled()) {
+          this.logger.debug(`State no longer valid, discarding result`);
+          return;
+        }
+        await callback(result);
+      } catch (err) {
+        if (err instanceof AbortError || isCancelled()) {
+          return;
+        }
+        if (!state.verifyState()) {
+          this.logger.debug(`State no longer valid, discarding error from proving job`, err);
+          return;
+        }
+        this.logger.error(`Error thrown when proving job`, err);
+        state.reject(`${err}`);
+      } finally {
+        const idx = this.pendingProvingJobs.indexOf(controller);
+        if (idx > -1) {
+          this.pendingProvingJobs.splice(idx, 1);
+        }
+      }
+    };
+
+    void this.deferredJobQueue.put(async () => {
+      // Kick off the broker submission without awaiting it — awaiting here would serialise all
+      // proving (one job at a time) and kill parallelism. The `sleep(0)` yields the event loop
+      // so the next queued job initiates on the following macrotask, pacing bursts rather than
+      // bounding in-flight broker concurrency (the broker's own queue handles that).
+      void safeJob();
+      await sleep(0);
+    });
+  }
+}