@aztec/prover-node 0.0.1-commit.a072138 → 0.0.1-commit.a5db02d
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +511 -0
- package/dest/actions/download-epoch-proving-job.js +1 -1
- package/dest/actions/rerun-epoch-proving-job.d.ts +5 -4
- package/dest/actions/rerun-epoch-proving-job.d.ts.map +1 -1
- package/dest/actions/rerun-epoch-proving-job.js +103 -23
- package/dest/actions/upload-epoch-proof-failure.d.ts +2 -2
- package/dest/actions/upload-epoch-proof-failure.d.ts.map +1 -1
- package/dest/bin/run-failed-epoch.js +6 -5
- package/dest/checkpoint-store.d.ts +88 -0
- package/dest/checkpoint-store.d.ts.map +1 -0
- package/dest/checkpoint-store.js +169 -0
- package/dest/config.d.ts +5 -8
- package/dest/config.d.ts.map +1 -1
- package/dest/config.js +17 -20
- package/dest/factory.d.ts +19 -13
- package/dest/factory.d.ts.map +1 -1
- package/dest/factory.js +40 -65
- package/dest/index.d.ts +2 -1
- package/dest/index.d.ts.map +1 -1
- package/dest/index.js +1 -0
- package/dest/job/checkpoint-prover.d.ts +124 -0
- package/dest/job/checkpoint-prover.d.ts.map +1 -0
- package/dest/job/checkpoint-prover.js +330 -0
- package/dest/job/epoch-session.d.ts +148 -0
- package/dest/job/epoch-session.d.ts.map +1 -0
- package/dest/job/{epoch-proving-job.js → epoch-session.js} +278 -297
- package/dest/job/top-tree-job.d.ts +82 -0
- package/dest/job/top-tree-job.d.ts.map +1 -0
- package/dest/job/top-tree-job.js +152 -0
- package/dest/metrics.d.ts +40 -3
- package/dest/metrics.d.ts.map +1 -1
- package/dest/metrics.js +101 -4
- package/dest/monitors/epoch-monitor.d.ts +1 -1
- package/dest/monitors/epoch-monitor.d.ts.map +1 -1
- package/dest/monitors/epoch-monitor.js +11 -9
- package/dest/proof-publishing-service.d.ts +161 -0
- package/dest/proof-publishing-service.d.ts.map +1 -0
- package/dest/proof-publishing-service.js +335 -0
- package/dest/prover-node-publisher.d.ts +22 -15
- package/dest/prover-node-publisher.d.ts.map +1 -1
- package/dest/prover-node-publisher.js +198 -61
- package/dest/prover-node.d.ts +118 -70
- package/dest/prover-node.d.ts.map +1 -1
- package/dest/prover-node.js +487 -226
- package/dest/prover-publisher-factory.d.ts +4 -4
- package/dest/prover-publisher-factory.d.ts.map +1 -1
- package/dest/prover-publisher-factory.js +3 -3
- package/dest/session-manager.d.ts +158 -0
- package/dest/session-manager.d.ts.map +1 -0
- package/dest/session-manager.js +486 -0
- package/dest/test/index.d.ts +7 -6
- package/dest/test/index.d.ts.map +1 -1
- package/package.json +23 -22
- package/src/actions/download-epoch-proving-job.ts +1 -1
- package/src/actions/rerun-epoch-proving-job.ts +115 -29
- package/src/actions/upload-epoch-proof-failure.ts +1 -1
- package/src/bin/run-failed-epoch.ts +5 -3
- package/src/checkpoint-store.ts +194 -0
- package/src/config.ts +25 -32
- package/src/factory.ts +68 -111
- package/src/index.ts +1 -0
- package/src/job/checkpoint-prover.ts +442 -0
- package/src/job/epoch-session.ts +438 -0
- package/src/job/top-tree-job.ts +227 -0
- package/src/metrics.ts +123 -10
- package/src/monitors/epoch-monitor.ts +5 -6
- package/src/proof-publishing-service.ts +427 -0
- package/src/prover-node-publisher.ts +232 -78
- package/src/prover-node.ts +562 -256
- package/src/prover-publisher-factory.ts +5 -5
- package/src/session-manager.ts +585 -0
- package/src/test/index.ts +6 -6
- package/dest/job/epoch-proving-job.d.ts +0 -63
- package/dest/job/epoch-proving-job.d.ts.map +0 -1
- package/src/job/epoch-proving-job.ts +0 -435
|
@@ -371,371 +371,352 @@ function _apply_decs_2203_r(targetClass, memberDecs, classDecs, parentClass) {
|
|
|
371
371
|
return (_apply_decs_2203_r = applyDecs2203RFactory())(targetClass, memberDecs, classDecs, parentClass);
|
|
372
372
|
}
|
|
373
373
|
var _dec, _initProto;
|
|
374
|
-
import { NUMBER_OF_L1_L2_MESSAGES_PER_ROLLUP } from '@aztec/constants';
|
|
375
|
-
import { asyncPool } from '@aztec/foundation/async-pool';
|
|
376
374
|
import { BlockNumber } from '@aztec/foundation/branded-types';
|
|
377
|
-
import { padArrayEnd } from '@aztec/foundation/collection';
|
|
378
|
-
import { Fr } from '@aztec/foundation/curves/bn254';
|
|
379
375
|
import { createLogger } from '@aztec/foundation/log';
|
|
380
|
-
import {
|
|
376
|
+
import { sleep } from '@aztec/foundation/sleep';
|
|
381
377
|
import { Timer } from '@aztec/foundation/timer';
|
|
382
|
-
import {
|
|
383
|
-
import { protocolContractsHash } from '@aztec/protocol-contracts';
|
|
384
|
-
import { buildFinalBlobChallenges } from '@aztec/prover-client/helpers';
|
|
385
|
-
import { PublicSimulatorConfig } from '@aztec/stdlib/avm';
|
|
378
|
+
import { TopTreeCancelledError } from '@aztec/prover-client/orchestrator';
|
|
386
379
|
import { EpochProvingJobTerminalState } from '@aztec/stdlib/interfaces/server';
|
|
387
|
-
import { CheckpointConstantData } from '@aztec/stdlib/rollup';
|
|
388
|
-
import { MerkleTreeId } from '@aztec/stdlib/trees';
|
|
389
380
|
import { Attributes, trackSpan } from '@aztec/telemetry-client';
|
|
390
381
|
import * as crypto from 'node:crypto';
|
|
391
|
-
import {
|
|
392
|
-
|
|
382
|
+
import { TopTreeJob } from './top-tree-job.js';
|
|
383
|
+
/** Stable string key for use in maps. */ export function specKey(spec) {
|
|
384
|
+
return `${spec.kind}:${spec.epochNumber}:${spec.fromSlot}-${spec.toSlot}`;
|
|
385
|
+
}
|
|
386
|
+
_dec = trackSpan('EpochSession.start', function() {
|
|
393
387
|
return {
|
|
394
|
-
[Attributes.EPOCH_NUMBER]: this.
|
|
388
|
+
[Attributes.EPOCH_NUMBER]: this.spec.epochNumber
|
|
395
389
|
};
|
|
396
390
|
});
|
|
397
391
|
/**
|
|
398
|
-
*
|
|
399
|
-
*
|
|
400
|
-
*
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
392
|
+
* One attempt at proving and publishing a contiguous slot range. The `SessionSpec` and
|
|
393
|
+
* the prover set are both frozen at construction time; the session does not adapt to
|
|
394
|
+
* reorgs or extensions of canonical content. Instead, `SessionManager` owns the
|
|
395
|
+
* reconciliation loop and replaces invalidated sessions wholesale (cancel + construct
|
|
396
|
+
* a fresh session with the new prover set).
|
|
397
|
+
*
|
|
398
|
+
* Each session does three things in sequence:
|
|
399
|
+
*
|
|
400
|
+
* 1. Run a `TopTreeJob` over its frozen prover subset to produce the epoch proof.
|
|
401
|
+
* 2. Hand the proof to the shared `ProofPublishingService` as a `PublishCandidate`.
|
|
402
|
+
* 3. Translate the service's outcome into a terminal session state.
|
|
403
|
+
*
|
|
404
|
+
* Everything to do with submission — predecessor gating, same-epoch dedup, deadline
|
|
405
|
+
* enforcement, and the L1 transaction itself — is the publishing service's concern.
|
|
406
|
+
* The session is just the producer of one candidate and the observer of its outcome.
|
|
407
|
+
*
|
|
408
|
+
* Lifecycle (happy path):
|
|
409
|
+
*
|
|
410
|
+
* initialized → awaiting-checkpoints → awaiting-root → publishing-proof → completed
|
|
411
|
+
*
|
|
412
|
+
* Terminal states map the publishing outcome: `published` → `completed`, `superseded` →
|
|
413
|
+
* `superseded`, `failed` → `failed`, `expired` → `timed-out`, `withdrawn` → `cancelled`.
|
|
414
|
+
* Additionally, the session-level deadline fires `cancel('deadline')` and transitions
|
|
415
|
+
* to `timed-out` for the pre-submit window (top-tree proving) — the publishing service
|
|
416
|
+
* handles the post-submit window via the candidate's `deadline`.
|
|
417
|
+
*
|
|
418
|
+
* `cancel()` is idempotent. It marks the session terminal, calls
|
|
419
|
+
* `publishingService.withdraw(uuid)` to drop any queued candidate (an in-flight publish
|
|
420
|
+
* runs to natural completion; the session has already settled), and tears down the
|
|
421
|
+
* top-tree job if proving is still in progress.
|
|
422
|
+
*/ export class EpochSession {
|
|
423
|
+
spec;
|
|
424
|
+
deps;
|
|
411
425
|
static{
|
|
412
426
|
({ e: [_initProto] } = _apply_decs_2203_r(this, [
|
|
413
427
|
[
|
|
414
428
|
_dec,
|
|
415
429
|
2,
|
|
416
|
-
"
|
|
430
|
+
"start"
|
|
417
431
|
]
|
|
418
432
|
], []));
|
|
419
433
|
}
|
|
420
|
-
|
|
421
|
-
log;
|
|
434
|
+
tracer;
|
|
422
435
|
uuid;
|
|
423
|
-
|
|
424
|
-
|
|
436
|
+
log;
|
|
437
|
+
state;
|
|
425
438
|
deadlineTimeoutHandler;
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
this.
|
|
433
|
-
this.
|
|
434
|
-
this.metrics = metrics;
|
|
435
|
-
this.deadline = deadline;
|
|
436
|
-
this.config = config;
|
|
439
|
+
topTreeJob;
|
|
440
|
+
/** Cancelled top-tree jobs whose teardown is still in flight. Awaited at session stop. */ pendingTopTreeCleanups;
|
|
441
|
+
completionPromise;
|
|
442
|
+
resolveCompletion;
|
|
443
|
+
/** Stable reference; never mutated after construction. */ checkpoints;
|
|
444
|
+
constructor(spec, checkpoints, deps){
|
|
445
|
+
this.spec = spec;
|
|
446
|
+
this.deps = deps;
|
|
437
447
|
this.state = (_initProto(this), 'initialized');
|
|
438
|
-
|
|
448
|
+
this.pendingTopTreeCleanups = [];
|
|
449
|
+
if (checkpoints.length === 0) {
|
|
450
|
+
throw new Error(`Cannot construct EpochSession for ${specKey(spec)}: empty checkpoint set`);
|
|
451
|
+
}
|
|
452
|
+
this.checkpoints = [
|
|
453
|
+
...checkpoints
|
|
454
|
+
];
|
|
439
455
|
this.uuid = crypto.randomUUID();
|
|
440
|
-
this.log = createLogger('prover-node:epoch-
|
|
441
|
-
...bindings,
|
|
442
|
-
instanceId: `
|
|
456
|
+
this.log = createLogger('prover-node:epoch-session', {
|
|
457
|
+
...deps.bindings,
|
|
458
|
+
instanceId: `session-${spec.kind}-${spec.epochNumber}-${spec.fromSlot}-${spec.toSlot}`
|
|
459
|
+
});
|
|
460
|
+
this.tracer = deps.metrics.tracer;
|
|
461
|
+
this.completionPromise = new Promise((resolve)=>{
|
|
462
|
+
this.resolveCompletion = resolve;
|
|
463
|
+
});
|
|
464
|
+
this.scheduleDeadlineStop();
|
|
465
|
+
this.log.info(`Created EpochSession ${this.uuid}`, {
|
|
466
|
+
uuid: this.uuid,
|
|
467
|
+
...spec,
|
|
468
|
+
checkpointCount: this.checkpoints.length,
|
|
469
|
+
checkpointIds: this.checkpoints.map((c)=>c.id)
|
|
443
470
|
});
|
|
444
|
-
this.tracer = metrics.tracer;
|
|
445
471
|
}
|
|
446
472
|
getId() {
|
|
447
473
|
return this.uuid;
|
|
448
474
|
}
|
|
475
|
+
getSpec() {
|
|
476
|
+
return this.spec;
|
|
477
|
+
}
|
|
449
478
|
getState() {
|
|
450
479
|
return this.state;
|
|
451
480
|
}
|
|
452
481
|
getEpochNumber() {
|
|
453
|
-
return this.
|
|
482
|
+
return this.spec.epochNumber;
|
|
483
|
+
}
|
|
484
|
+
getKind() {
|
|
485
|
+
return this.spec.kind;
|
|
454
486
|
}
|
|
455
487
|
getDeadline() {
|
|
456
|
-
return this.deadline;
|
|
488
|
+
return this.deps.deadline;
|
|
457
489
|
}
|
|
458
|
-
|
|
459
|
-
return this.
|
|
490
|
+
getCheckpoints() {
|
|
491
|
+
return this.checkpoints;
|
|
460
492
|
}
|
|
461
|
-
|
|
462
|
-
return this.
|
|
493
|
+
/** Resolves when the session reaches a terminal state. */ whenDone() {
|
|
494
|
+
return this.completionPromise;
|
|
463
495
|
}
|
|
464
|
-
|
|
465
|
-
return this.
|
|
496
|
+
/** True if the session is in a terminal state. */ isTerminal() {
|
|
497
|
+
return EpochProvingJobTerminalState.includes(this.state);
|
|
466
498
|
}
|
|
467
|
-
|
|
468
|
-
return this.
|
|
499
|
+
/** First block this session proves. */ getStartBlockNumber() {
|
|
500
|
+
return BlockNumber(this.checkpoints[0].checkpoint.blocks[0].number);
|
|
469
501
|
}
|
|
470
|
-
|
|
471
|
-
|
|
502
|
+
/** Last block this session proves. */ getEndBlockNumber() {
|
|
503
|
+
const lastCheckpoint = this.checkpoints[this.checkpoints.length - 1];
|
|
504
|
+
return BlockNumber(lastCheckpoint.checkpoint.blocks[lastCheckpoint.checkpoint.blocks.length - 1].number);
|
|
472
505
|
}
|
|
473
506
|
/**
|
|
474
|
-
*
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
if (!this.config.skipEpochCheck) {
|
|
478
|
-
await this.scheduleEpochCheck();
|
|
479
|
-
}
|
|
480
|
-
const attestations = this.attestations.map((attestation)=>attestation.toViem());
|
|
481
|
-
const epochNumber = this.epochNumber;
|
|
482
|
-
const epochSizeCheckpoints = this.checkpoints.length;
|
|
483
|
-
const epochSizeBlocks = this.checkpoints.reduce((accum, checkpoint)=>accum + checkpoint.blocks.length, 0);
|
|
484
|
-
const epochSizeTxs = this.checkpoints.reduce((accum, checkpoint)=>accum + checkpoint.blocks.reduce((accumC, block)=>accumC + block.body.txEffects.length, 0), 0);
|
|
485
|
-
const fromCheckpoint = this.checkpoints[0].number;
|
|
486
|
-
const toCheckpoint = this.checkpoints.at(-1).number;
|
|
487
|
-
const fromBlock = this.checkpoints[0].blocks[0].number;
|
|
488
|
-
const toBlock = this.checkpoints.at(-1).blocks.at(-1).number;
|
|
489
|
-
this.log.info(`Starting epoch ${epochNumber} proving job with checkpoints ${fromCheckpoint} to ${toCheckpoint}`, {
|
|
490
|
-
fromBlock,
|
|
491
|
-
toBlock,
|
|
492
|
-
epochSizeTxs,
|
|
493
|
-
epochNumber,
|
|
494
|
-
uuid: this.uuid
|
|
495
|
-
});
|
|
496
|
-
this.progressState('processing');
|
|
497
|
-
const timer = new Timer();
|
|
498
|
-
const { promise, resolve } = promiseWithResolvers();
|
|
499
|
-
this.runPromise = promise;
|
|
507
|
+
* Kicks off proving + submission. Fires and forgets — callers should await `whenDone()`.
|
|
508
|
+
* Returns a promise that resolves to the final state for callers that want to wait inline.
|
|
509
|
+
*/ async start() {
|
|
500
510
|
try {
|
|
501
|
-
|
|
502
|
-
const finalBlobBatchingChallenges = await buildFinalBlobChallenges(blobFieldsPerCheckpoint);
|
|
503
|
-
this.prover.startNewEpoch(epochNumber, epochSizeCheckpoints, finalBlobBatchingChallenges);
|
|
504
|
-
await this.prover.startChonkVerifierCircuits(Array.from(this.txs.values()));
|
|
505
|
-
// Everything in the epoch should have the same chainId and version.
|
|
506
|
-
const { chainId, version } = this.checkpoints[0].blocks[0].header.globalVariables;
|
|
507
|
-
const previousBlockHeaders = this.gatherPreviousBlockHeaders();
|
|
508
|
-
await asyncPool(this.config.parallelBlockLimit ?? 32, this.checkpoints, async (checkpoint)=>{
|
|
509
|
-
this.checkState();
|
|
510
|
-
const checkpointIndex = checkpoint.number - fromCheckpoint;
|
|
511
|
-
const checkpointConstants = CheckpointConstantData.from({
|
|
512
|
-
chainId,
|
|
513
|
-
version,
|
|
514
|
-
vkTreeRoot: getVKTreeRoot(),
|
|
515
|
-
protocolContractsHash: protocolContractsHash,
|
|
516
|
-
proverId: this.prover.getProverId().toField(),
|
|
517
|
-
slotNumber: checkpoint.header.slotNumber,
|
|
518
|
-
coinbase: checkpoint.header.coinbase,
|
|
519
|
-
feeRecipient: checkpoint.header.feeRecipient,
|
|
520
|
-
gasFees: checkpoint.header.gasFees
|
|
521
|
-
});
|
|
522
|
-
const previousHeader = previousBlockHeaders[checkpointIndex];
|
|
523
|
-
const l1ToL2Messages = this.getL1ToL2Messages(checkpoint);
|
|
524
|
-
this.log.verbose(`Starting processing checkpoint ${checkpoint.number}`, {
|
|
525
|
-
number: checkpoint.number,
|
|
526
|
-
checkpointHash: checkpoint.hash().toString(),
|
|
527
|
-
lastArchive: checkpoint.header.lastArchiveRoot,
|
|
528
|
-
previousHeader: previousHeader.hash(),
|
|
529
|
-
uuid: this.uuid
|
|
530
|
-
});
|
|
531
|
-
await this.prover.startNewCheckpoint(checkpointIndex, checkpointConstants, l1ToL2Messages, checkpoint.blocks.length, previousHeader);
|
|
532
|
-
for (const block of checkpoint.blocks){
|
|
533
|
-
const globalVariables = block.header.globalVariables;
|
|
534
|
-
const txs = this.getTxs(block);
|
|
535
|
-
this.log.verbose(`Starting processing block ${block.number}`, {
|
|
536
|
-
number: block.number,
|
|
537
|
-
blockHash: (await block.hash()).toString(),
|
|
538
|
-
lastArchive: block.header.lastArchive.root,
|
|
539
|
-
noteHashTreeRoot: block.header.state.partial.noteHashTree.root,
|
|
540
|
-
nullifierTreeRoot: block.header.state.partial.nullifierTree.root,
|
|
541
|
-
publicDataTreeRoot: block.header.state.partial.publicDataTree.root,
|
|
542
|
-
...globalVariables,
|
|
543
|
-
numTxs: txs.length
|
|
544
|
-
});
|
|
545
|
-
// Start block proving
|
|
546
|
-
await this.prover.startNewBlock(block.number, globalVariables.timestamp, txs.length);
|
|
547
|
-
// Process public fns
|
|
548
|
-
const db = await this.createFork(BlockNumber(block.number - 1), l1ToL2Messages);
|
|
549
|
-
const config = PublicSimulatorConfig.from({
|
|
550
|
-
proverId: this.prover.getProverId().toField(),
|
|
551
|
-
skipFeeEnforcement: false,
|
|
552
|
-
collectDebugLogs: false,
|
|
553
|
-
collectHints: true,
|
|
554
|
-
collectPublicInputs: true,
|
|
555
|
-
collectStatistics: false
|
|
556
|
-
});
|
|
557
|
-
const publicProcessor = this.publicProcessorFactory.create(db, globalVariables, config);
|
|
558
|
-
const processed = await this.processTxs(publicProcessor, txs);
|
|
559
|
-
await this.prover.addTxs(processed);
|
|
560
|
-
await db.close();
|
|
561
|
-
this.log.verbose(`Processed all ${txs.length} txs for block ${block.number}`, {
|
|
562
|
-
blockNumber: block.number,
|
|
563
|
-
blockHash: (await block.hash()).toString(),
|
|
564
|
-
uuid: this.uuid
|
|
565
|
-
});
|
|
566
|
-
// Mark block as completed to pad it
|
|
567
|
-
const expectedBlockHeader = block.header;
|
|
568
|
-
await this.prover.setBlockCompleted(block.number, expectedBlockHeader);
|
|
569
|
-
}
|
|
570
|
-
});
|
|
571
|
-
const executionTime = timer.ms();
|
|
572
|
-
this.progressState('awaiting-prover');
|
|
573
|
-
const { publicInputs, proof, batchedBlobInputs } = await this.prover.finalizeEpoch();
|
|
574
|
-
this.log.info(`Finalized proof for epoch ${epochNumber}`, {
|
|
575
|
-
epochNumber,
|
|
576
|
-
uuid: this.uuid,
|
|
577
|
-
duration: timer.ms()
|
|
578
|
-
});
|
|
579
|
-
this.progressState('publishing-proof');
|
|
580
|
-
if (this.config.skipSubmitProof) {
|
|
581
|
-
this.log.info(`Proof publishing is disabled. Dropping valid proof for epoch ${epochNumber} (checkpoints ${fromCheckpoint} to ${toCheckpoint})`);
|
|
582
|
-
this.state = 'completed';
|
|
583
|
-
this.metrics.recordProvingJob(executionTime, timer.ms(), epochSizeCheckpoints, epochSizeBlocks, epochSizeTxs);
|
|
584
|
-
return;
|
|
585
|
-
}
|
|
586
|
-
const success = await this.publisher.submitEpochProof({
|
|
587
|
-
fromCheckpoint,
|
|
588
|
-
toCheckpoint,
|
|
589
|
-
epochNumber,
|
|
590
|
-
publicInputs,
|
|
591
|
-
proof,
|
|
592
|
-
batchedBlobInputs,
|
|
593
|
-
attestations
|
|
594
|
-
});
|
|
595
|
-
if (!success) {
|
|
596
|
-
throw new Error('Failed to submit epoch proof to L1');
|
|
597
|
-
}
|
|
598
|
-
this.log.info(`Submitted proof for epoch ${epochNumber} (checkpoints ${fromCheckpoint} to ${toCheckpoint})`, {
|
|
599
|
-
epochNumber,
|
|
600
|
-
uuid: this.uuid
|
|
601
|
-
});
|
|
602
|
-
this.state = 'completed';
|
|
603
|
-
this.metrics.recordProvingJob(executionTime, timer.ms(), epochSizeCheckpoints, epochSizeBlocks, epochSizeTxs);
|
|
511
|
+
await this.run();
|
|
604
512
|
} catch (err) {
|
|
605
|
-
|
|
606
|
-
this.log.warn(`Halted execution of epoch ${epochNumber} prover job`, {
|
|
607
|
-
uuid: this.uuid,
|
|
608
|
-
epochNumber,
|
|
609
|
-
details: err.message
|
|
610
|
-
});
|
|
611
|
-
return;
|
|
612
|
-
}
|
|
613
|
-
this.log.error(`Error running epoch ${epochNumber} prover job`, err, {
|
|
513
|
+
this.log.error(`Error in EpochSession ${this.uuid}`, err, {
|
|
614
514
|
uuid: this.uuid,
|
|
615
|
-
|
|
515
|
+
...this.spec
|
|
616
516
|
});
|
|
617
|
-
if (this.
|
|
517
|
+
if (!this.isTerminal()) {
|
|
618
518
|
this.state = 'failed';
|
|
619
519
|
}
|
|
620
520
|
} finally{
|
|
621
521
|
clearTimeout(this.deadlineTimeoutHandler);
|
|
622
|
-
await this.
|
|
623
|
-
|
|
624
|
-
resolve();
|
|
522
|
+
await this.teardownTopTreeIfNeeded();
|
|
523
|
+
this.resolveCompletion(this.state);
|
|
625
524
|
}
|
|
525
|
+
return this.state;
|
|
626
526
|
}
|
|
627
527
|
/**
|
|
628
|
-
*
|
|
629
|
-
*
|
|
630
|
-
*/ async
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
528
|
+
* Cancels the session. Idempotent. Withdraws any submitted candidate from the
|
|
529
|
+
* publishing service so the in-flight publisher (if any) is interrupted.
|
|
530
|
+
*/ async cancel(reason = 'cancelled', { abortJobs = true } = {}) {
|
|
531
|
+
if (this.isTerminal()) {
|
|
532
|
+
return;
|
|
533
|
+
}
|
|
534
|
+
this.log.info(`Cancelling EpochSession ${this.uuid}: ${reason}`, {
|
|
535
|
+
uuid: this.uuid,
|
|
536
|
+
...this.spec,
|
|
537
|
+
previousState: this.state,
|
|
538
|
+
reason
|
|
636
539
|
});
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
this.state = state;
|
|
643
|
-
}
|
|
644
|
-
checkState() {
|
|
645
|
-
if (this.state === 'timed-out' || this.state === 'stopped' || this.state === 'failed' || this.state === 'reorg') {
|
|
646
|
-
throw new HaltExecutionError(this.state);
|
|
540
|
+
this.state = 'cancelled';
|
|
541
|
+
try {
|
|
542
|
+
this.deps.publishingService.withdraw(this.uuid);
|
|
543
|
+
} catch (err) {
|
|
544
|
+
this.log.error(`Error withdrawing candidate from publishing service`, err);
|
|
647
545
|
}
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
546
|
+
if (this.topTreeJob && !this.topTreeJob.isCancelled()) {
|
|
547
|
+
const job = this.topTreeJob;
|
|
548
|
+
this.topTreeJob = undefined;
|
|
549
|
+
// On a clean shutdown we leave the in-flight broker jobs alone so a restart can reuse them;
|
|
550
|
+
// other cancellations (reorg, supersede, deadline) abort them since their inputs are stale.
|
|
551
|
+
job.cancel(abortJobs);
|
|
552
|
+
this.pendingTopTreeCleanups.push(job);
|
|
654
553
|
}
|
|
554
|
+
await this.teardownTopTreeIfNeeded();
|
|
555
|
+
this.resolveCompletion(this.state);
|
|
655
556
|
}
|
|
656
|
-
|
|
657
|
-
const
|
|
658
|
-
if (
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
557
|
+
async run() {
|
|
558
|
+
const timer = new Timer();
|
|
559
|
+
if (this.deps.config.finalizationDelayMs && this.deps.config.finalizationDelayMs > 0) {
|
|
560
|
+
this.log.warn(`Waiting ${this.deps.config.finalizationDelayMs}ms before starting top-tree proving`, {
|
|
561
|
+
uuid: this.uuid,
|
|
562
|
+
...this.spec
|
|
563
|
+
});
|
|
564
|
+
await sleep(this.deps.config.finalizationDelayMs);
|
|
565
|
+
if (this.isTerminal()) {
|
|
566
|
+
return;
|
|
662
567
|
}
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
568
|
+
}
|
|
569
|
+
// Stage 1 — top-tree proving.
|
|
570
|
+
const topTreeJob = new TopTreeJob(this.spec.epochNumber, this.checkpoints, {
|
|
571
|
+
proverFactory: this.deps.proverFactory,
|
|
572
|
+
metrics: this.deps.metrics,
|
|
573
|
+
log: this.log,
|
|
574
|
+
hooks: this.toTopTreeHooks()
|
|
575
|
+
});
|
|
576
|
+
this.topTreeJob = topTreeJob;
|
|
577
|
+
const { fromCheckpoint, toCheckpoint, count } = topTreeJob.getRange();
|
|
578
|
+
this.state = 'awaiting-checkpoints';
|
|
579
|
+
let proof;
|
|
580
|
+
try {
|
|
581
|
+
proof = await topTreeJob.start();
|
|
582
|
+
} catch (err) {
|
|
583
|
+
if (err instanceof TopTreeCancelledError) {
|
|
584
|
+
// Session cancel kicked off the underlying teardown; nothing more to do here.
|
|
585
|
+
this.log.info(`Top-tree cancelled for EpochSession ${this.uuid}`, {
|
|
668
586
|
uuid: this.uuid,
|
|
669
|
-
|
|
587
|
+
...this.spec
|
|
670
588
|
});
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
epochNumber: this.epochNumber
|
|
675
|
-
});
|
|
676
|
-
});
|
|
677
|
-
}, timeout);
|
|
589
|
+
return;
|
|
590
|
+
}
|
|
591
|
+
throw err;
|
|
678
592
|
}
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
593
|
+
this.topTreeJob = undefined;
|
|
594
|
+
this.log.info(`Top-tree proof ready for EpochSession ${this.uuid}`, {
|
|
595
|
+
uuid: this.uuid,
|
|
596
|
+
...this.spec,
|
|
597
|
+
fromCheckpoint,
|
|
598
|
+
toCheckpoint,
|
|
599
|
+
durationMs: timer.ms()
|
|
600
|
+
});
|
|
601
|
+
// Stage 2 — hand the proof to the publishing service and wait for its verdict.
|
|
602
|
+
await this.submitProof(proof, fromCheckpoint, toCheckpoint, count, timer);
|
|
603
|
+
}
|
|
604
|
+
async submitProof(proof, fromCheckpoint, toCheckpoint, checkpointCount, timer) {
|
|
605
|
+
// Attestations come from the highest-numbered registered checkpoint — that's the one
|
|
606
|
+
// whose attestations the L1 contract checks for the proven range.
|
|
607
|
+
const lastCheckpoint = this.checkpoints[this.checkpoints.length - 1];
|
|
608
|
+
const attestations = lastCheckpoint.attestations.map((a)=>a.toViem());
|
|
609
|
+
const epochSizeBlocks = this.checkpoints.reduce((acc, c)=>acc + c.checkpoint.blocks.length, 0);
|
|
610
|
+
const epochSizeTxs = this.checkpoints.reduce((acc, c)=>acc + c.checkpoint.blocks.reduce((bAcc, block)=>bAcc + block.body.txEffects.length, 0), 0);
|
|
611
|
+
// Reflect the publish phase. Guard against a terminal state set concurrently by cancel() — the
|
|
612
|
+
// post-submit isTerminal() check below relies on cancel still winning.
|
|
613
|
+
if (!this.isTerminal()) {
|
|
614
|
+
this.state = 'publishing-proof';
|
|
615
|
+
}
|
|
616
|
+
const outcome = await this.deps.publishingService.submit({
|
|
617
|
+
id: this.uuid,
|
|
618
|
+
epoch: this.spec.epochNumber,
|
|
619
|
+
kind: this.spec.kind,
|
|
620
|
+
startBlock: this.getStartBlockNumber(),
|
|
621
|
+
endBlock: this.getEndBlockNumber(),
|
|
622
|
+
deadline: this.deps.deadline,
|
|
623
|
+
fromCheckpoint,
|
|
624
|
+
toCheckpoint,
|
|
625
|
+
publicInputs: proof.publicInputs,
|
|
626
|
+
proof: proof.proof,
|
|
627
|
+
batchedBlobInputs: proof.batchedBlobInputs,
|
|
628
|
+
attestations,
|
|
629
|
+
headers: this.checkpoints.map((c)=>c.checkpoint.header)
|
|
630
|
+
});
|
|
631
|
+
if (this.isTerminal()) {
|
|
632
|
+
// cancel() already set the terminal state — don't clobber it.
|
|
687
633
|
return;
|
|
688
634
|
}
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
const blockHashes = await Promise.all(blockHeaders.map((header)=>header.hash()));
|
|
693
|
-
const thisBlocks = this.checkpoints.flatMap((checkpoint)=>checkpoint.blocks);
|
|
694
|
-
const thisBlockHashes = await Promise.all(thisBlocks.map((block)=>block.hash()));
|
|
695
|
-
if (blockHeaders.length !== thisBlocks.length || !blockHashes.every((block, i)=>block.equals(thisBlockHashes[i]))) {
|
|
696
|
-
this.log.warn('Epoch blocks changed underfoot', {
|
|
635
|
+
switch(outcome){
|
|
636
|
+
case 'published':
|
|
637
|
+
this.log.info(`Submitted proof for epoch ${this.spec.epochNumber} (checkpoints ${fromCheckpoint}..${toCheckpoint})`, {
|
|
697
638
|
uuid: this.uuid,
|
|
698
|
-
|
|
699
|
-
oldBlockHashes: thisBlockHashes,
|
|
700
|
-
newBlockHashes: blockHashes
|
|
639
|
+
...this.spec
|
|
701
640
|
});
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
641
|
+
this.state = 'completed';
|
|
642
|
+
this.deps.metrics.recordProvingJob(timer.ms(), checkpointCount, epochSizeBlocks, epochSizeTxs);
|
|
643
|
+
return;
|
|
644
|
+
case 'superseded':
|
|
645
|
+
this.log.info(`EpochSession ${this.uuid} superseded by a longer candidate`, {
|
|
646
|
+
uuid: this.uuid,
|
|
647
|
+
...this.spec
|
|
648
|
+
});
|
|
649
|
+
this.state = 'superseded';
|
|
650
|
+
return;
|
|
651
|
+
case 'withdrawn':
|
|
652
|
+
// cancel() ran but the terminal-state check above missed it. Defensive: treat as cancelled.
|
|
653
|
+
this.state = 'cancelled';
|
|
654
|
+
return;
|
|
655
|
+
case 'expired':
|
|
656
|
+
this.log.warn(`EpochSession ${this.uuid} expired before publishing`, {
|
|
657
|
+
uuid: this.uuid,
|
|
658
|
+
...this.spec
|
|
659
|
+
});
|
|
660
|
+
this.state = 'timed-out';
|
|
661
|
+
return;
|
|
662
|
+
case 'failed':
|
|
663
|
+
throw new Error('Failed to submit epoch proof to L1');
|
|
664
|
+
}
|
|
713
665
|
}
|
|
714
|
-
|
|
715
|
-
|
|
666
|
+
async teardownTopTreeIfNeeded() {
|
|
667
|
+
if (this.topTreeJob) {
|
|
668
|
+
const job = this.topTreeJob;
|
|
669
|
+
this.topTreeJob = undefined;
|
|
670
|
+
job.cancel();
|
|
671
|
+
this.pendingTopTreeCleanups.push(job);
|
|
672
|
+
}
|
|
673
|
+
if (this.pendingTopTreeCleanups.length > 0) {
|
|
674
|
+
await Promise.allSettled(this.pendingTopTreeCleanups.map((j)=>j.whenDone()));
|
|
675
|
+
this.pendingTopTreeCleanups.length = 0;
|
|
676
|
+
}
|
|
716
677
|
}
|
|
717
|
-
|
|
718
|
-
|
|
678
|
+
scheduleDeadlineStop() {
|
|
679
|
+
const deadline = this.deps.deadline;
|
|
680
|
+
if (!deadline) {
|
|
681
|
+
return;
|
|
682
|
+
}
|
|
683
|
+
const timeout = Math.max(deadline.getTime() - this.deps.dateProvider.now(), 0);
|
|
684
|
+
this.deadlineTimeoutHandler = setTimeout(()=>{
|
|
685
|
+
void this.handleDeadline();
|
|
686
|
+
}, timeout);
|
|
719
687
|
}
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
if (
|
|
726
|
-
|
|
727
|
-
throw new Error(`Txs failed processing: ${failedTxs.map(({ error }, index)=>`${failedTxHashes[index]} (${error})`).join(', ')}`);
|
|
688
|
+
/**
|
|
689
|
+
* Returns a promise that resolves once cancellation has propagated and the state has
|
|
690
|
+
* been flipped from 'cancelled' to 'timed-out'. Protected so unit tests can drive the
|
|
691
|
+
* deadline path without waiting on the real `setTimeout` to fire.
|
|
692
|
+
*/ async handleDeadline() {
|
|
693
|
+
if (this.isTerminal()) {
|
|
694
|
+
return;
|
|
728
695
|
}
|
|
729
|
-
|
|
730
|
-
|
|
696
|
+
this.log.warn(`EpochSession ${this.uuid} hit deadline`, {
|
|
697
|
+
uuid: this.uuid,
|
|
698
|
+
...this.spec
|
|
699
|
+
});
|
|
700
|
+
await this.cancel('deadline');
|
|
701
|
+
// After cancel, override state if it was the canonical timeout case so observers see 'timed-out'.
|
|
702
|
+
if (this.state === 'cancelled') {
|
|
703
|
+
this.state = 'timed-out';
|
|
731
704
|
}
|
|
732
|
-
return processedTxs;
|
|
733
705
|
}
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
706
|
+
toTopTreeHooks() {
|
|
707
|
+
const hooks = this.deps.hooks;
|
|
708
|
+
return {
|
|
709
|
+
// `beforeProve` fires once the sub-tree (checkpoint block) proofs are ready and the root prove is
|
|
710
|
+
// about to start — the boundary between `awaiting-checkpoints` and proving the top tree. Don't
|
|
711
|
+
// clobber a terminal state set concurrently by cancel().
|
|
712
|
+
beforeProve: async ()=>{
|
|
713
|
+
if (!this.isTerminal()) {
|
|
714
|
+
this.state = 'awaiting-root';
|
|
715
|
+
}
|
|
716
|
+
await hooks?.beforeTopTreeProve?.();
|
|
717
|
+
},
|
|
718
|
+
afterProve: hooks?.afterTopTreeProve,
|
|
719
|
+
proveOverride: hooks?.topTreeProveOverride
|
|
720
|
+
};
|
|
740
721
|
}
|
|
741
722
|
}
|