@aztec/prover-node 0.0.1-commit.3469e52 → 0.0.1-commit.350e0a4d
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +518 -0
- package/dest/actions/download-epoch-proving-job.js +1 -1
- package/dest/actions/rerun-epoch-proving-job.d.ts +6 -4
- package/dest/actions/rerun-epoch-proving-job.d.ts.map +1 -1
- package/dest/actions/rerun-epoch-proving-job.js +187 -27
- package/dest/actions/upload-epoch-proof-failure.d.ts +2 -2
- package/dest/actions/upload-epoch-proof-failure.d.ts.map +1 -1
- package/dest/bin/run-failed-epoch.js +6 -5
- package/dest/checkpoint-store.d.ts +87 -0
- package/dest/checkpoint-store.d.ts.map +1 -0
- package/dest/checkpoint-store.js +186 -0
- package/dest/config.d.ts +7 -8
- package/dest/config.d.ts.map +1 -1
- package/dest/config.js +24 -20
- package/dest/factory.d.ts +19 -13
- package/dest/factory.d.ts.map +1 -1
- package/dest/factory.js +41 -69
- package/dest/index.d.ts +2 -1
- package/dest/index.d.ts.map +1 -1
- package/dest/index.js +1 -0
- package/dest/job/checkpoint-prover.d.ts +134 -0
- package/dest/job/checkpoint-prover.d.ts.map +1 -0
- package/dest/job/checkpoint-prover.js +353 -0
- package/dest/job/epoch-session.d.ts +146 -0
- package/dest/job/epoch-session.d.ts.map +1 -0
- package/dest/job/{epoch-proving-job.js → epoch-session.js} +277 -295
- package/dest/job/top-tree-job.d.ts +82 -0
- package/dest/job/top-tree-job.d.ts.map +1 -0
- package/dest/job/top-tree-job.js +152 -0
- package/dest/metrics.d.ts +40 -3
- package/dest/metrics.d.ts.map +1 -1
- package/dest/metrics.js +112 -7
- package/dest/monitors/epoch-monitor.d.ts +1 -1
- package/dest/monitors/epoch-monitor.d.ts.map +1 -1
- package/dest/monitors/epoch-monitor.js +11 -9
- package/dest/proof-publishing-service.d.ts +161 -0
- package/dest/proof-publishing-service.d.ts.map +1 -0
- package/dest/proof-publishing-service.js +335 -0
- package/dest/prover-node-publisher.d.ts +28 -17
- package/dest/prover-node-publisher.d.ts.map +1 -1
- package/dest/prover-node-publisher.js +205 -65
- package/dest/prover-node.d.ts +118 -70
- package/dest/prover-node.d.ts.map +1 -1
- package/dest/prover-node.js +489 -226
- package/dest/prover-publisher-factory.d.ts +9 -5
- package/dest/prover-publisher-factory.d.ts.map +1 -1
- package/dest/prover-publisher-factory.js +8 -5
- package/dest/session-manager.d.ts +158 -0
- package/dest/session-manager.d.ts.map +1 -0
- package/dest/session-manager.js +452 -0
- package/dest/test/index.d.ts +7 -6
- package/dest/test/index.d.ts.map +1 -1
- package/package.json +23 -22
- package/src/actions/download-epoch-proving-job.ts +1 -1
- package/src/actions/rerun-epoch-proving-job.ts +117 -29
- package/src/actions/upload-epoch-proof-failure.ts +1 -1
- package/src/bin/run-failed-epoch.ts +5 -3
- package/src/checkpoint-store.ts +218 -0
- package/src/config.ts +35 -32
- package/src/factory.ts +69 -112
- package/src/index.ts +1 -0
- package/src/job/checkpoint-prover.ts +468 -0
- package/src/job/epoch-session.ts +436 -0
- package/src/job/top-tree-job.ts +227 -0
- package/src/metrics.ts +129 -12
- package/src/monitors/epoch-monitor.ts +5 -6
- package/src/proof-publishing-service.ts +427 -0
- package/src/prover-node-publisher.ts +241 -81
- package/src/prover-node.ts +563 -254
- package/src/prover-publisher-factory.ts +19 -10
- package/src/session-manager.ts +552 -0
- package/src/test/index.ts +6 -6
- package/dest/job/epoch-proving-job.d.ts +0 -62
- package/dest/job/epoch-proving-job.d.ts.map +0 -1
- package/src/job/epoch-proving-job.ts +0 -427
|
@@ -371,368 +371,350 @@ function _apply_decs_2203_r(targetClass, memberDecs, classDecs, parentClass) {
|
|
|
371
371
|
return (_apply_decs_2203_r = applyDecs2203RFactory())(targetClass, memberDecs, classDecs, parentClass);
|
|
372
372
|
}
|
|
373
373
|
var _dec, _initProto;
|
|
374
|
-
import { NUMBER_OF_L1_L2_MESSAGES_PER_ROLLUP } from '@aztec/constants';
|
|
375
|
-
import { asyncPool } from '@aztec/foundation/async-pool';
|
|
376
374
|
import { BlockNumber } from '@aztec/foundation/branded-types';
|
|
377
|
-
import { padArrayEnd } from '@aztec/foundation/collection';
|
|
378
|
-
import { Fr } from '@aztec/foundation/curves/bn254';
|
|
379
375
|
import { createLogger } from '@aztec/foundation/log';
|
|
380
|
-
import {
|
|
376
|
+
import { sleep } from '@aztec/foundation/sleep';
|
|
381
377
|
import { Timer } from '@aztec/foundation/timer';
|
|
382
|
-
import {
|
|
383
|
-
import { protocolContractsHash } from '@aztec/protocol-contracts';
|
|
384
|
-
import { buildFinalBlobChallenges } from '@aztec/prover-client/helpers';
|
|
385
|
-
import { PublicSimulatorConfig } from '@aztec/stdlib/avm';
|
|
378
|
+
import { TopTreeCancelledError } from '@aztec/prover-client/orchestrator';
|
|
386
379
|
import { EpochProvingJobTerminalState } from '@aztec/stdlib/interfaces/server';
|
|
387
|
-
import { CheckpointConstantData } from '@aztec/stdlib/rollup';
|
|
388
|
-
import { MerkleTreeId } from '@aztec/stdlib/trees';
|
|
389
380
|
import { Attributes, trackSpan } from '@aztec/telemetry-client';
|
|
390
381
|
import * as crypto from 'node:crypto';
|
|
391
|
-
import {
|
|
392
|
-
|
|
382
|
+
import { TopTreeJob } from './top-tree-job.js';
|
|
383
|
+
/** Stable string key for use in maps. */ export function specKey(spec) {
|
|
384
|
+
return `${spec.kind}:${spec.epochNumber}:${spec.fromSlot}-${spec.toSlot}`;
|
|
385
|
+
}
|
|
386
|
+
_dec = trackSpan('EpochSession.start', function() {
|
|
393
387
|
return {
|
|
394
|
-
[Attributes.EPOCH_NUMBER]: this.
|
|
388
|
+
[Attributes.EPOCH_NUMBER]: this.spec.epochNumber
|
|
395
389
|
};
|
|
396
390
|
});
|
|
397
391
|
/**
|
|
398
|
-
*
|
|
399
|
-
*
|
|
400
|
-
*
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
392
|
+
* One attempt at proving and publishing a contiguous slot range. The `SessionSpec` and
|
|
393
|
+
* the prover set are both frozen at construction time; the session does not adapt to
|
|
394
|
+
* reorgs or extensions of canonical content. Instead, `SessionManager` owns the
|
|
395
|
+
* reconciliation loop and replaces invalidated sessions wholesale (cancel + construct
|
|
396
|
+
* a fresh session with the new prover set).
|
|
397
|
+
*
|
|
398
|
+
* Each session does three things in sequence:
|
|
399
|
+
*
|
|
400
|
+
* 1. Run a `TopTreeJob` over its frozen prover subset to produce the epoch proof.
|
|
401
|
+
* 2. Hand the proof to the shared `ProofPublishingService` as a `PublishCandidate`.
|
|
402
|
+
* 3. Translate the service's outcome into a terminal session state.
|
|
403
|
+
*
|
|
404
|
+
* Everything to do with submission — predecessor gating, same-epoch dedup, deadline
|
|
405
|
+
* enforcement, and the L1 transaction itself — is the publishing service's concern.
|
|
406
|
+
* The session is just the producer of one candidate and the observer of its outcome.
|
|
407
|
+
*
|
|
408
|
+
* Lifecycle (happy path):
|
|
409
|
+
*
|
|
410
|
+
* initialized → awaiting-checkpoints → awaiting-root → publishing-proof → completed
|
|
411
|
+
*
|
|
412
|
+
* Terminal states map the publishing outcome: `published` → `completed`, `superseded` →
|
|
413
|
+
* `superseded`, `failed` → `failed`, `expired` → `timed-out`, `withdrawn` → `cancelled`.
|
|
414
|
+
* Additionally, the session-level deadline fires `cancel('deadline')` and transitions
|
|
415
|
+
* to `timed-out` for the pre-submit window (top-tree proving) — the publishing service
|
|
416
|
+
* handles the post-submit window via the candidate's `deadline`.
|
|
417
|
+
*
|
|
418
|
+
* `cancel()` is idempotent. It marks the session terminal, calls
|
|
419
|
+
* `publishingService.withdraw(uuid)` to drop any queued candidate (an in-flight publish
|
|
420
|
+
* runs to natural completion; the session has already settled), and tears down the
|
|
421
|
+
* top-tree job if proving is still in progress.
|
|
422
|
+
*/ export class EpochSession {
|
|
423
|
+
spec;
|
|
424
|
+
deps;
|
|
411
425
|
static{
|
|
412
426
|
({ e: [_initProto] } = _apply_decs_2203_r(this, [
|
|
413
427
|
[
|
|
414
428
|
_dec,
|
|
415
429
|
2,
|
|
416
|
-
"
|
|
430
|
+
"start"
|
|
417
431
|
]
|
|
418
432
|
], []));
|
|
419
433
|
}
|
|
420
|
-
|
|
421
|
-
log;
|
|
434
|
+
tracer;
|
|
422
435
|
uuid;
|
|
423
|
-
|
|
424
|
-
|
|
436
|
+
log;
|
|
437
|
+
state;
|
|
425
438
|
deadlineTimeoutHandler;
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
this.
|
|
433
|
-
this.
|
|
434
|
-
this.metrics = metrics;
|
|
435
|
-
this.deadline = deadline;
|
|
436
|
-
this.config = config;
|
|
439
|
+
topTreeJob;
|
|
440
|
+
/** Cancelled top-tree jobs whose teardown is still in flight. Awaited at session stop. */ pendingTopTreeCleanups;
|
|
441
|
+
completionPromise;
|
|
442
|
+
resolveCompletion;
|
|
443
|
+
/** Stable reference; never mutated after construction. */ checkpoints;
|
|
444
|
+
constructor(spec, checkpoints, deps){
|
|
445
|
+
this.spec = spec;
|
|
446
|
+
this.deps = deps;
|
|
437
447
|
this.state = (_initProto(this), 'initialized');
|
|
438
|
-
this.
|
|
439
|
-
|
|
448
|
+
this.pendingTopTreeCleanups = [];
|
|
449
|
+
if (checkpoints.length === 0) {
|
|
450
|
+
throw new Error(`Cannot construct EpochSession for ${specKey(spec)}: empty checkpoint set`);
|
|
451
|
+
}
|
|
452
|
+
this.checkpoints = [
|
|
453
|
+
...checkpoints
|
|
454
|
+
];
|
|
440
455
|
this.uuid = crypto.randomUUID();
|
|
441
|
-
this.
|
|
456
|
+
this.log = createLogger('prover-node:epoch-session', {
|
|
457
|
+
...deps.bindings,
|
|
458
|
+
instanceId: `session-${spec.kind}-${spec.epochNumber}-${spec.fromSlot}-${spec.toSlot}`
|
|
459
|
+
});
|
|
460
|
+
this.tracer = deps.metrics.tracer;
|
|
461
|
+
this.completionPromise = new Promise((resolve)=>{
|
|
462
|
+
this.resolveCompletion = resolve;
|
|
463
|
+
});
|
|
464
|
+
this.scheduleDeadlineStop();
|
|
465
|
+
this.log.info(`Created EpochSession ${this.uuid}`, {
|
|
466
|
+
uuid: this.uuid,
|
|
467
|
+
...spec,
|
|
468
|
+
checkpointCount: this.checkpoints.length,
|
|
469
|
+
checkpointIds: this.checkpoints.map((c)=>c.id)
|
|
470
|
+
});
|
|
442
471
|
}
|
|
443
472
|
getId() {
|
|
444
473
|
return this.uuid;
|
|
445
474
|
}
|
|
475
|
+
getSpec() {
|
|
476
|
+
return this.spec;
|
|
477
|
+
}
|
|
446
478
|
getState() {
|
|
447
479
|
return this.state;
|
|
448
480
|
}
|
|
449
481
|
getEpochNumber() {
|
|
450
|
-
return this.
|
|
482
|
+
return this.spec.epochNumber;
|
|
483
|
+
}
|
|
484
|
+
getKind() {
|
|
485
|
+
return this.spec.kind;
|
|
451
486
|
}
|
|
452
487
|
getDeadline() {
|
|
453
|
-
return this.deadline;
|
|
488
|
+
return this.deps.deadline;
|
|
454
489
|
}
|
|
455
|
-
|
|
456
|
-
return this.
|
|
490
|
+
getCheckpoints() {
|
|
491
|
+
return this.checkpoints;
|
|
457
492
|
}
|
|
458
|
-
|
|
459
|
-
return this.
|
|
493
|
+
/** Resolves when the session reaches a terminal state. */ whenDone() {
|
|
494
|
+
return this.completionPromise;
|
|
460
495
|
}
|
|
461
|
-
|
|
462
|
-
return this.
|
|
496
|
+
/** True if the session is in a terminal state. */ isTerminal() {
|
|
497
|
+
return EpochProvingJobTerminalState.includes(this.state);
|
|
463
498
|
}
|
|
464
|
-
|
|
465
|
-
return this.
|
|
499
|
+
/** First block this session proves. */ getStartBlockNumber() {
|
|
500
|
+
return BlockNumber(this.checkpoints[0].checkpoint.blocks[0].number);
|
|
466
501
|
}
|
|
467
|
-
|
|
468
|
-
|
|
502
|
+
/** Last block this session proves. */ getEndBlockNumber() {
|
|
503
|
+
const lastCheckpoint = this.checkpoints[this.checkpoints.length - 1];
|
|
504
|
+
return BlockNumber(lastCheckpoint.checkpoint.blocks[lastCheckpoint.checkpoint.blocks.length - 1].number);
|
|
469
505
|
}
|
|
470
506
|
/**
|
|
471
|
-
*
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
if (!this.config.skipEpochCheck) {
|
|
475
|
-
await this.scheduleEpochCheck();
|
|
476
|
-
}
|
|
477
|
-
const attestations = this.attestations.map((attestation)=>attestation.toViem());
|
|
478
|
-
const epochNumber = this.epochNumber;
|
|
479
|
-
const epochSizeCheckpoints = this.checkpoints.length;
|
|
480
|
-
const epochSizeBlocks = this.checkpoints.reduce((accum, checkpoint)=>accum + checkpoint.blocks.length, 0);
|
|
481
|
-
const epochSizeTxs = this.checkpoints.reduce((accum, checkpoint)=>accum + checkpoint.blocks.reduce((accumC, block)=>accumC + block.body.txEffects.length, 0), 0);
|
|
482
|
-
const fromCheckpoint = this.checkpoints[0].number;
|
|
483
|
-
const toCheckpoint = this.checkpoints.at(-1).number;
|
|
484
|
-
const fromBlock = this.checkpoints[0].blocks[0].number;
|
|
485
|
-
const toBlock = this.checkpoints.at(-1).blocks.at(-1).number;
|
|
486
|
-
this.log.info(`Starting epoch ${epochNumber} proving job with checkpoints ${fromCheckpoint} to ${toCheckpoint}`, {
|
|
487
|
-
fromBlock,
|
|
488
|
-
toBlock,
|
|
489
|
-
epochSizeTxs,
|
|
490
|
-
epochNumber,
|
|
491
|
-
uuid: this.uuid
|
|
492
|
-
});
|
|
493
|
-
this.progressState('processing');
|
|
494
|
-
const timer = new Timer();
|
|
495
|
-
const { promise, resolve } = promiseWithResolvers();
|
|
496
|
-
this.runPromise = promise;
|
|
507
|
+
* Kicks off proving + submission. Fires and forgets — callers should await `whenDone()`.
|
|
508
|
+
* Returns a promise that resolves to the final state for callers that want to wait inline.
|
|
509
|
+
*/ async start() {
|
|
497
510
|
try {
|
|
498
|
-
|
|
499
|
-
const finalBlobBatchingChallenges = await buildFinalBlobChallenges(blobFieldsPerCheckpoint);
|
|
500
|
-
this.prover.startNewEpoch(epochNumber, epochSizeCheckpoints, finalBlobBatchingChallenges);
|
|
501
|
-
await this.prover.startChonkVerifierCircuits(Array.from(this.txs.values()));
|
|
502
|
-
// Everything in the epoch should have the same chainId and version.
|
|
503
|
-
const { chainId, version } = this.checkpoints[0].blocks[0].header.globalVariables;
|
|
504
|
-
const previousBlockHeaders = this.gatherPreviousBlockHeaders();
|
|
505
|
-
await asyncPool(this.config.parallelBlockLimit ?? 32, this.checkpoints, async (checkpoint)=>{
|
|
506
|
-
this.checkState();
|
|
507
|
-
const checkpointIndex = checkpoint.number - fromCheckpoint;
|
|
508
|
-
const checkpointConstants = CheckpointConstantData.from({
|
|
509
|
-
chainId,
|
|
510
|
-
version,
|
|
511
|
-
vkTreeRoot: getVKTreeRoot(),
|
|
512
|
-
protocolContractsHash: protocolContractsHash,
|
|
513
|
-
proverId: this.prover.getProverId().toField(),
|
|
514
|
-
slotNumber: checkpoint.header.slotNumber,
|
|
515
|
-
coinbase: checkpoint.header.coinbase,
|
|
516
|
-
feeRecipient: checkpoint.header.feeRecipient,
|
|
517
|
-
gasFees: checkpoint.header.gasFees
|
|
518
|
-
});
|
|
519
|
-
const previousHeader = previousBlockHeaders[checkpointIndex];
|
|
520
|
-
const l1ToL2Messages = this.getL1ToL2Messages(checkpoint);
|
|
521
|
-
this.log.verbose(`Starting processing checkpoint ${checkpoint.number}`, {
|
|
522
|
-
number: checkpoint.number,
|
|
523
|
-
checkpointHash: checkpoint.hash().toString(),
|
|
524
|
-
lastArchive: checkpoint.header.lastArchiveRoot,
|
|
525
|
-
previousHeader: previousHeader.hash(),
|
|
526
|
-
uuid: this.uuid
|
|
527
|
-
});
|
|
528
|
-
await this.prover.startNewCheckpoint(checkpointIndex, checkpointConstants, l1ToL2Messages, checkpoint.blocks.length, previousHeader);
|
|
529
|
-
for (const block of checkpoint.blocks){
|
|
530
|
-
const globalVariables = block.header.globalVariables;
|
|
531
|
-
const txs = this.getTxs(block);
|
|
532
|
-
this.log.verbose(`Starting processing block ${block.number}`, {
|
|
533
|
-
number: block.number,
|
|
534
|
-
blockHash: (await block.hash()).toString(),
|
|
535
|
-
lastArchive: block.header.lastArchive.root,
|
|
536
|
-
noteHashTreeRoot: block.header.state.partial.noteHashTree.root,
|
|
537
|
-
nullifierTreeRoot: block.header.state.partial.nullifierTree.root,
|
|
538
|
-
publicDataTreeRoot: block.header.state.partial.publicDataTree.root,
|
|
539
|
-
...globalVariables,
|
|
540
|
-
numTxs: txs.length
|
|
541
|
-
});
|
|
542
|
-
// Start block proving
|
|
543
|
-
await this.prover.startNewBlock(block.number, globalVariables.timestamp, txs.length);
|
|
544
|
-
// Process public fns
|
|
545
|
-
const db = await this.createFork(BlockNumber(block.number - 1), l1ToL2Messages);
|
|
546
|
-
const config = PublicSimulatorConfig.from({
|
|
547
|
-
proverId: this.prover.getProverId().toField(),
|
|
548
|
-
skipFeeEnforcement: false,
|
|
549
|
-
collectDebugLogs: false,
|
|
550
|
-
collectHints: true,
|
|
551
|
-
collectPublicInputs: true,
|
|
552
|
-
collectStatistics: false
|
|
553
|
-
});
|
|
554
|
-
const publicProcessor = this.publicProcessorFactory.create(db, globalVariables, config);
|
|
555
|
-
const processed = await this.processTxs(publicProcessor, txs);
|
|
556
|
-
await this.prover.addTxs(processed);
|
|
557
|
-
await db.close();
|
|
558
|
-
this.log.verbose(`Processed all ${txs.length} txs for block ${block.number}`, {
|
|
559
|
-
blockNumber: block.number,
|
|
560
|
-
blockHash: (await block.hash()).toString(),
|
|
561
|
-
uuid: this.uuid
|
|
562
|
-
});
|
|
563
|
-
// Mark block as completed to pad it
|
|
564
|
-
const expectedBlockHeader = block.header;
|
|
565
|
-
await this.prover.setBlockCompleted(block.number, expectedBlockHeader);
|
|
566
|
-
}
|
|
567
|
-
});
|
|
568
|
-
const executionTime = timer.ms();
|
|
569
|
-
this.progressState('awaiting-prover');
|
|
570
|
-
const { publicInputs, proof, batchedBlobInputs } = await this.prover.finalizeEpoch();
|
|
571
|
-
this.log.info(`Finalized proof for epoch ${epochNumber}`, {
|
|
572
|
-
epochNumber,
|
|
573
|
-
uuid: this.uuid,
|
|
574
|
-
duration: timer.ms()
|
|
575
|
-
});
|
|
576
|
-
this.progressState('publishing-proof');
|
|
577
|
-
if (this.config.skipSubmitProof) {
|
|
578
|
-
this.log.info(`Proof publishing is disabled. Dropping valid proof for epoch ${epochNumber} (checkpoints ${fromCheckpoint} to ${toCheckpoint})`);
|
|
579
|
-
this.state = 'completed';
|
|
580
|
-
this.metrics.recordProvingJob(executionTime, timer.ms(), epochSizeCheckpoints, epochSizeBlocks, epochSizeTxs);
|
|
581
|
-
return;
|
|
582
|
-
}
|
|
583
|
-
const success = await this.publisher.submitEpochProof({
|
|
584
|
-
fromCheckpoint,
|
|
585
|
-
toCheckpoint,
|
|
586
|
-
epochNumber,
|
|
587
|
-
publicInputs,
|
|
588
|
-
proof,
|
|
589
|
-
batchedBlobInputs,
|
|
590
|
-
attestations
|
|
591
|
-
});
|
|
592
|
-
if (!success) {
|
|
593
|
-
throw new Error('Failed to submit epoch proof to L1');
|
|
594
|
-
}
|
|
595
|
-
this.log.info(`Submitted proof for epoch ${epochNumber} (checkpoints ${fromCheckpoint} to ${toCheckpoint})`, {
|
|
596
|
-
epochNumber,
|
|
597
|
-
uuid: this.uuid
|
|
598
|
-
});
|
|
599
|
-
this.state = 'completed';
|
|
600
|
-
this.metrics.recordProvingJob(executionTime, timer.ms(), epochSizeCheckpoints, epochSizeBlocks, epochSizeTxs);
|
|
511
|
+
await this.run();
|
|
601
512
|
} catch (err) {
|
|
602
|
-
|
|
603
|
-
this.log.warn(`Halted execution of epoch ${epochNumber} prover job`, {
|
|
604
|
-
uuid: this.uuid,
|
|
605
|
-
epochNumber,
|
|
606
|
-
details: err.message
|
|
607
|
-
});
|
|
608
|
-
return;
|
|
609
|
-
}
|
|
610
|
-
this.log.error(`Error running epoch ${epochNumber} prover job`, err, {
|
|
513
|
+
this.log.error(`Error in EpochSession ${this.uuid}`, err, {
|
|
611
514
|
uuid: this.uuid,
|
|
612
|
-
|
|
515
|
+
...this.spec
|
|
613
516
|
});
|
|
614
|
-
if (this.
|
|
517
|
+
if (!this.isTerminal()) {
|
|
615
518
|
this.state = 'failed';
|
|
616
519
|
}
|
|
617
520
|
} finally{
|
|
618
521
|
clearTimeout(this.deadlineTimeoutHandler);
|
|
619
|
-
await this.
|
|
620
|
-
|
|
621
|
-
resolve();
|
|
522
|
+
await this.teardownTopTreeIfNeeded();
|
|
523
|
+
this.resolveCompletion(this.state);
|
|
622
524
|
}
|
|
525
|
+
return this.state;
|
|
623
526
|
}
|
|
624
527
|
/**
|
|
625
|
-
*
|
|
626
|
-
*
|
|
627
|
-
*/ async
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
528
|
+
* Cancels the session. Idempotent. Withdraws any submitted candidate from the
|
|
529
|
+
* publishing service so the in-flight publisher (if any) is interrupted.
|
|
530
|
+
*/ async cancel(reason = 'cancelled') {
|
|
531
|
+
if (this.isTerminal()) {
|
|
532
|
+
return;
|
|
533
|
+
}
|
|
534
|
+
this.log.info(`Cancelling EpochSession ${this.uuid}: ${reason}`, {
|
|
535
|
+
uuid: this.uuid,
|
|
536
|
+
...this.spec,
|
|
537
|
+
previousState: this.state,
|
|
538
|
+
reason
|
|
633
539
|
});
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
this.state = state;
|
|
640
|
-
}
|
|
641
|
-
checkState() {
|
|
642
|
-
if (this.state === 'timed-out' || this.state === 'stopped' || this.state === 'failed' || this.state === 'reorg') {
|
|
643
|
-
throw new HaltExecutionError(this.state);
|
|
540
|
+
this.state = 'cancelled';
|
|
541
|
+
try {
|
|
542
|
+
this.deps.publishingService.withdraw(this.uuid);
|
|
543
|
+
} catch (err) {
|
|
544
|
+
this.log.error(`Error withdrawing candidate from publishing service`, err);
|
|
644
545
|
}
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
await this.runPromise;
|
|
546
|
+
if (this.topTreeJob && !this.topTreeJob.isCancelled()) {
|
|
547
|
+
const job = this.topTreeJob;
|
|
548
|
+
this.topTreeJob = undefined;
|
|
549
|
+
job.cancel();
|
|
550
|
+
this.pendingTopTreeCleanups.push(job);
|
|
651
551
|
}
|
|
552
|
+
await this.teardownTopTreeIfNeeded();
|
|
553
|
+
this.resolveCompletion(this.state);
|
|
652
554
|
}
|
|
653
|
-
|
|
654
|
-
const
|
|
655
|
-
if (
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
555
|
+
async run() {
|
|
556
|
+
const timer = new Timer();
|
|
557
|
+
if (this.deps.config.finalizationDelayMs && this.deps.config.finalizationDelayMs > 0) {
|
|
558
|
+
this.log.warn(`Waiting ${this.deps.config.finalizationDelayMs}ms before starting top-tree proving`, {
|
|
559
|
+
uuid: this.uuid,
|
|
560
|
+
...this.spec
|
|
561
|
+
});
|
|
562
|
+
await sleep(this.deps.config.finalizationDelayMs);
|
|
563
|
+
if (this.isTerminal()) {
|
|
564
|
+
return;
|
|
659
565
|
}
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
|
|
566
|
+
}
|
|
567
|
+
// Stage 1 — top-tree proving.
|
|
568
|
+
const topTreeJob = new TopTreeJob(this.spec.epochNumber, this.checkpoints, {
|
|
569
|
+
proverFactory: this.deps.proverFactory,
|
|
570
|
+
metrics: this.deps.metrics,
|
|
571
|
+
log: this.log,
|
|
572
|
+
hooks: this.toTopTreeHooks()
|
|
573
|
+
});
|
|
574
|
+
this.topTreeJob = topTreeJob;
|
|
575
|
+
const { fromCheckpoint, toCheckpoint, count } = topTreeJob.getRange();
|
|
576
|
+
this.state = 'awaiting-checkpoints';
|
|
577
|
+
let proof;
|
|
578
|
+
try {
|
|
579
|
+
proof = await topTreeJob.start();
|
|
580
|
+
} catch (err) {
|
|
581
|
+
if (err instanceof TopTreeCancelledError) {
|
|
582
|
+
// Session cancel kicked off the underlying teardown; nothing more to do here.
|
|
583
|
+
this.log.info(`Top-tree cancelled for EpochSession ${this.uuid}`, {
|
|
665
584
|
uuid: this.uuid,
|
|
666
|
-
|
|
667
|
-
});
|
|
668
|
-
this.stop('timed-out').catch((err)=>{
|
|
669
|
-
this.log.error('Error stopping job', err, {
|
|
670
|
-
uuid: this.uuid,
|
|
671
|
-
epochNumber: this.epochNumber
|
|
672
|
-
});
|
|
585
|
+
...this.spec
|
|
673
586
|
});
|
|
674
|
-
|
|
587
|
+
return;
|
|
588
|
+
}
|
|
589
|
+
throw err;
|
|
675
590
|
}
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
591
|
+
this.topTreeJob = undefined;
|
|
592
|
+
this.log.info(`Top-tree proof ready for EpochSession ${this.uuid}`, {
|
|
593
|
+
uuid: this.uuid,
|
|
594
|
+
...this.spec,
|
|
595
|
+
fromCheckpoint,
|
|
596
|
+
toCheckpoint,
|
|
597
|
+
durationMs: timer.ms()
|
|
598
|
+
});
|
|
599
|
+
// Stage 2 — hand the proof to the publishing service and wait for its verdict.
|
|
600
|
+
await this.submitProof(proof, fromCheckpoint, toCheckpoint, count, timer);
|
|
601
|
+
}
|
|
602
|
+
async submitProof(proof, fromCheckpoint, toCheckpoint, checkpointCount, timer) {
|
|
603
|
+
// Attestations come from the highest-numbered registered checkpoint — that's the one
|
|
604
|
+
// whose attestations the L1 contract checks for the proven range.
|
|
605
|
+
const lastCheckpoint = this.checkpoints[this.checkpoints.length - 1];
|
|
606
|
+
const attestations = lastCheckpoint.attestations.map((a)=>a.toViem());
|
|
607
|
+
const epochSizeBlocks = this.checkpoints.reduce((acc, c)=>acc + c.checkpoint.blocks.length, 0);
|
|
608
|
+
const epochSizeTxs = this.checkpoints.reduce((acc, c)=>acc + c.checkpoint.blocks.reduce((bAcc, block)=>bAcc + block.body.txEffects.length, 0), 0);
|
|
609
|
+
// Reflect the publish phase. Guard against a terminal state set concurrently by cancel() — the
|
|
610
|
+
// post-submit isTerminal() check below relies on cancel still winning.
|
|
611
|
+
if (!this.isTerminal()) {
|
|
612
|
+
this.state = 'publishing-proof';
|
|
613
|
+
}
|
|
614
|
+
const outcome = await this.deps.publishingService.submit({
|
|
615
|
+
id: this.uuid,
|
|
616
|
+
epoch: this.spec.epochNumber,
|
|
617
|
+
kind: this.spec.kind,
|
|
618
|
+
startBlock: this.getStartBlockNumber(),
|
|
619
|
+
endBlock: this.getEndBlockNumber(),
|
|
620
|
+
deadline: this.deps.deadline,
|
|
621
|
+
fromCheckpoint,
|
|
622
|
+
toCheckpoint,
|
|
623
|
+
publicInputs: proof.publicInputs,
|
|
624
|
+
proof: proof.proof,
|
|
625
|
+
batchedBlobInputs: proof.batchedBlobInputs,
|
|
626
|
+
attestations,
|
|
627
|
+
headers: this.checkpoints.map((c)=>c.checkpoint.header)
|
|
628
|
+
});
|
|
629
|
+
if (this.isTerminal()) {
|
|
630
|
+
// cancel() already set the terminal state — don't clobber it.
|
|
684
631
|
return;
|
|
685
632
|
}
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
const blockHashes = await Promise.all(blocks.map((block)=>block.hash()));
|
|
690
|
-
const thisBlocks = this.checkpoints.flatMap((checkpoint)=>checkpoint.blocks);
|
|
691
|
-
const thisBlockHashes = await Promise.all(thisBlocks.map((block)=>block.hash()));
|
|
692
|
-
if (blocks.length !== thisBlocks.length || !blockHashes.every((block, i)=>block.equals(thisBlockHashes[i]))) {
|
|
693
|
-
this.log.warn('Epoch blocks changed underfoot', {
|
|
633
|
+
switch(outcome){
|
|
634
|
+
case 'published':
|
|
635
|
+
this.log.info(`Submitted proof for epoch ${this.spec.epochNumber} (checkpoints ${fromCheckpoint}..${toCheckpoint})`, {
|
|
694
636
|
uuid: this.uuid,
|
|
695
|
-
|
|
696
|
-
oldBlockHashes: thisBlockHashes,
|
|
697
|
-
newBlockHashes: blockHashes
|
|
637
|
+
...this.spec
|
|
698
638
|
});
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
|
|
639
|
+
this.state = 'completed';
|
|
640
|
+
this.deps.metrics.recordProvingJob(timer.ms(), checkpointCount, epochSizeBlocks, epochSizeTxs);
|
|
641
|
+
return;
|
|
642
|
+
case 'superseded':
|
|
643
|
+
this.log.info(`EpochSession ${this.uuid} superseded by a longer candidate`, {
|
|
644
|
+
uuid: this.uuid,
|
|
645
|
+
...this.spec
|
|
646
|
+
});
|
|
647
|
+
this.state = 'superseded';
|
|
648
|
+
return;
|
|
649
|
+
case 'withdrawn':
|
|
650
|
+
// cancel() ran but the terminal-state check above missed it. Defensive: treat as cancelled.
|
|
651
|
+
this.state = 'cancelled';
|
|
652
|
+
return;
|
|
653
|
+
case 'expired':
|
|
654
|
+
this.log.warn(`EpochSession ${this.uuid} expired before publishing`, {
|
|
655
|
+
uuid: this.uuid,
|
|
656
|
+
...this.spec
|
|
657
|
+
});
|
|
658
|
+
this.state = 'timed-out';
|
|
659
|
+
return;
|
|
660
|
+
case 'failed':
|
|
661
|
+
throw new Error('Failed to submit epoch proof to L1');
|
|
662
|
+
}
|
|
710
663
|
}
|
|
711
|
-
|
|
712
|
-
|
|
664
|
+
async teardownTopTreeIfNeeded() {
|
|
665
|
+
if (this.topTreeJob) {
|
|
666
|
+
const job = this.topTreeJob;
|
|
667
|
+
this.topTreeJob = undefined;
|
|
668
|
+
job.cancel();
|
|
669
|
+
this.pendingTopTreeCleanups.push(job);
|
|
670
|
+
}
|
|
671
|
+
if (this.pendingTopTreeCleanups.length > 0) {
|
|
672
|
+
await Promise.allSettled(this.pendingTopTreeCleanups.map((j)=>j.whenDone()));
|
|
673
|
+
this.pendingTopTreeCleanups.length = 0;
|
|
674
|
+
}
|
|
713
675
|
}
|
|
714
|
-
|
|
715
|
-
|
|
676
|
+
scheduleDeadlineStop() {
|
|
677
|
+
const deadline = this.deps.deadline;
|
|
678
|
+
if (!deadline) {
|
|
679
|
+
return;
|
|
680
|
+
}
|
|
681
|
+
const timeout = Math.max(deadline.getTime() - this.deps.dateProvider.now(), 0);
|
|
682
|
+
this.deadlineTimeoutHandler = setTimeout(()=>{
|
|
683
|
+
void this.handleDeadline();
|
|
684
|
+
}, timeout);
|
|
716
685
|
}
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
if (
|
|
723
|
-
|
|
724
|
-
throw new Error(`Txs failed processing: ${failedTxs.map(({ error }, index)=>`${failedTxHashes[index]} (${error})`).join(', ')}`);
|
|
686
|
+
/**
|
|
687
|
+
* Returns a promise that resolves once cancellation has propagated and the state has
|
|
688
|
+
* been flipped from 'cancelled' to 'timed-out'. Protected so unit tests can drive the
|
|
689
|
+
* deadline path without waiting on the real `setTimeout` to fire.
|
|
690
|
+
*/ async handleDeadline() {
|
|
691
|
+
if (this.isTerminal()) {
|
|
692
|
+
return;
|
|
725
693
|
}
|
|
726
|
-
|
|
727
|
-
|
|
694
|
+
this.log.warn(`EpochSession ${this.uuid} hit deadline`, {
|
|
695
|
+
uuid: this.uuid,
|
|
696
|
+
...this.spec
|
|
697
|
+
});
|
|
698
|
+
await this.cancel('deadline');
|
|
699
|
+
// After cancel, override state if it was the canonical timeout case so observers see 'timed-out'.
|
|
700
|
+
if (this.state === 'cancelled') {
|
|
701
|
+
this.state = 'timed-out';
|
|
728
702
|
}
|
|
729
|
-
return processedTxs;
|
|
730
703
|
}
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
704
|
+
toTopTreeHooks() {
|
|
705
|
+
const hooks = this.deps.hooks;
|
|
706
|
+
return {
|
|
707
|
+
// `beforeProve` fires once the sub-tree (checkpoint block) proofs are ready and the root prove is
|
|
708
|
+
// about to start — the boundary between `awaiting-checkpoints` and proving the top tree. Don't
|
|
709
|
+
// clobber a terminal state set concurrently by cancel().
|
|
710
|
+
beforeProve: async ()=>{
|
|
711
|
+
if (!this.isTerminal()) {
|
|
712
|
+
this.state = 'awaiting-root';
|
|
713
|
+
}
|
|
714
|
+
await hooks?.beforeTopTreeProve?.();
|
|
715
|
+
},
|
|
716
|
+
afterProve: hooks?.afterTopTreeProve,
|
|
717
|
+
proveOverride: hooks?.topTreeProveOverride
|
|
718
|
+
};
|
|
737
719
|
}
|
|
738
720
|
}
|