@aztec/sequencer-client 0.0.1-commit.0dc957cde → 0.0.1-commit.0ec55a70b
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dest/config.d.ts +2 -1
- package/dest/config.d.ts.map +1 -1
- package/dest/config.js +12 -7
- package/dest/publisher/sequencer-publisher.d.ts +1 -1
- package/dest/publisher/sequencer-publisher.d.ts.map +1 -1
- package/dest/publisher/sequencer-publisher.js +2 -2
- package/dest/sequencer/checkpoint_proposal_job.d.ts +8 -2
- package/dest/sequencer/checkpoint_proposal_job.d.ts.map +1 -1
- package/dest/sequencer/checkpoint_proposal_job.js +189 -19
- package/dest/sequencer/metrics.d.ts +3 -1
- package/dest/sequencer/metrics.d.ts.map +1 -1
- package/dest/sequencer/metrics.js +7 -0
- package/dest/sequencer/sequencer.d.ts +12 -4
- package/dest/sequencer/sequencer.d.ts.map +1 -1
- package/dest/sequencer/sequencer.js +33 -8
- package/dest/test/utils.d.ts +1 -1
- package/dest/test/utils.d.ts.map +1 -1
- package/dest/test/utils.js +7 -6
- package/package.json +27 -27
- package/src/config.ts +11 -4
- package/src/publisher/sequencer-publisher.ts +2 -4
- package/src/sequencer/README.md +20 -21
- package/src/sequencer/checkpoint_proposal_job.ts +211 -17
- package/src/sequencer/metrics.ts +9 -0
- package/src/sequencer/sequencer.ts +37 -9
- package/src/test/utils.ts +28 -10
|
@@ -46,8 +46,10 @@ import { type L1ToL2MessageSource, computeInHashFromL1ToL2Messages } from '@azte
|
|
|
46
46
|
import type {
|
|
47
47
|
BlockProposal,
|
|
48
48
|
BlockProposalOptions,
|
|
49
|
+
CheckpointAttestation,
|
|
49
50
|
CheckpointProposal,
|
|
50
51
|
CheckpointProposalOptions,
|
|
52
|
+
CoordinationSignatureContext,
|
|
51
53
|
} from '@aztec/stdlib/p2p';
|
|
52
54
|
import { orderAttestations, trimAttestations } from '@aztec/stdlib/p2p';
|
|
53
55
|
import type { L2BlockBuiltStats } from '@aztec/stdlib/stats';
|
|
@@ -97,6 +99,7 @@ type CheckpointProposalResult = {
|
|
|
97
99
|
*/
|
|
98
100
|
export class CheckpointProposalJob implements Traceable {
|
|
99
101
|
protected readonly log: Logger;
|
|
102
|
+
private readonly checkpointEventLog: Logger;
|
|
100
103
|
|
|
101
104
|
/** Tracks the fire-and-forget L1 submission promise so it can be awaited during shutdown. */
|
|
102
105
|
private pendingL1Submission: Promise<void> | undefined;
|
|
@@ -104,6 +107,10 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
104
107
|
/** Pipelined parent chain state used while building and later submitting this checkpoint. */
|
|
105
108
|
private pipelinedParentSimulationOverridesPlan?: SimulationOverridesPlan;
|
|
106
109
|
|
|
110
|
+
private getSignatureContext(): CoordinationSignatureContext {
|
|
111
|
+
return this.signatureContext;
|
|
112
|
+
}
|
|
113
|
+
|
|
107
114
|
constructor(
|
|
108
115
|
private readonly slotNow: SlotNumber,
|
|
109
116
|
private readonly targetSlot: SlotNumber,
|
|
@@ -124,6 +131,7 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
124
131
|
private readonly checkpointsBuilder: FullNodeCheckpointsBuilder,
|
|
125
132
|
private readonly blockSink: L2BlockSink,
|
|
126
133
|
private readonly l1Constants: SequencerRollupConstants,
|
|
134
|
+
private readonly signatureContext: CoordinationSignatureContext,
|
|
127
135
|
protected config: ResolvedSequencerConfig,
|
|
128
136
|
protected timetable: SequencerTimetable,
|
|
129
137
|
private readonly slasherClient: SlasherClientInterface | undefined,
|
|
@@ -141,6 +149,10 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
141
149
|
...bindings,
|
|
142
150
|
instanceId: `slot-${this.slotNow}`,
|
|
143
151
|
});
|
|
152
|
+
this.checkpointEventLog = createLogger('sequencer:checkpoint-events', {
|
|
153
|
+
...bindings,
|
|
154
|
+
instanceId: `slot-${this.slotNow}`,
|
|
155
|
+
});
|
|
144
156
|
}
|
|
145
157
|
|
|
146
158
|
/** Awaits the pending L1 submission if one is in progress. Call during shutdown. */
|
|
@@ -149,6 +161,13 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
149
161
|
await this.pendingL1Submission;
|
|
150
162
|
}
|
|
151
163
|
|
|
164
|
+
private logCheckpointEvent(eventName: string, message: string, fields: Record<string, unknown>): void {
|
|
165
|
+
this.checkpointEventLog.debug(message, {
|
|
166
|
+
eventName: `sequencer-checkpoint-${eventName}`,
|
|
167
|
+
...fields,
|
|
168
|
+
});
|
|
169
|
+
}
|
|
170
|
+
|
|
152
171
|
/**
|
|
153
172
|
* Executes the checkpoint proposal job.
|
|
154
173
|
* Builds blocks, assembles checkpoint, and broadcasts the proposal (blocking).
|
|
@@ -248,10 +267,34 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
248
267
|
const l1Response = await this.publisher.sendRequestsAt(submitAfter);
|
|
249
268
|
const proposedAction = l1Response?.successfulActions.find(a => a === 'propose');
|
|
250
269
|
if (proposedAction) {
|
|
270
|
+
this.logCheckpointEvent('published', `Checkpoint published for slot ${this.targetSlot}`, {
|
|
271
|
+
slot: this.targetSlot,
|
|
272
|
+
checkpointNumber: this.checkpointNumber,
|
|
273
|
+
successfulActions: l1Response?.successfulActions,
|
|
274
|
+
sentActions: l1Response?.sentActions,
|
|
275
|
+
});
|
|
251
276
|
this.eventEmitter.emit('checkpoint-published', { checkpoint: this.checkpointNumber, slot: this.targetSlot });
|
|
252
277
|
const coinbase = checkpoint.header.coinbase;
|
|
253
278
|
await this.metrics.incFilledSlot(this.publisher.getSenderAddress().toString(), coinbase);
|
|
254
279
|
} else {
|
|
280
|
+
this.logCheckpointEvent('publish-failed', `Checkpoint publish failed for slot ${this.targetSlot}`, {
|
|
281
|
+
slot: this.targetSlot,
|
|
282
|
+
checkpointNumber: this.checkpointNumber,
|
|
283
|
+
successfulActions: l1Response?.successfulActions,
|
|
284
|
+
failedActions: l1Response?.failedActions,
|
|
285
|
+
sentActions: l1Response?.sentActions,
|
|
286
|
+
expiredActions: l1Response?.expiredActions,
|
|
287
|
+
reason: 'propose_action_not_successful',
|
|
288
|
+
});
|
|
289
|
+
this.log.warn(`Checkpoint publish failed for slot ${this.targetSlot}`, {
|
|
290
|
+
slot: this.targetSlot,
|
|
291
|
+
checkpointNumber: this.checkpointNumber,
|
|
292
|
+
successfulActions: l1Response?.successfulActions,
|
|
293
|
+
failedActions: l1Response?.failedActions,
|
|
294
|
+
sentActions: l1Response?.sentActions,
|
|
295
|
+
expiredActions: l1Response?.expiredActions,
|
|
296
|
+
reason: 'propose_action_not_successful',
|
|
297
|
+
});
|
|
255
298
|
this.eventEmitter.emit('checkpoint-publish-failed', { ...l1Response, slot: this.targetSlot });
|
|
256
299
|
if (isPipelining) {
|
|
257
300
|
this.metrics.recordPipelineDiscard();
|
|
@@ -261,7 +304,16 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
261
304
|
if (err instanceof SequencerInterruptedError) {
|
|
262
305
|
return;
|
|
263
306
|
}
|
|
264
|
-
this.
|
|
307
|
+
this.logCheckpointEvent('publish-failed', `Checkpoint publish failed for slot ${this.targetSlot}`, {
|
|
308
|
+
slot: this.targetSlot,
|
|
309
|
+
checkpointNumber: this.checkpointNumber,
|
|
310
|
+
reason: err instanceof Error ? err.message : String(err),
|
|
311
|
+
});
|
|
312
|
+
this.log.error(`Background attestation/L1 pipeline failed for slot ${this.targetSlot}`, err, {
|
|
313
|
+
slot: this.targetSlot,
|
|
314
|
+
checkpointNumber: this.checkpointNumber,
|
|
315
|
+
reason: err instanceof Error ? err.message : String(err),
|
|
316
|
+
});
|
|
265
317
|
this.eventEmitter.emit('checkpoint-publish-failed', { slot: this.targetSlot });
|
|
266
318
|
if (isPipelining) {
|
|
267
319
|
this.metrics.recordPipelineDiscard();
|
|
@@ -466,11 +518,15 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
466
518
|
|
|
467
519
|
// Start the checkpoint
|
|
468
520
|
this.setStateFn(SequencerState.INITIALIZING_CHECKPOINT, this.targetSlot);
|
|
469
|
-
this.
|
|
521
|
+
this.logCheckpointEvent('slot-started', `Starting checkpoint proposal for slot ${this.targetSlot}`, {
|
|
470
522
|
buildSlot: this.slotNow,
|
|
471
523
|
submissionSlot: this.targetSlot,
|
|
524
|
+
slot: this.targetSlot,
|
|
525
|
+
checkpointNumber: this.checkpointNumber,
|
|
472
526
|
pipelining: this.epochCache.isProposerPipeliningEnabled(),
|
|
473
527
|
proposer: this.proposer?.toString(),
|
|
528
|
+
attestorAddress: this.attestorAddress.toString(),
|
|
529
|
+
publisherAddress: this.publisher.getSenderAddress().toString(),
|
|
474
530
|
coinbase: coinbase.toString(),
|
|
475
531
|
});
|
|
476
532
|
this.metrics.incOpenSlot(this.targetSlot, this.proposer?.toString() ?? 'unknown');
|
|
@@ -562,16 +618,38 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
562
618
|
}
|
|
563
619
|
|
|
564
620
|
if (blocksInCheckpoint.length === 0) {
|
|
565
|
-
this.
|
|
621
|
+
this.logCheckpointEvent('build-failed', `Checkpoint build failed for slot ${this.targetSlot}`, {
|
|
622
|
+
slot: this.targetSlot,
|
|
623
|
+
checkpointNumber: this.checkpointNumber,
|
|
624
|
+
reason: 'no_blocks_built',
|
|
625
|
+
});
|
|
626
|
+
this.log.warn(`No blocks were built for slot ${this.targetSlot}`, {
|
|
627
|
+
slot: this.targetSlot,
|
|
628
|
+
checkpointNumber: this.checkpointNumber,
|
|
629
|
+
reason: 'no_blocks_built',
|
|
630
|
+
});
|
|
566
631
|
this.eventEmitter.emit('checkpoint-empty', { slot: this.targetSlot });
|
|
567
632
|
return undefined;
|
|
568
633
|
}
|
|
569
634
|
|
|
570
635
|
const minBlocksForCheckpoint = this.config.minBlocksForCheckpoint;
|
|
571
636
|
if (minBlocksForCheckpoint !== undefined && blocksInCheckpoint.length < minBlocksForCheckpoint) {
|
|
637
|
+
this.logCheckpointEvent('build-failed', `Checkpoint build failed for slot ${this.targetSlot}`, {
|
|
638
|
+
slot: this.targetSlot,
|
|
639
|
+
checkpointNumber: this.checkpointNumber,
|
|
640
|
+
blocksBuilt: blocksInCheckpoint.length,
|
|
641
|
+
minBlocksForCheckpoint,
|
|
642
|
+
reason: 'min_blocks_not_met',
|
|
643
|
+
});
|
|
572
644
|
this.log.warn(
|
|
573
645
|
`Checkpoint has fewer blocks than minimum (${blocksInCheckpoint.length} < ${minBlocksForCheckpoint}), skipping proposal`,
|
|
574
|
-
{
|
|
646
|
+
{
|
|
647
|
+
slot: this.targetSlot,
|
|
648
|
+
checkpointNumber: this.checkpointNumber,
|
|
649
|
+
blocksBuilt: blocksInCheckpoint.length,
|
|
650
|
+
minBlocksForCheckpoint,
|
|
651
|
+
reason: 'min_blocks_not_met',
|
|
652
|
+
},
|
|
575
653
|
);
|
|
576
654
|
return undefined;
|
|
577
655
|
}
|
|
@@ -592,7 +670,18 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
592
670
|
maxTxsPerCheckpoint: this.config.maxTxsPerCheckpoint,
|
|
593
671
|
});
|
|
594
672
|
} catch (err) {
|
|
673
|
+
this.logCheckpointEvent('build-failed', `Checkpoint build failed for slot ${this.targetSlot}`, {
|
|
674
|
+
slot: this.targetSlot,
|
|
675
|
+
checkpointNumber: this.checkpointNumber,
|
|
676
|
+
blocksBuilt: blocksInCheckpoint.length,
|
|
677
|
+
reason: 'invalid_checkpoint',
|
|
678
|
+
checkpoint: checkpoint.header.toInspect(),
|
|
679
|
+
});
|
|
595
680
|
this.log.error(`Built an invalid checkpoint at slot ${this.slotNow} (skipping proposal)`, err, {
|
|
681
|
+
slot: this.targetSlot,
|
|
682
|
+
checkpointNumber: this.checkpointNumber,
|
|
683
|
+
blocksBuilt: blocksInCheckpoint.length,
|
|
684
|
+
reason: 'invalid_checkpoint',
|
|
596
685
|
checkpoint: checkpoint.header.toInspect(),
|
|
597
686
|
});
|
|
598
687
|
return undefined;
|
|
@@ -605,6 +694,17 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
605
694
|
checkpoint.getStats().txCount,
|
|
606
695
|
Number(checkpoint.header.totalManaUsed.toBigInt()),
|
|
607
696
|
);
|
|
697
|
+
this.logCheckpointEvent('built', `Checkpoint built for slot ${this.targetSlot}`, {
|
|
698
|
+
slot: this.targetSlot,
|
|
699
|
+
buildSlot: this.slotNow,
|
|
700
|
+
checkpointNumber: this.checkpointNumber,
|
|
701
|
+
proposer: this.proposer?.toString(),
|
|
702
|
+
attestorAddress: this.attestorAddress.toString(),
|
|
703
|
+
publisherAddress: this.publisher.getSenderAddress().toString(),
|
|
704
|
+
blocksBuilt: blocksInCheckpoint.length,
|
|
705
|
+
txCount: checkpoint.getStats().txCount,
|
|
706
|
+
totalMana: Number(checkpoint.header.totalManaUsed.toBigInt()),
|
|
707
|
+
});
|
|
608
708
|
|
|
609
709
|
// In fisherman mode, return the checkpoint without broadcasting or collecting attestations
|
|
610
710
|
if (this.config.fishermanMode) {
|
|
@@ -634,8 +734,10 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
634
734
|
);
|
|
635
735
|
|
|
636
736
|
const blockProposedAt = this.dateProvider.now();
|
|
637
|
-
|
|
638
|
-
|
|
737
|
+
if (!this.config.skipBroadcastProposals) {
|
|
738
|
+
await this.p2pClient.broadcastCheckpointProposal(proposal);
|
|
739
|
+
this.checkpointMetrics.noteCheckpointBroadcast(this.dateProvider.now());
|
|
740
|
+
}
|
|
639
741
|
|
|
640
742
|
// Return immediately after broadcast — attestation collection happens in the background
|
|
641
743
|
return { checkpoint, proposal, blockProposedAt };
|
|
@@ -675,6 +777,15 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
675
777
|
const indexWithinCheckpoint = IndexWithinCheckpoint(blocksBuilt);
|
|
676
778
|
const blockNumber = BlockNumber(initialBlockNumber + blocksBuilt);
|
|
677
779
|
|
|
780
|
+
if (blocksBuilt >= this.config.maxBlocksPerCheckpoint) {
|
|
781
|
+
this.log.debug(`Reached max blocks per checkpoint`, {
|
|
782
|
+
slot: this.targetSlot,
|
|
783
|
+
blocksBuilt,
|
|
784
|
+
maxBlocksPerCheckpoint: this.config.maxBlocksPerCheckpoint,
|
|
785
|
+
});
|
|
786
|
+
break;
|
|
787
|
+
}
|
|
788
|
+
|
|
678
789
|
const secondsIntoSlot = this.getSecondsIntoSlot();
|
|
679
790
|
const timingInfo = this.timetable.canStartNextBlock(secondsIntoSlot);
|
|
680
791
|
|
|
@@ -756,7 +867,9 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
756
867
|
}
|
|
757
868
|
|
|
758
869
|
// Once we have a signed proposal and the archiver agreed with our proposed block, then we broadcast it.
|
|
759
|
-
proposal &&
|
|
870
|
+
if (proposal && !this.config.skipBroadcastProposals) {
|
|
871
|
+
await this.p2pClient.broadcastProposal(proposal);
|
|
872
|
+
}
|
|
760
873
|
|
|
761
874
|
// Wait until the next block's start time
|
|
762
875
|
await this.waitUntilNextSubslot(timingInfo.deadline);
|
|
@@ -830,9 +943,26 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
830
943
|
// Wait until we have enough txs to build the block
|
|
831
944
|
const { availableTxs, canStartBuilding, minTxs } = await this.waitForMinTxs(opts);
|
|
832
945
|
if (!canStartBuilding) {
|
|
946
|
+
this.logCheckpointEvent('block-build-failed', `Block build failed for slot ${this.targetSlot}`, {
|
|
947
|
+
reason: 'insufficient_txs',
|
|
948
|
+
blockNumber,
|
|
949
|
+
slot: this.targetSlot,
|
|
950
|
+
checkpointNumber: this.checkpointNumber,
|
|
951
|
+
indexWithinCheckpoint,
|
|
952
|
+
availableTxs,
|
|
953
|
+
minTxs,
|
|
954
|
+
});
|
|
833
955
|
this.log.warn(
|
|
834
956
|
`Not enough txs to build block ${blockNumber} at index ${indexWithinCheckpoint} in slot ${this.targetSlot} (got ${availableTxs} txs but needs ${minTxs})`,
|
|
835
|
-
{
|
|
957
|
+
{
|
|
958
|
+
reason: 'insufficient_txs',
|
|
959
|
+
blockNumber,
|
|
960
|
+
slot: this.targetSlot,
|
|
961
|
+
checkpointNumber: this.checkpointNumber,
|
|
962
|
+
indexWithinCheckpoint,
|
|
963
|
+
availableTxs,
|
|
964
|
+
minTxs,
|
|
965
|
+
},
|
|
836
966
|
);
|
|
837
967
|
this.eventEmitter.emit('block-tx-count-check-failed', { minTxs, availableTxs, slot: this.targetSlot });
|
|
838
968
|
this.metrics.recordBlockProposalFailed('insufficient_txs');
|
|
@@ -884,10 +1014,21 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
884
1014
|
await this.dropFailedTxsFromP2P(buildResult.failedTxs);
|
|
885
1015
|
|
|
886
1016
|
if (buildResult.status === 'insufficient-valid-txs') {
|
|
1017
|
+
this.logCheckpointEvent('block-build-failed', `Block build failed for slot ${this.targetSlot}`, {
|
|
1018
|
+
reason: 'insufficient_valid_txs',
|
|
1019
|
+
slot: this.targetSlot,
|
|
1020
|
+
checkpointNumber: this.checkpointNumber,
|
|
1021
|
+
blockNumber,
|
|
1022
|
+
numTxs: buildResult.processedCount,
|
|
1023
|
+
indexWithinCheckpoint,
|
|
1024
|
+
minValidTxs,
|
|
1025
|
+
});
|
|
887
1026
|
this.log.warn(
|
|
888
1027
|
`Block ${blockNumber} at index ${indexWithinCheckpoint} on slot ${this.targetSlot} has too few valid txs to be proposed`,
|
|
889
1028
|
{
|
|
1029
|
+
reason: 'insufficient_valid_txs',
|
|
890
1030
|
slot: this.targetSlot,
|
|
1031
|
+
checkpointNumber: this.checkpointNumber,
|
|
891
1032
|
blockNumber,
|
|
892
1033
|
numTxs: buildResult.processedCount,
|
|
893
1034
|
indexWithinCheckpoint,
|
|
@@ -936,7 +1077,18 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
936
1077
|
reason: err.message,
|
|
937
1078
|
slot: this.targetSlot,
|
|
938
1079
|
});
|
|
939
|
-
this.
|
|
1080
|
+
this.logCheckpointEvent('block-build-failed', `Block build failed for slot ${this.targetSlot}`, {
|
|
1081
|
+
reason: err instanceof Error ? err.message : String(err),
|
|
1082
|
+
slot: this.targetSlot,
|
|
1083
|
+
checkpointNumber: this.checkpointNumber,
|
|
1084
|
+
blockNumber,
|
|
1085
|
+
});
|
|
1086
|
+
this.log.error(`Error building block`, err, {
|
|
1087
|
+
reason: err instanceof Error ? err.message : String(err),
|
|
1088
|
+
slot: this.targetSlot,
|
|
1089
|
+
checkpointNumber: this.checkpointNumber,
|
|
1090
|
+
blockNumber,
|
|
1091
|
+
});
|
|
940
1092
|
this.metrics.recordBlockProposalFailed(err.name || 'unknown_error');
|
|
941
1093
|
this.metrics.recordFailedBlock();
|
|
942
1094
|
return { error: err };
|
|
@@ -1048,7 +1200,7 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
1048
1200
|
): Promise<CommitteeAttestationsAndSigners | undefined> {
|
|
1049
1201
|
if (this.config.fishermanMode) {
|
|
1050
1202
|
this.log.debug('Skipping attestation collection in fisherman mode');
|
|
1051
|
-
return CommitteeAttestationsAndSigners.empty();
|
|
1203
|
+
return CommitteeAttestationsAndSigners.empty(this.getSignatureContext());
|
|
1052
1204
|
}
|
|
1053
1205
|
|
|
1054
1206
|
const slotNumber = proposal.slotNumber;
|
|
@@ -1058,7 +1210,7 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
1058
1210
|
throw new Error('No committee when collecting attestations');
|
|
1059
1211
|
} else if (committee.length === 0) {
|
|
1060
1212
|
this.log.verbose(`Attesting committee is empty`);
|
|
1061
|
-
return CommitteeAttestationsAndSigners.empty();
|
|
1213
|
+
return CommitteeAttestationsAndSigners.empty(this.getSignatureContext());
|
|
1062
1214
|
} else {
|
|
1063
1215
|
this.log.debug(`Attesting committee length is ${committee.length}`, { committee });
|
|
1064
1216
|
}
|
|
@@ -1068,7 +1220,13 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
1068
1220
|
if (this.config.skipCollectingAttestations) {
|
|
1069
1221
|
this.log.warn('Skipping attestation collection as per config (attesting with own keys only)');
|
|
1070
1222
|
const attestations = await this.validatorClient?.collectOwnAttestations(proposal, this.checkpointNumber);
|
|
1071
|
-
|
|
1223
|
+
this.logCheckpointAttestations('collected', committee, attestations ?? [], numberOfRequiredAttestations, {
|
|
1224
|
+
reason: 'collect_own_only',
|
|
1225
|
+
});
|
|
1226
|
+
return new CommitteeAttestationsAndSigners(
|
|
1227
|
+
orderAttestations(attestations ?? [], committee),
|
|
1228
|
+
this.getSignatureContext(),
|
|
1229
|
+
);
|
|
1072
1230
|
}
|
|
1073
1231
|
|
|
1074
1232
|
const attestationTimeAllowed = this.config.enforceTimeTable
|
|
@@ -1104,6 +1262,9 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
1104
1262
|
|
|
1105
1263
|
// Rollup contract requires that the signatures are provided in the order of the committee
|
|
1106
1264
|
const sorted = orderAttestations(trimmed, committee);
|
|
1265
|
+
this.logCheckpointAttestations('collected', committee, attestations, numberOfRequiredAttestations, {
|
|
1266
|
+
submittedCount: trimmed.length,
|
|
1267
|
+
});
|
|
1107
1268
|
|
|
1108
1269
|
// Manipulate the attestations if we've been configured to do so
|
|
1109
1270
|
if (
|
|
@@ -1115,15 +1276,23 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
1115
1276
|
return this.manipulateAttestations(proposal.slotNumber, epoch, seed, committee, sorted);
|
|
1116
1277
|
}
|
|
1117
1278
|
|
|
1118
|
-
return new CommitteeAttestationsAndSigners(sorted);
|
|
1279
|
+
return new CommitteeAttestationsAndSigners(sorted, this.getSignatureContext());
|
|
1119
1280
|
} catch (err) {
|
|
1120
1281
|
if (err && err instanceof AttestationTimeoutError) {
|
|
1121
1282
|
collectedAttestationsCount = err.collectedCount;
|
|
1283
|
+
this.logCheckpointAttestations('failed', committee, undefined, numberOfRequiredAttestations, {
|
|
1284
|
+
collectedCount: collectedAttestationsCount,
|
|
1285
|
+
reason: 'timeout',
|
|
1286
|
+
});
|
|
1122
1287
|
this.log.error(
|
|
1123
1288
|
`Timeout while waiting for attestations for checkpoint proposal at slot ${proposal.slotNumber} (collected ${collectedAttestationsCount}/${numberOfRequiredAttestations})`,
|
|
1124
1289
|
err,
|
|
1125
1290
|
);
|
|
1126
1291
|
} else {
|
|
1292
|
+
this.logCheckpointAttestations('failed', committee, undefined, numberOfRequiredAttestations, {
|
|
1293
|
+
collectedCount: collectedAttestationsCount,
|
|
1294
|
+
reason: err instanceof Error ? err.message : String(err),
|
|
1295
|
+
});
|
|
1127
1296
|
this.log.error(`Error collecting attestations for checkpoint proposal at slot ${proposal.slotNumber}`, err);
|
|
1128
1297
|
}
|
|
1129
1298
|
return undefined;
|
|
@@ -1132,6 +1301,31 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
1132
1301
|
}
|
|
1133
1302
|
}
|
|
1134
1303
|
|
|
1304
|
+
private logCheckpointAttestations(
|
|
1305
|
+
status: 'collected' | 'failed',
|
|
1306
|
+
committee: EthAddress[],
|
|
1307
|
+
attestations: CheckpointAttestation[] | undefined,
|
|
1308
|
+
requiredAttestations: number,
|
|
1309
|
+
opts: { collectedCount?: number; submittedCount?: number; reason?: string } = {},
|
|
1310
|
+
) {
|
|
1311
|
+
const signedValidators =
|
|
1312
|
+
attestations
|
|
1313
|
+
?.map(attestation => attestation.getSender()?.toString())
|
|
1314
|
+
.filter((address): address is `0x${string}` => address !== undefined) ?? [];
|
|
1315
|
+
const collectedCount = opts.collectedCount ?? new Set(signedValidators).size;
|
|
1316
|
+
const missingValidatorCount = status === 'failed' ? Math.max(0, requiredAttestations - collectedCount) : undefined;
|
|
1317
|
+
this.logCheckpointEvent(`attestations-${status}`, `Checkpoint attestations ${status} for slot ${this.targetSlot}`, {
|
|
1318
|
+
slot: this.targetSlot,
|
|
1319
|
+
checkpointNumber: this.checkpointNumber,
|
|
1320
|
+
committeeSize: committee.length,
|
|
1321
|
+
requiredAttestations,
|
|
1322
|
+
collectedAttestations: collectedCount,
|
|
1323
|
+
...(opts.submittedCount !== undefined && { submittedAttestations: opts.submittedCount }),
|
|
1324
|
+
...(missingValidatorCount !== undefined && { missingValidatorCount }),
|
|
1325
|
+
...(opts.reason !== undefined && { reason: opts.reason }),
|
|
1326
|
+
});
|
|
1327
|
+
}
|
|
1328
|
+
|
|
1135
1329
|
/** Breaks the attestations before publishing based on attack configs */
|
|
1136
1330
|
private manipulateAttestations(
|
|
1137
1331
|
slotNumber: SlotNumber,
|
|
@@ -1175,7 +1369,7 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
1175
1369
|
unfreeze(attestations[targetIndex]).signature = generateRecoverableSignature();
|
|
1176
1370
|
}
|
|
1177
1371
|
}
|
|
1178
|
-
return new CommitteeAttestationsAndSigners(attestations);
|
|
1372
|
+
return new CommitteeAttestationsAndSigners(attestations, this.getSignatureContext());
|
|
1179
1373
|
}
|
|
1180
1374
|
|
|
1181
1375
|
if (this.config.shuffleAttestationOrdering) {
|
|
@@ -1197,11 +1391,11 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
1197
1391
|
[shuffled[i], shuffled[j]] = [shuffled[j], shuffled[i]];
|
|
1198
1392
|
}
|
|
1199
1393
|
|
|
1200
|
-
const signers = new CommitteeAttestationsAndSigners(attestations).getSigners();
|
|
1201
|
-
return new MaliciousCommitteeAttestationsAndSigners(shuffled, signers);
|
|
1394
|
+
const signers = new CommitteeAttestationsAndSigners(attestations, this.getSignatureContext()).getSigners();
|
|
1395
|
+
return new MaliciousCommitteeAttestationsAndSigners(shuffled, signers, this.getSignatureContext());
|
|
1202
1396
|
}
|
|
1203
1397
|
|
|
1204
|
-
return new CommitteeAttestationsAndSigners(attestations);
|
|
1398
|
+
return new CommitteeAttestationsAndSigners(attestations, this.getSignatureContext());
|
|
1205
1399
|
}
|
|
1206
1400
|
|
|
1207
1401
|
private async dropFailedTxsFromP2P(failedTxs: FailedTx[]) {
|
package/src/sequencer/metrics.ts
CHANGED
|
@@ -26,6 +26,7 @@ export class SequencerMetrics {
|
|
|
26
26
|
private blockBuildDuration: Histogram;
|
|
27
27
|
private blockBuildManaPerSecond: Gauge;
|
|
28
28
|
private stateTransitionBufferDuration: Histogram;
|
|
29
|
+
private stateDuration: Histogram;
|
|
29
30
|
|
|
30
31
|
// these are gauges because for individual sequencers building a block is not something that happens often enough to warrant a histogram
|
|
31
32
|
private timeToCollectAttestations: Gauge;
|
|
@@ -90,6 +91,8 @@ export class SequencerMetrics {
|
|
|
90
91
|
|
|
91
92
|
this.stateTransitionBufferDuration = this.meter.createHistogram(Metrics.SEQUENCER_STATE_TRANSITION_BUFFER_DURATION);
|
|
92
93
|
|
|
94
|
+
this.stateDuration = this.meter.createHistogram(Metrics.SEQUENCER_STATE_DURATION);
|
|
95
|
+
|
|
93
96
|
this.rewards = this.meter.createGauge(Metrics.SEQUENCER_CURRENT_SLOT_REWARDS);
|
|
94
97
|
|
|
95
98
|
this.slots = createUpDownCounterWithDefault(this.meter, Metrics.SEQUENCER_SLOT_COUNT);
|
|
@@ -252,6 +255,12 @@ export class SequencerMetrics {
|
|
|
252
255
|
});
|
|
253
256
|
}
|
|
254
257
|
|
|
258
|
+
recordStateDuration(durationMs: number, state: SequencerState) {
|
|
259
|
+
this.stateDuration.record(Math.ceil(durationMs), {
|
|
260
|
+
[Attributes.SEQUENCER_STATE]: state,
|
|
261
|
+
});
|
|
262
|
+
}
|
|
263
|
+
|
|
255
264
|
recordPipelineDepth(depth: number) {
|
|
256
265
|
this.pipelineDepth.record(depth);
|
|
257
266
|
}
|
|
@@ -6,7 +6,7 @@ import { BlockNumber, CheckpointNumber, EpochNumber, SlotNumber } from '@aztec/f
|
|
|
6
6
|
import { merge, omit, pick } from '@aztec/foundation/collection';
|
|
7
7
|
import { Fr } from '@aztec/foundation/curves/bn254';
|
|
8
8
|
import { EthAddress } from '@aztec/foundation/eth-address';
|
|
9
|
-
import { createLogger } from '@aztec/foundation/log';
|
|
9
|
+
import { type Logger, createLogger } from '@aztec/foundation/log';
|
|
10
10
|
import { RunningPromise } from '@aztec/foundation/running-promise';
|
|
11
11
|
import type { DateProvider } from '@aztec/foundation/timer';
|
|
12
12
|
import type { TypedEventEmitter } from '@aztec/foundation/types';
|
|
@@ -14,6 +14,7 @@ import type { P2P } from '@aztec/p2p';
|
|
|
14
14
|
import type { SlasherClientInterface } from '@aztec/slasher';
|
|
15
15
|
import type { BlockData, L2BlockSink, L2BlockSource, ValidateCheckpointResult } from '@aztec/stdlib/block';
|
|
16
16
|
import type { Checkpoint, ProposedCheckpointData } from '@aztec/stdlib/checkpoint';
|
|
17
|
+
import type { ChainConfig } from '@aztec/stdlib/config';
|
|
17
18
|
import { getSlotStartBuildTimestamp } from '@aztec/stdlib/epoch-helpers';
|
|
18
19
|
import {
|
|
19
20
|
type ResolvedSequencerConfig,
|
|
@@ -22,6 +23,7 @@ import {
|
|
|
22
23
|
type WorldStateSynchronizer,
|
|
23
24
|
} from '@aztec/stdlib/interfaces/server';
|
|
24
25
|
import type { L1ToL2MessageSource } from '@aztec/stdlib/messaging';
|
|
26
|
+
import type { CoordinationSignatureContext } from '@aztec/stdlib/p2p';
|
|
25
27
|
import { pickFromSchema } from '@aztec/stdlib/schemas';
|
|
26
28
|
import { MerkleTreeId } from '@aztec/stdlib/trees';
|
|
27
29
|
import { Attributes, type TelemetryClient, type Tracer, getTelemetryClient, trackSpan } from '@aztec/telemetry-client';
|
|
@@ -56,8 +58,11 @@ export { SequencerState };
|
|
|
56
58
|
export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<SequencerEvents>) {
|
|
57
59
|
private runningPromise?: RunningPromise;
|
|
58
60
|
private state = SequencerState.STOPPED;
|
|
61
|
+
private stateSlotNumber: SlotNumber | undefined;
|
|
62
|
+
private stateEnteredAtMs = performance.now();
|
|
59
63
|
private metrics: SequencerMetrics;
|
|
60
64
|
private checkpointProposalJobMetrics: CheckpointProposalJobMetrics;
|
|
65
|
+
private readonly stateLog: Logger;
|
|
61
66
|
|
|
62
67
|
/** The last slot for which we attempted to perform our voting duties with degraded block production */
|
|
63
68
|
private lastSlotForFallbackVote: SlotNumber | undefined;
|
|
@@ -82,6 +87,7 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
82
87
|
|
|
83
88
|
/** Config for the sequencer */
|
|
84
89
|
protected config: ResolvedSequencerConfig = DefaultSequencerConfig;
|
|
90
|
+
private readonly signatureContext: CoordinationSignatureContext;
|
|
85
91
|
|
|
86
92
|
constructor(
|
|
87
93
|
protected publisherFactory: SequencerPublisherFactory,
|
|
@@ -97,17 +103,22 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
97
103
|
protected dateProvider: DateProvider,
|
|
98
104
|
protected epochCache: EpochCache,
|
|
99
105
|
protected rollupContract: RollupContract,
|
|
100
|
-
config: SequencerConfig,
|
|
106
|
+
config: SequencerConfig & Pick<ChainConfig, 'l1ChainId' | 'l1Contracts'>,
|
|
101
107
|
protected telemetry: TelemetryClient = getTelemetryClient(),
|
|
102
108
|
protected log = createLogger('sequencer'),
|
|
103
109
|
) {
|
|
104
110
|
super();
|
|
111
|
+
this.stateLog = log.createChild('state');
|
|
105
112
|
|
|
106
113
|
// Add [FISHERMAN] prefix to logger if in fisherman mode
|
|
107
114
|
if (config.fishermanMode) {
|
|
108
115
|
this.log = log.createChild('[FISHERMAN]');
|
|
109
116
|
}
|
|
110
117
|
|
|
118
|
+
this.signatureContext = {
|
|
119
|
+
chainId: config.l1ChainId,
|
|
120
|
+
rollupAddress: config.l1Contracts.rollupAddress,
|
|
121
|
+
};
|
|
111
122
|
this.metrics = new SequencerMetrics(telemetry, this.rollupContract, 'Sequencer');
|
|
112
123
|
this.checkpointProposalJobMetrics = new CheckpointProposalJobMetrics(telemetry);
|
|
113
124
|
this.updateConfig(config);
|
|
@@ -377,7 +388,7 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
377
388
|
// and the archive at that checkpoint so L1 simulation sees the correct chain tip.
|
|
378
389
|
const parentCheckpointNumber = CheckpointNumber(checkpointNumber - 1);
|
|
379
390
|
l1SimulationOverridesBuilder.forPendingCheckpoint(parentCheckpointNumber).withPendingArchive(syncedTo.archive);
|
|
380
|
-
this.metrics.recordPipelineDepth(
|
|
391
|
+
this.metrics.recordPipelineDepth(syncedTo.checkpointNumber - syncedTo.checkpointedCheckpointNumber);
|
|
381
392
|
|
|
382
393
|
this.log.verbose(
|
|
383
394
|
`Building on top of proposed checkpoint (pending=${syncedTo.proposedCheckpointData?.checkpointNumber})`,
|
|
@@ -489,6 +500,7 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
489
500
|
this.checkpointsBuilder,
|
|
490
501
|
this.l2BlockSource,
|
|
491
502
|
this.l1Constants,
|
|
503
|
+
this.signatureContext,
|
|
492
504
|
this.config,
|
|
493
505
|
this.timetable,
|
|
494
506
|
this.slasherClient,
|
|
@@ -536,19 +548,35 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
536
548
|
this.timetable.assertTimeLeft(proposedState, secondsIntoSlot);
|
|
537
549
|
}
|
|
538
550
|
|
|
551
|
+
const oldState = this.state;
|
|
552
|
+
const oldStateSlotNumber = this.stateSlotNumber;
|
|
553
|
+
const stateChanged = proposedState !== oldState;
|
|
554
|
+
const transitionAtMs = performance.now();
|
|
555
|
+
const stateDurationMs = transitionAtMs - this.stateEnteredAtMs;
|
|
556
|
+
|
|
539
557
|
const boringStates = [SequencerState.IDLE, SequencerState.SYNCHRONIZING];
|
|
540
558
|
const logLevel =
|
|
541
|
-
boringStates.includes(proposedState) && boringStates.includes(
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
559
|
+
boringStates.includes(proposedState) && boringStates.includes(oldState) ? ('trace' as const) : ('debug' as const);
|
|
560
|
+
this.stateLog[logLevel](`Transitioning from ${oldState} to ${proposedState}`, {
|
|
561
|
+
oldState,
|
|
562
|
+
newState: proposedState,
|
|
563
|
+
slotNumber,
|
|
564
|
+
stateSlotNumber: oldStateSlotNumber,
|
|
565
|
+
secondsIntoSlot,
|
|
566
|
+
...(stateChanged && { stateDurationMs: Math.ceil(stateDurationMs) }),
|
|
567
|
+
});
|
|
545
568
|
|
|
546
569
|
this.emit('state-changed', {
|
|
547
|
-
oldState
|
|
570
|
+
oldState,
|
|
548
571
|
newState: proposedState,
|
|
549
572
|
secondsIntoSlot,
|
|
550
573
|
slot: slotNumber,
|
|
551
574
|
});
|
|
575
|
+
if (stateChanged) {
|
|
576
|
+
this.metrics.recordStateDuration(stateDurationMs, oldState);
|
|
577
|
+
this.stateEnteredAtMs = transitionAtMs;
|
|
578
|
+
this.stateSlotNumber = slotNumber;
|
|
579
|
+
}
|
|
552
580
|
this.state = proposedState;
|
|
553
581
|
}
|
|
554
582
|
|
|
@@ -581,7 +609,7 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
581
609
|
this.p2pClient.getStatus().then(p2p => p2p.syncedToL2Block),
|
|
582
610
|
this.l1ToL2MessageSource.getL2Tips().then(t => ({ proposed: t.proposed, checkpointed: t.checkpointed })),
|
|
583
611
|
this.l2BlockSource.getPendingChainValidationStatus(),
|
|
584
|
-
this.l2BlockSource.
|
|
612
|
+
this.l2BlockSource.getLastProposedCheckpoint(),
|
|
585
613
|
] as const);
|
|
586
614
|
|
|
587
615
|
const [worldState, l2Tips, p2p, l1ToL2MessageSourceTips, pendingChainValidationStatus, proposedCheckpointData] =
|
package/src/test/utils.ts
CHANGED
|
@@ -10,7 +10,11 @@ import { PublicDataWrite } from '@aztec/stdlib/avm';
|
|
|
10
10
|
import { CommitteeAttestation, L2Block } from '@aztec/stdlib/block';
|
|
11
11
|
import { BlockProposal, CheckpointAttestation, CheckpointProposal, ConsensusPayload } from '@aztec/stdlib/p2p';
|
|
12
12
|
import { CheckpointHeader } from '@aztec/stdlib/rollup';
|
|
13
|
-
import {
|
|
13
|
+
import {
|
|
14
|
+
TEST_COORDINATION_SIGNATURE_CONTEXT,
|
|
15
|
+
makeAppendOnlyTreeSnapshot,
|
|
16
|
+
mockTxForRollup,
|
|
17
|
+
} from '@aztec/stdlib/testing';
|
|
14
18
|
import { BlockHeader, GlobalVariables, type Tx, makeProcessedTxFromPrivateOnlyTx } from '@aztec/stdlib/tx';
|
|
15
19
|
|
|
16
20
|
import type { MockProxy } from 'jest-mock-extended';
|
|
@@ -109,6 +113,7 @@ export function createBlockProposal(block: L2Block, signature: Signature): Block
|
|
|
109
113
|
block.archive.root,
|
|
110
114
|
txHashes,
|
|
111
115
|
signature,
|
|
116
|
+
TEST_COORDINATION_SIGNATURE_CONTEXT,
|
|
112
117
|
);
|
|
113
118
|
}
|
|
114
119
|
|
|
@@ -123,12 +128,19 @@ export function createCheckpointProposal(
|
|
|
123
128
|
): CheckpointProposal {
|
|
124
129
|
const txHashes = block.body.txEffects.map(tx => tx.txHash);
|
|
125
130
|
const checkpointHeader = createCheckpointHeaderFromBlock(block);
|
|
126
|
-
return new CheckpointProposal(
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
131
|
+
return new CheckpointProposal(
|
|
132
|
+
checkpointHeader,
|
|
133
|
+
block.archive.root,
|
|
134
|
+
feeAssetPriceModifier,
|
|
135
|
+
checkpointSignature,
|
|
136
|
+
TEST_COORDINATION_SIGNATURE_CONTEXT,
|
|
137
|
+
{
|
|
138
|
+
blockHeader: block.header,
|
|
139
|
+
indexWithinCheckpoint: block.indexWithinCheckpoint,
|
|
140
|
+
txHashes,
|
|
141
|
+
signature: blockSignature ?? checkpointSignature, // Use checkpoint signature as block signature if not provided
|
|
142
|
+
},
|
|
143
|
+
);
|
|
132
144
|
}
|
|
133
145
|
|
|
134
146
|
/**
|
|
@@ -143,10 +155,16 @@ export function createCheckpointAttestation(
|
|
|
143
155
|
feeAssetPriceModifier: bigint = 0n,
|
|
144
156
|
): CheckpointAttestation {
|
|
145
157
|
const checkpointHeader = createCheckpointHeaderFromBlock(block);
|
|
146
|
-
const payload = new ConsensusPayload(
|
|
158
|
+
const payload = new ConsensusPayload(
|
|
159
|
+
checkpointHeader,
|
|
160
|
+
block.archive.root,
|
|
161
|
+
feeAssetPriceModifier,
|
|
162
|
+
TEST_COORDINATION_SIGNATURE_CONTEXT,
|
|
163
|
+
);
|
|
147
164
|
const attestation = new CheckpointAttestation(payload, signature, signature);
|
|
148
|
-
//
|
|
149
|
-
(attestation as any).
|
|
165
|
+
// Bypass signature recovery for testing since we use random signatures
|
|
166
|
+
(attestation as any).getSender = () => sender;
|
|
167
|
+
(attestation as any).getProposer = () => sender;
|
|
150
168
|
return attestation;
|
|
151
169
|
}
|
|
152
170
|
|