@aztec/sequencer-client 0.0.1-commit.b6e433891 → 0.0.1-commit.b9865e97
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +281 -21
- package/dest/client/sequencer-client.d.ts +8 -3
- package/dest/client/sequencer-client.d.ts.map +1 -1
- package/dest/client/sequencer-client.js +17 -23
- package/dest/config.d.ts +6 -3
- package/dest/config.d.ts.map +1 -1
- package/dest/config.js +35 -15
- package/dest/global_variable_builder/fee_predictor.d.ts +37 -0
- package/dest/global_variable_builder/fee_predictor.d.ts.map +1 -0
- package/dest/global_variable_builder/fee_predictor.js +128 -0
- package/dest/global_variable_builder/fee_provider.d.ts +21 -0
- package/dest/global_variable_builder/fee_provider.d.ts.map +1 -0
- package/dest/global_variable_builder/fee_provider.js +58 -0
- package/dest/global_variable_builder/global_builder.d.ts +14 -14
- package/dest/global_variable_builder/global_builder.d.ts.map +1 -1
- package/dest/global_variable_builder/global_builder.js +16 -51
- package/dest/global_variable_builder/index.d.ts +4 -2
- package/dest/global_variable_builder/index.d.ts.map +1 -1
- package/dest/global_variable_builder/index.js +2 -0
- package/dest/index.d.ts +2 -2
- package/dest/index.d.ts.map +1 -1
- package/dest/index.js +1 -1
- package/dest/publisher/config.d.ts +15 -3
- package/dest/publisher/config.d.ts.map +1 -1
- package/dest/publisher/config.js +19 -4
- package/dest/publisher/l1_tx_failed_store/failed_tx_store.d.ts +3 -4
- package/dest/publisher/l1_tx_failed_store/failed_tx_store.d.ts.map +1 -1
- package/dest/publisher/sequencer-bundle-simulator.d.ts +96 -0
- package/dest/publisher/sequencer-bundle-simulator.d.ts.map +1 -0
- package/dest/publisher/sequencer-bundle-simulator.js +198 -0
- package/dest/publisher/sequencer-publisher-factory.d.ts +3 -5
- package/dest/publisher/sequencer-publisher-factory.d.ts.map +1 -1
- package/dest/publisher/sequencer-publisher-factory.js +2 -3
- package/dest/publisher/sequencer-publisher.d.ts +70 -64
- package/dest/publisher/sequencer-publisher.d.ts.map +1 -1
- package/dest/publisher/sequencer-publisher.js +312 -533
- package/dest/sequencer/automine/automine_factory.d.ts +54 -0
- package/dest/sequencer/automine/automine_factory.d.ts.map +1 -0
- package/dest/sequencer/automine/automine_factory.js +84 -0
- package/dest/sequencer/automine/automine_sequencer.d.ts +151 -0
- package/dest/sequencer/automine/automine_sequencer.d.ts.map +1 -0
- package/dest/sequencer/automine/automine_sequencer.js +509 -0
- package/dest/sequencer/chain_state_overrides.d.ts +61 -0
- package/dest/sequencer/chain_state_overrides.d.ts.map +1 -0
- package/dest/sequencer/chain_state_overrides.js +98 -0
- package/dest/sequencer/checkpoint_proposal_job.d.ts +55 -13
- package/dest/sequencer/checkpoint_proposal_job.d.ts.map +1 -1
- package/dest/sequencer/checkpoint_proposal_job.js +668 -164
- package/dest/sequencer/checkpoint_proposal_job_metrics.d.ts +34 -0
- package/dest/sequencer/checkpoint_proposal_job_metrics.d.ts.map +1 -0
- package/dest/sequencer/checkpoint_proposal_job_metrics.js +72 -0
- package/dest/sequencer/checkpoint_voter.d.ts +1 -2
- package/dest/sequencer/checkpoint_voter.d.ts.map +1 -1
- package/dest/sequencer/checkpoint_voter.js +2 -5
- package/dest/sequencer/events.d.ts +47 -2
- package/dest/sequencer/events.d.ts.map +1 -1
- package/dest/sequencer/index.d.ts +3 -1
- package/dest/sequencer/index.d.ts.map +1 -1
- package/dest/sequencer/index.js +2 -0
- package/dest/sequencer/metrics.d.ts +9 -10
- package/dest/sequencer/metrics.d.ts.map +1 -1
- package/dest/sequencer/metrics.js +34 -20
- package/dest/sequencer/sequencer.d.ts +59 -11
- package/dest/sequencer/sequencer.d.ts.map +1 -1
- package/dest/sequencer/sequencer.js +304 -74
- package/dest/sequencer/timetable.d.ts +11 -1
- package/dest/sequencer/timetable.d.ts.map +1 -1
- package/dest/sequencer/timetable.js +41 -36
- package/dest/sequencer/types.d.ts +2 -2
- package/dest/sequencer/types.d.ts.map +1 -1
- package/dest/test/utils.d.ts +1 -1
- package/dest/test/utils.d.ts.map +1 -1
- package/dest/test/utils.js +7 -6
- package/package.json +27 -27
- package/src/client/sequencer-client.ts +28 -31
- package/src/config.ts +40 -15
- package/src/global_variable_builder/README.md +44 -0
- package/src/global_variable_builder/fee_predictor.ts +172 -0
- package/src/global_variable_builder/fee_provider.ts +75 -0
- package/src/global_variable_builder/global_builder.ts +25 -63
- package/src/global_variable_builder/index.ts +3 -1
- package/src/index.ts +10 -1
- package/src/publisher/config.ts +40 -6
- package/src/publisher/l1_tx_failed_store/failed_tx_store.ts +3 -1
- package/src/publisher/sequencer-bundle-simulator.ts +254 -0
- package/src/publisher/sequencer-publisher-factory.ts +3 -6
- package/src/publisher/sequencer-publisher.ts +368 -577
- package/src/sequencer/README.md +162 -450
- package/src/sequencer/automine/README.md +46 -0
- package/src/sequencer/automine/automine_factory.ts +145 -0
- package/src/sequencer/automine/automine_sequencer.ts +595 -0
- package/src/sequencer/chain_state_overrides.ts +162 -0
- package/src/sequencer/checkpoint_proposal_job.ts +769 -185
- package/src/sequencer/checkpoint_proposal_job_metrics.ts +128 -0
- package/src/sequencer/checkpoint_voter.ts +1 -12
- package/src/sequencer/events.ts +51 -2
- package/src/sequencer/index.ts +2 -0
- package/src/sequencer/metrics.ts +43 -24
- package/src/sequencer/sequencer.ts +360 -83
- package/src/sequencer/timetable.ts +49 -44
- package/src/sequencer/types.ts +1 -1
- package/src/test/utils.ts +28 -10
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import type { EpochCache } from '@aztec/epoch-cache';
|
|
2
|
+
import type { SimulationOverridesPlan } from '@aztec/ethereum/contracts';
|
|
2
3
|
import {
|
|
3
4
|
BlockNumber,
|
|
4
5
|
CheckpointNumber,
|
|
@@ -13,11 +14,12 @@ import {
|
|
|
13
14
|
generateUnrecoverableSignature,
|
|
14
15
|
} from '@aztec/foundation/crypto/secp256k1-signer';
|
|
15
16
|
import { Fr } from '@aztec/foundation/curves/bn254';
|
|
17
|
+
import { InterruptError, TimeoutError } from '@aztec/foundation/error';
|
|
16
18
|
import { EthAddress } from '@aztec/foundation/eth-address';
|
|
17
19
|
import { Signature } from '@aztec/foundation/eth-signature';
|
|
18
20
|
import { filter } from '@aztec/foundation/iterator';
|
|
19
21
|
import { type Logger, type LoggerBindings, createLogger } from '@aztec/foundation/log';
|
|
20
|
-
import {
|
|
22
|
+
import { InterruptibleSleep } from '@aztec/foundation/sleep';
|
|
21
23
|
import { type DateProvider, Timer } from '@aztec/foundation/timer';
|
|
22
24
|
import { type TypedEventEmitter, isErrorClass, unfreeze } from '@aztec/foundation/types';
|
|
23
25
|
import type { P2P } from '@aztec/p2p';
|
|
@@ -29,9 +31,16 @@ import {
|
|
|
29
31
|
type L2BlockSink,
|
|
30
32
|
type L2BlockSource,
|
|
31
33
|
MaliciousCommitteeAttestationsAndSigners,
|
|
34
|
+
type ProposedCheckpointSink,
|
|
35
|
+
type ValidateCheckpointResult,
|
|
32
36
|
} from '@aztec/stdlib/block';
|
|
33
|
-
import {
|
|
34
|
-
|
|
37
|
+
import {
|
|
38
|
+
type Checkpoint,
|
|
39
|
+
type ProposedCheckpointData,
|
|
40
|
+
getPreviousCheckpointOutHashes,
|
|
41
|
+
validateCheckpoint,
|
|
42
|
+
} from '@aztec/stdlib/checkpoint';
|
|
43
|
+
import { computeQuorum, getSlotStartBuildTimestamp, getTimestampForSlot } from '@aztec/stdlib/epoch-helpers';
|
|
35
44
|
import { Gas } from '@aztec/stdlib/gas';
|
|
36
45
|
import {
|
|
37
46
|
type BlockBuilderOptions,
|
|
@@ -43,8 +52,10 @@ import { type L1ToL2MessageSource, computeInHashFromL1ToL2Messages } from '@azte
|
|
|
43
52
|
import type {
|
|
44
53
|
BlockProposal,
|
|
45
54
|
BlockProposalOptions,
|
|
55
|
+
CheckpointAttestation,
|
|
46
56
|
CheckpointProposal,
|
|
47
57
|
CheckpointProposalOptions,
|
|
58
|
+
CoordinationSignatureContext,
|
|
48
59
|
} from '@aztec/stdlib/p2p';
|
|
49
60
|
import { orderAttestations, trimAttestations } from '@aztec/stdlib/p2p';
|
|
50
61
|
import type { L2BlockBuiltStats } from '@aztec/stdlib/stats';
|
|
@@ -56,6 +67,8 @@ import { DutyAlreadySignedError, SlashingProtectionError } from '@aztec/validato
|
|
|
56
67
|
|
|
57
68
|
import type { GlobalVariableBuilder } from '../global_variable_builder/global_builder.js';
|
|
58
69
|
import type { InvalidateCheckpointRequest, SequencerPublisher } from '../publisher/sequencer-publisher.js';
|
|
70
|
+
import { buildCheckpointSimulationOverridesPlan } from './chain_state_overrides.js';
|
|
71
|
+
import type { CheckpointProposalJobMetricsRecorder } from './checkpoint_proposal_job_metrics.js';
|
|
59
72
|
import { CheckpointVoter } from './checkpoint_voter.js';
|
|
60
73
|
import { SequencerInterruptedError } from './errors.js';
|
|
61
74
|
import type { SequencerEvents } from './events.js';
|
|
@@ -66,6 +79,21 @@ import { SequencerState } from './utils.js';
|
|
|
66
79
|
|
|
67
80
|
/** How much time to sleep while waiting for min transactions to accumulate for a block */
|
|
68
81
|
const TXS_POLLING_MS = 500;
|
|
82
|
+
const ARCHIVER_SYNC_POLLING_MS = 200;
|
|
83
|
+
|
|
84
|
+
/** Result from proposeCheckpoint when a checkpoint was successfully built and broadcast. */
|
|
85
|
+
type CheckpointProposalBroadcast = {
|
|
86
|
+
checkpoint: Checkpoint;
|
|
87
|
+
proposal: CheckpointProposal;
|
|
88
|
+
blockProposedAt: number;
|
|
89
|
+
};
|
|
90
|
+
|
|
91
|
+
/** Result after attestation collection and signing, ready for L1 submission. */
|
|
92
|
+
type CheckpointProposalResult = {
|
|
93
|
+
checkpoint: Checkpoint;
|
|
94
|
+
attestations: CommitteeAttestationsAndSigners;
|
|
95
|
+
attestationsSignature: Signature;
|
|
96
|
+
};
|
|
69
97
|
|
|
70
98
|
/**
|
|
71
99
|
* Handles the execution of a checkpoint proposal after the initial preparation phase.
|
|
@@ -75,14 +103,32 @@ const TXS_POLLING_MS = 500;
|
|
|
75
103
|
*/
|
|
76
104
|
export class CheckpointProposalJob implements Traceable {
|
|
77
105
|
protected readonly log: Logger;
|
|
106
|
+
private readonly checkpointEventLog: Logger;
|
|
107
|
+
|
|
108
|
+
/** Tracks the fire-and-forget L1 submission promise so it can be awaited during shutdown. */
|
|
109
|
+
private pendingL1Submission: Promise<void> | undefined;
|
|
110
|
+
private readonly interruptibleSleep = new InterruptibleSleep();
|
|
111
|
+
private interrupted = false;
|
|
112
|
+
|
|
113
|
+
/**
|
|
114
|
+
* Chain state overrides built once per slot in proposeCheckpoint after the checkpoint is
|
|
115
|
+
* complete. Carries the pending parent override (archive + slot + fee header) for pipelining,
|
|
116
|
+
* or the invalidation pending override when rolling back. Consumed by
|
|
117
|
+
* publisher.validateBlockHeader before broadcast.
|
|
118
|
+
*/
|
|
119
|
+
private checkpointSimulationOverridesPlan?: SimulationOverridesPlan;
|
|
120
|
+
|
|
121
|
+
private getSignatureContext(): CoordinationSignatureContext {
|
|
122
|
+
return this.signatureContext;
|
|
123
|
+
}
|
|
78
124
|
|
|
79
125
|
constructor(
|
|
80
126
|
private readonly slotNow: SlotNumber,
|
|
81
127
|
private readonly targetSlot: SlotNumber,
|
|
82
|
-
private readonly epochNow: EpochNumber,
|
|
83
128
|
private readonly targetEpoch: EpochNumber,
|
|
84
129
|
private readonly checkpointNumber: CheckpointNumber,
|
|
85
130
|
private readonly syncedToBlockNumber: BlockNumber,
|
|
131
|
+
private readonly checkpointedCheckpointNumber: CheckpointNumber,
|
|
86
132
|
// TODO(palla/mbps): Can we remove the proposer in favor of attestorAddress? Need to check fisherman-node flows.
|
|
87
133
|
private readonly proposer: EthAddress | undefined,
|
|
88
134
|
private readonly publisher: SequencerPublisher,
|
|
@@ -95,38 +141,75 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
95
141
|
private readonly l1ToL2MessageSource: L1ToL2MessageSource,
|
|
96
142
|
private readonly l2BlockSource: L2BlockSource,
|
|
97
143
|
private readonly checkpointsBuilder: FullNodeCheckpointsBuilder,
|
|
98
|
-
private readonly blockSink: L2BlockSink,
|
|
144
|
+
private readonly blockSink: L2BlockSink & ProposedCheckpointSink,
|
|
99
145
|
private readonly l1Constants: SequencerRollupConstants,
|
|
146
|
+
private readonly signatureContext: CoordinationSignatureContext,
|
|
100
147
|
protected config: ResolvedSequencerConfig,
|
|
101
148
|
protected timetable: SequencerTimetable,
|
|
102
149
|
private readonly slasherClient: SlasherClientInterface | undefined,
|
|
103
150
|
private readonly epochCache: EpochCache,
|
|
104
151
|
private readonly dateProvider: DateProvider,
|
|
105
152
|
private readonly metrics: SequencerMetrics,
|
|
106
|
-
private readonly
|
|
107
|
-
|
|
153
|
+
private readonly checkpointMetrics: CheckpointProposalJobMetricsRecorder,
|
|
154
|
+
protected readonly eventEmitter: TypedEventEmitter<SequencerEvents>,
|
|
155
|
+
private readonly setStateFn: (state: SequencerState, slot?: SlotNumber, timeReferenceSlot?: SlotNumber) => void,
|
|
108
156
|
public readonly tracer: Tracer,
|
|
109
157
|
bindings?: LoggerBindings,
|
|
158
|
+
private readonly proposedCheckpointData?: ProposedCheckpointData,
|
|
110
159
|
) {
|
|
111
160
|
this.log = createLogger('sequencer:checkpoint-proposal', {
|
|
112
161
|
...bindings,
|
|
113
162
|
instanceId: `slot-${this.slotNow}`,
|
|
114
163
|
});
|
|
164
|
+
this.checkpointEventLog = createLogger('sequencer:checkpoint-events', {
|
|
165
|
+
...bindings,
|
|
166
|
+
instanceId: `slot-${this.slotNow}`,
|
|
167
|
+
});
|
|
115
168
|
}
|
|
116
169
|
|
|
117
|
-
/**
|
|
118
|
-
|
|
119
|
-
|
|
170
|
+
/** Awaits the pending L1 submission if one is in progress. Call during shutdown. */
|
|
171
|
+
public async awaitPendingSubmission(): Promise<void> {
|
|
172
|
+
this.log.info('Awaiting pending L1 payload submission');
|
|
173
|
+
await this.pendingL1Submission;
|
|
120
174
|
}
|
|
121
175
|
|
|
122
|
-
/**
|
|
123
|
-
|
|
124
|
-
|
|
176
|
+
/** Interrupts job-owned waits, including the publisher's send-at-slot sleep, so shutdown can finish. */
|
|
177
|
+
public interrupt(): void {
|
|
178
|
+
this.interrupted = true;
|
|
179
|
+
this.interruptibleSleep.interrupt(true);
|
|
180
|
+
this.publisher.interrupt();
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
private async awaitInterruptibleSleep(ms: number): Promise<void> {
|
|
184
|
+
if (this.interrupted) {
|
|
185
|
+
throw new SequencerInterruptedError();
|
|
186
|
+
}
|
|
187
|
+
if (ms <= 0) {
|
|
188
|
+
return;
|
|
189
|
+
}
|
|
190
|
+
try {
|
|
191
|
+
await this.interruptibleSleep.sleep(ms);
|
|
192
|
+
} catch (err) {
|
|
193
|
+
if (err instanceof InterruptError) {
|
|
194
|
+
throw new SequencerInterruptedError();
|
|
195
|
+
}
|
|
196
|
+
throw err;
|
|
197
|
+
}
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
private logCheckpointEvent(eventName: string, message: string, fields: Record<string, unknown>): void {
|
|
201
|
+
this.checkpointEventLog.debug(message, {
|
|
202
|
+
eventName: `sequencer-checkpoint-${eventName}`,
|
|
203
|
+
...fields,
|
|
204
|
+
});
|
|
125
205
|
}
|
|
126
206
|
|
|
127
207
|
/**
|
|
128
208
|
* Executes the checkpoint proposal job.
|
|
129
|
-
*
|
|
209
|
+
* Builds blocks, assembles checkpoint, and broadcasts the proposal (blocking).
|
|
210
|
+
* Attestation collection, signing, and L1 submission are backgrounded so the
|
|
211
|
+
* work loop can return to IDLE immediately for consecutive slot proposals.
|
|
212
|
+
* Returns the built checkpoint if successful, undefined otherwise.
|
|
130
213
|
*/
|
|
131
214
|
@trackSpan('CheckpointProposalJob.execute')
|
|
132
215
|
public async execute(): Promise<Checkpoint | undefined> {
|
|
@@ -145,56 +228,302 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
145
228
|
this.log,
|
|
146
229
|
).enqueueVotes();
|
|
147
230
|
|
|
148
|
-
// Build
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
231
|
+
// Build blocks, assemble checkpoint, and broadcast proposal (BLOCKING).
|
|
232
|
+
// Returns after broadcast — attestation collection is deferred.
|
|
233
|
+
const broadcast = await this.proposeCheckpoint();
|
|
234
|
+
|
|
235
|
+
if (!broadcast) {
|
|
236
|
+
await Promise.all(votesPromises);
|
|
237
|
+
// Still submit votes even without a checkpoint.
|
|
238
|
+
// Under proposer pipelining, vote-offenses signatures are EIP-712-bound to `targetSlot`
|
|
239
|
+
// (the pipelined slot in which the multicall is expected to mine). Submitting at the
|
|
240
|
+
// wall-clock time would let the multicall mine in a different L2 slot, causing
|
|
241
|
+
// signature verification to fail silently inside Multicall3. Delay submission to the
|
|
242
|
+
// start of `targetSlot` so the tx mines in the slot the vote was signed for.
|
|
243
|
+
if (!this.config.fishermanMode) {
|
|
244
|
+
this.pendingL1Submission = this.publisher.sendRequestsAt(this.targetSlot).then(() => {});
|
|
245
|
+
}
|
|
246
|
+
return undefined;
|
|
156
247
|
}
|
|
157
248
|
|
|
249
|
+
const { checkpoint } = broadcast;
|
|
250
|
+
this.metrics.recordCheckpointProposalSuccess();
|
|
251
|
+
|
|
158
252
|
// Do not post anything to L1 if we are fishermen, but do perform L1 fee analysis
|
|
159
253
|
if (this.config.fishermanMode) {
|
|
160
254
|
await this.handleCheckpointEndAsFisherman(checkpoint);
|
|
161
|
-
return;
|
|
255
|
+
return checkpoint;
|
|
162
256
|
}
|
|
163
257
|
|
|
164
|
-
//
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
258
|
+
// Background the attestation → signing → L1 pipeline so the work loop is unblocked
|
|
259
|
+
this.pendingL1Submission = this.waitForAttestationsAndEnqueueSubmissionAsync(broadcast, votesPromises);
|
|
260
|
+
|
|
261
|
+
// Return the built checkpoint immediately — the work loop is now unblocked
|
|
262
|
+
return checkpoint;
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
/**
|
|
266
|
+
* Background pipeline: collects attestations, signs them, enqueues the checkpoint, and submits to L1.
|
|
267
|
+
* Runs as a fire-and-forget task stored in `pendingL1Submission` so the work loop is unblocked.
|
|
268
|
+
*/
|
|
269
|
+
private async waitForAttestationsAndEnqueueSubmissionAsync(
|
|
270
|
+
broadcast: CheckpointProposalBroadcast,
|
|
271
|
+
votesPromises: Promise<unknown>[],
|
|
272
|
+
): Promise<void> {
|
|
273
|
+
const { checkpoint } = broadcast;
|
|
274
|
+
|
|
275
|
+
try {
|
|
276
|
+
// Wait for all votes actions, enqueued at the beginning, to resolve
|
|
277
|
+
await Promise.all(votesPromises);
|
|
278
|
+
|
|
279
|
+
// Try to collect attestations from the committee
|
|
280
|
+
const signedAttestations = await this.getSignedCommitteeAttestations(broadcast);
|
|
281
|
+
|
|
282
|
+
// Wait for the previous checkpoint to land on L1 before submitting, so we can check it
|
|
283
|
+
// matches the proposed checkpoint we used as parent, and has valid attestations.
|
|
284
|
+
if (signedAttestations && (await this.waitForValidParentCheckpointOnL1())) {
|
|
285
|
+
await this.enqueueCheckpointForSubmission({ checkpoint, ...signedAttestations });
|
|
286
|
+
}
|
|
287
|
+
|
|
288
|
+
// If we failed to collect attestations, at least check if we need to issue an invalidation
|
|
289
|
+
if (!signedAttestations && (await this.waitForSyncedL2SlotNumber(this.slotNow))) {
|
|
290
|
+
const validationStatus = await this.l2BlockSource.getPendingChainValidationStatus();
|
|
291
|
+
if (!validationStatus.valid) {
|
|
292
|
+
this.log.warn(
|
|
293
|
+
`Checkpoint ${validationStatus.checkpoint.checkpointNumber} has invalid attestations, enqueuing invalidation in spite of attestation collection failure`,
|
|
294
|
+
{ checkpoint: validationStatus.checkpoint, reason: validationStatus.reason },
|
|
295
|
+
);
|
|
296
|
+
await this.enqueueInvalidation(validationStatus);
|
|
297
|
+
}
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
// Send whatever was enqueued: votes + (propose | invalidation | nothing).
|
|
301
|
+
const l1Response = await this.publisher.sendRequestsAt(this.targetSlot);
|
|
302
|
+
const proposedAction = l1Response?.successfulActions.find(a => a === 'propose');
|
|
303
|
+
if (proposedAction) {
|
|
304
|
+
this.logCheckpointEvent('published', `Checkpoint published for slot ${this.targetSlot}`, {
|
|
305
|
+
slot: this.targetSlot,
|
|
306
|
+
checkpointNumber: this.checkpointNumber,
|
|
307
|
+
successfulActions: l1Response?.successfulActions,
|
|
308
|
+
sentActions: l1Response?.sentActions,
|
|
309
|
+
});
|
|
310
|
+
this.eventEmitter.emit('checkpoint-published', { checkpoint: this.checkpointNumber, slot: this.targetSlot });
|
|
311
|
+
const coinbase = checkpoint.header.coinbase;
|
|
312
|
+
await this.metrics.incFilledSlot(this.publisher.getSenderAddress().toString(), coinbase);
|
|
313
|
+
} else {
|
|
314
|
+
this.logCheckpointEvent('publish-failed', `Checkpoint publish failed for slot ${this.targetSlot}`, {
|
|
315
|
+
slot: this.targetSlot,
|
|
316
|
+
checkpointNumber: this.checkpointNumber,
|
|
317
|
+
successfulActions: l1Response?.successfulActions,
|
|
318
|
+
failedActions: l1Response?.failedActions,
|
|
319
|
+
sentActions: l1Response?.sentActions,
|
|
320
|
+
expiredActions: l1Response?.expiredActions,
|
|
321
|
+
reason: 'propose_action_not_successful',
|
|
322
|
+
});
|
|
323
|
+
this.log.warn(`Checkpoint publish failed for slot ${this.targetSlot}`, {
|
|
324
|
+
slot: this.targetSlot,
|
|
325
|
+
checkpointNumber: this.checkpointNumber,
|
|
326
|
+
successfulActions: l1Response?.successfulActions,
|
|
327
|
+
failedActions: l1Response?.failedActions,
|
|
328
|
+
sentActions: l1Response?.sentActions,
|
|
329
|
+
expiredActions: l1Response?.expiredActions,
|
|
330
|
+
reason: 'propose_action_not_successful',
|
|
331
|
+
});
|
|
332
|
+
this.eventEmitter.emit('checkpoint-publish-failed', { ...l1Response, slot: this.targetSlot });
|
|
333
|
+
this.metrics.recordPipelineDiscard();
|
|
334
|
+
}
|
|
335
|
+
} catch (err) {
|
|
336
|
+
if (err instanceof SequencerInterruptedError) {
|
|
337
|
+
return;
|
|
338
|
+
}
|
|
339
|
+
this.logCheckpointEvent('publish-failed', `Checkpoint publish failed for slot ${this.targetSlot}`, {
|
|
340
|
+
slot: this.targetSlot,
|
|
341
|
+
checkpointNumber: this.checkpointNumber,
|
|
342
|
+
reason: err instanceof Error ? err.message : String(err),
|
|
343
|
+
});
|
|
344
|
+
this.log.error(`Background attestation/L1 pipeline failed for slot ${this.targetSlot}`, err, {
|
|
345
|
+
slot: this.targetSlot,
|
|
346
|
+
checkpointNumber: this.checkpointNumber,
|
|
347
|
+
reason: err instanceof Error ? err.message : String(err),
|
|
348
|
+
});
|
|
349
|
+
this.eventEmitter.emit('checkpoint-publish-failed', { slot: this.targetSlot });
|
|
350
|
+
this.metrics.recordPipelineDiscard();
|
|
351
|
+
}
|
|
352
|
+
}
|
|
353
|
+
|
|
354
|
+
/** Enqueues the checkpoint for L1 submission. Called after pipeline sleep in execute(). */
|
|
355
|
+
private async enqueueCheckpointForSubmission(result: CheckpointProposalResult): Promise<void> {
|
|
356
|
+
const { checkpoint, attestations, attestationsSignature } = result;
|
|
357
|
+
|
|
358
|
+
// Build-frame deadlines are measured against `slotNow`; observers still see `targetSlot`.
|
|
359
|
+
this.setStateFn(SequencerState.PUBLISHING_CHECKPOINT, this.targetSlot, this.slotNow);
|
|
360
|
+
const aztecSlotDuration = this.l1Constants.slotDuration;
|
|
361
|
+
const submissionSlotStart = Number(getTimestampForSlot(this.targetSlot, this.l1Constants));
|
|
362
|
+
const txTimeoutAt = new Date((submissionSlotStart + aztecSlotDuration) * 1000);
|
|
363
|
+
|
|
364
|
+
// If we have been configured to potentially skip publishing checkpoint then roll the dice here
|
|
365
|
+
if (
|
|
366
|
+
this.config.skipPublishingCheckpointsPercent !== undefined &&
|
|
367
|
+
this.config.skipPublishingCheckpointsPercent > 0
|
|
368
|
+
) {
|
|
369
|
+
const roll = Math.max(0, randomInt(100));
|
|
370
|
+
if (roll < this.config.skipPublishingCheckpointsPercent) {
|
|
371
|
+
this.log.warn(
|
|
372
|
+
`Skipping publishing proposal for checkpoint ${checkpoint.number}. Configured percentage: ${this.config.skipPublishingCheckpointsPercent}, generated value: ${roll}`,
|
|
373
|
+
);
|
|
374
|
+
return;
|
|
375
|
+
}
|
|
376
|
+
}
|
|
377
|
+
|
|
378
|
+
await this.publisher.enqueueProposeCheckpoint(checkpoint, attestations, attestationsSignature, {
|
|
379
|
+
txTimeoutAt,
|
|
380
|
+
});
|
|
381
|
+
}
|
|
382
|
+
|
|
383
|
+
/**
|
|
384
|
+
* Wait until the archiver syncs past the given L2 slot number.
|
|
385
|
+
* The deadline is the end of `this.targetSlot`, beyond which any pipelined work would miss its
|
|
386
|
+
* L1 submission window and is no longer useful.
|
|
387
|
+
*/
|
|
388
|
+
private async waitForSyncedL2SlotNumber(waitForSlot: SlotNumber): Promise<boolean> {
|
|
389
|
+
const targetSlotStart = Number(getTimestampForSlot(this.targetSlot, this.l1Constants));
|
|
390
|
+
const targetSlotEndMs = (targetSlotStart + this.l1Constants.slotDuration) * 1000;
|
|
391
|
+
const syncDelayTolerance = this.l1Constants.ethereumSlotDuration * 2 * 1000;
|
|
392
|
+
const timeoutSeconds = Math.max(0.1, (targetSlotEndMs + syncDelayTolerance - this.dateProvider.now()) / 1000);
|
|
393
|
+
|
|
394
|
+
try {
|
|
395
|
+
const timer = new Timer();
|
|
396
|
+
while (true) {
|
|
397
|
+
const syncedSlot = await this.l2BlockSource.getSyncedL2SlotNumber();
|
|
398
|
+
if (syncedSlot !== undefined && syncedSlot >= waitForSlot) {
|
|
399
|
+
return true;
|
|
400
|
+
}
|
|
401
|
+
if (timeoutSeconds && timer.s() > timeoutSeconds) {
|
|
402
|
+
throw new TimeoutError(`Timeout awaiting archiver sync past slot ${waitForSlot}`);
|
|
403
|
+
}
|
|
404
|
+
await this.awaitInterruptibleSleep(ARCHIVER_SYNC_POLLING_MS);
|
|
405
|
+
}
|
|
406
|
+
} catch (err) {
|
|
407
|
+
if (err instanceof SequencerInterruptedError) {
|
|
408
|
+
throw err;
|
|
409
|
+
}
|
|
410
|
+
this.log.warn(
|
|
411
|
+
`Archiver did not sync L1 past slot ${waitForSlot} before slot ${this.targetSlot} expired, discarding pipelined work`,
|
|
412
|
+
{ checkpointNumber: this.checkpointNumber },
|
|
413
|
+
);
|
|
414
|
+
this.emitPipelinedCheckpointDiscarded('archiver-sync-timeout');
|
|
415
|
+
return false;
|
|
416
|
+
}
|
|
417
|
+
}
|
|
418
|
+
|
|
419
|
+
/**
|
|
420
|
+
* Waits for the parent checkpoint to land on L1 before submitting a pipelined checkpoint.
|
|
421
|
+
* Polls until the archiver has synced L1 past the parent's slot, then verifies:
|
|
422
|
+
* - If we built on a proposed parent: it must have landed on L1 with matching hash and valid attestations.
|
|
423
|
+
* - If we built without a proposed parent: no new checkpoint must have appeared for that slot.
|
|
424
|
+
* If the parent has invalid attestations, enqueues an invalidation. Returns whether to proceed with the proposal.
|
|
425
|
+
*/
|
|
426
|
+
protected async waitForValidParentCheckpointOnL1(): Promise<boolean> {
|
|
427
|
+
if (this.config.skipWaitForValidParentCheckpointOnL1) {
|
|
428
|
+
this.log.warn(`Skipping waitForValidParentCheckpointOnL1 due to test configuration`, {
|
|
429
|
+
checkpointNumber: this.checkpointNumber,
|
|
172
430
|
});
|
|
173
|
-
|
|
431
|
+
return true;
|
|
432
|
+
}
|
|
433
|
+
|
|
434
|
+
const parentCheckpointNumber = CheckpointNumber(this.checkpointNumber - 1);
|
|
174
435
|
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
436
|
+
// Wait until archiver has synced L1 past the parent's slot (slotNow)
|
|
437
|
+
if (!(await this.waitForSyncedL2SlotNumber(this.slotNow))) {
|
|
438
|
+
return false;
|
|
439
|
+
}
|
|
440
|
+
|
|
441
|
+
const tips = await this.l2BlockSource.getL2Tips();
|
|
442
|
+
const checkpointedNumber = tips.checkpointed.checkpoint.number;
|
|
443
|
+
|
|
444
|
+
// We built on top of a proposed checkpoint. Verify it landed on L1 as expected.
|
|
445
|
+
if (this.proposedCheckpointData) {
|
|
446
|
+
// After syncing from L1 we see the chain tip has invalid attestations. This means the parent checkpoint was posted
|
|
447
|
+
// with invalid attestations, or it built on top of something with invalid attestations and didnt invalidate them.
|
|
448
|
+
// Either way, we thought our parent would be valid, so we have to throw away our work. But at least we'll try and
|
|
449
|
+
// invalidate on L1 so we clean up the chain for the next proposer. And we'll slash them, but that's handled elsewhere.
|
|
450
|
+
const validationStatus = await this.l2BlockSource.getPendingChainValidationStatus();
|
|
451
|
+
if (!validationStatus.valid) {
|
|
180
452
|
this.log.warn(
|
|
181
|
-
`Parent checkpoint
|
|
453
|
+
`Parent checkpoint ${parentCheckpointNumber} has invalid attestations, discarding pipelined work`,
|
|
454
|
+
{ checkpointNumber: this.checkpointNumber, reason: validationStatus.reason },
|
|
182
455
|
);
|
|
183
|
-
|
|
456
|
+
this.emitPipelinedCheckpointDiscarded('parent-invalid-attestations');
|
|
457
|
+
await this.enqueueInvalidation(validationStatus);
|
|
458
|
+
return false;
|
|
459
|
+
}
|
|
460
|
+
|
|
461
|
+
// The pending chain is valid. But did the parent checkpoint land on L1 at all?
|
|
462
|
+
if (checkpointedNumber < parentCheckpointNumber) {
|
|
463
|
+
this.log.warn(`Parent checkpoint ${parentCheckpointNumber} did not land on L1, discarding pipelined work`, {
|
|
464
|
+
checkpointNumber: this.checkpointNumber,
|
|
465
|
+
checkpointedNumber,
|
|
466
|
+
});
|
|
467
|
+
this.emitPipelinedCheckpointDiscarded('parent-not-on-l1');
|
|
468
|
+
return false;
|
|
469
|
+
}
|
|
470
|
+
|
|
471
|
+
// It landed. But is it the one we were expecting?
|
|
472
|
+
const expectedHash = this.proposedCheckpointData.header.hash().toString();
|
|
473
|
+
if (tips.checkpointed.checkpoint.hash !== expectedHash) {
|
|
474
|
+
this.log.warn(`Parent checkpoint ${parentCheckpointNumber} hash mismatch on L1, discarding pipelined work`, {
|
|
475
|
+
checkpointNumber: this.checkpointNumber,
|
|
476
|
+
expectedHash,
|
|
477
|
+
actualHash: tips.checkpointed.checkpoint.hash,
|
|
478
|
+
});
|
|
479
|
+
this.emitPipelinedCheckpointDiscarded('parent-hash-mismatch');
|
|
480
|
+
return false;
|
|
184
481
|
}
|
|
482
|
+
|
|
483
|
+
return true;
|
|
484
|
+
} else {
|
|
485
|
+
// We didn't see a proposed checkpoint at build time, so we built on checkpointed parent from two slots ago.
|
|
486
|
+
// But if a new checkpoint for the previous slot appeared on L1 in the meantime, our checkpoint assumed the wrong parent,
|
|
487
|
+
// so we have to discard our work. This can happen if we're somehow cut off from p2p and fail to see the checkpoint
|
|
488
|
+
// proposal for the previous slot.
|
|
489
|
+
if (checkpointedNumber > parentCheckpointNumber) {
|
|
490
|
+
this.log.warn(
|
|
491
|
+
`Unexpected checkpoint ${checkpointedNumber} landed on L1 after we built on top of parent ${parentCheckpointNumber}, discarding pipelined work`,
|
|
492
|
+
{ checkpointNumber: this.checkpointNumber, checkpointedNumber },
|
|
493
|
+
);
|
|
494
|
+
this.emitPipelinedCheckpointDiscarded('unexpected-parent-appeared');
|
|
495
|
+
return false;
|
|
496
|
+
}
|
|
497
|
+
|
|
498
|
+
return true;
|
|
185
499
|
}
|
|
500
|
+
}
|
|
186
501
|
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
502
|
+
/** Emits the pipelined-checkpoint-discarded event and records the metric. */
|
|
503
|
+
private emitPipelinedCheckpointDiscarded(reason: string): void {
|
|
504
|
+
this.metrics.recordPipelineParentCheckpointMismatch(reason);
|
|
505
|
+
this.eventEmitter.emit('pipelined-checkpoint-discarded', {
|
|
506
|
+
slot: this.targetSlot,
|
|
507
|
+
checkpointNumber: this.checkpointNumber,
|
|
508
|
+
reason,
|
|
509
|
+
});
|
|
510
|
+
}
|
|
511
|
+
|
|
512
|
+
/** Simulates and enqueues an invalidation request for the invalid parent checkpoint. */
|
|
513
|
+
private async enqueueInvalidation(validationStatus: ValidateCheckpointResult): Promise<void> {
|
|
514
|
+
if (this.config.skipInvalidateBlockAsProposer) {
|
|
515
|
+
this.log.warn(`Skipping checkpoint invalidation as proposer due to test configuration`);
|
|
516
|
+
return;
|
|
517
|
+
}
|
|
518
|
+
const invalidateRequest = await this.publisher.simulateInvalidateCheckpoint(validationStatus);
|
|
519
|
+
if (invalidateRequest) {
|
|
520
|
+
const submissionSlotStart = Number(getTimestampForSlot(this.targetSlot, this.l1Constants));
|
|
521
|
+
const txTimeoutAt = new Date((submissionSlotStart + this.l1Constants.slotDuration) * 1000);
|
|
522
|
+
this.publisher.enqueueInvalidateCheckpoint(invalidateRequest, { txTimeoutAt });
|
|
523
|
+
} else {
|
|
524
|
+
this.log.info(`Invalidation simulation returned undefined, checkpoint may have been removed already`, {
|
|
525
|
+
checkpointNumber: this.checkpointNumber,
|
|
526
|
+
});
|
|
198
527
|
}
|
|
199
528
|
}
|
|
200
529
|
|
|
@@ -205,19 +534,32 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
205
534
|
[Attributes.SLOT_NUMBER]: this.targetSlot,
|
|
206
535
|
};
|
|
207
536
|
})
|
|
208
|
-
private async proposeCheckpoint(): Promise<
|
|
537
|
+
private async proposeCheckpoint(): Promise<CheckpointProposalBroadcast | undefined> {
|
|
209
538
|
try {
|
|
539
|
+
const now = this.dateProvider.now();
|
|
540
|
+
if (this.proposedCheckpointData) {
|
|
541
|
+
// Measure against the wall-clock slot whose build window we are currently using.
|
|
542
|
+
// In pipelining mode `targetSlot` is intentionally one slot ahead, which makes the
|
|
543
|
+
// target-slot boundary a full slot away from the actual build start time.
|
|
544
|
+
const slotBoundaryMs = Number(getTimestampForSlot(this.slotNow, this.l1Constants)) * 1000;
|
|
545
|
+
this.checkpointMetrics.recordPipelinedCheckpointBuildStartOffsetFromSlotBoundary(now - slotBoundaryMs);
|
|
546
|
+
}
|
|
547
|
+
this.checkpointMetrics.startCheckpointTiming(now);
|
|
548
|
+
|
|
210
549
|
// Get operator configured coinbase and fee recipient for this attestor
|
|
211
550
|
const coinbase = this.validatorClient.getCoinbaseForAttestor(this.attestorAddress);
|
|
212
551
|
const feeRecipient = this.validatorClient.getFeeRecipientForAttestor(this.attestorAddress);
|
|
213
552
|
|
|
214
553
|
// Start the checkpoint
|
|
215
|
-
this.setStateFn(SequencerState.INITIALIZING_CHECKPOINT, this.targetSlot);
|
|
216
|
-
this.
|
|
217
|
-
buildSlot: this.
|
|
554
|
+
this.setStateFn(SequencerState.INITIALIZING_CHECKPOINT, this.targetSlot, this.slotNow);
|
|
555
|
+
this.logCheckpointEvent('slot-started', `Starting checkpoint proposal for slot ${this.targetSlot}`, {
|
|
556
|
+
buildSlot: this.slotNow,
|
|
218
557
|
submissionSlot: this.targetSlot,
|
|
219
|
-
|
|
558
|
+
slot: this.targetSlot,
|
|
559
|
+
checkpointNumber: this.checkpointNumber,
|
|
220
560
|
proposer: this.proposer?.toString(),
|
|
561
|
+
attestorAddress: this.attestorAddress.toString(),
|
|
562
|
+
publisherAddress: this.publisher.getSenderAddress().toString(),
|
|
221
563
|
coinbase: coinbase.toString(),
|
|
222
564
|
});
|
|
223
565
|
this.metrics.incOpenSlot(this.targetSlot, this.proposer?.toString() ?? 'unknown');
|
|
@@ -227,24 +569,51 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
227
569
|
this.publisher.enqueueInvalidateCheckpoint(this.invalidateCheckpoint);
|
|
228
570
|
}
|
|
229
571
|
|
|
230
|
-
//
|
|
572
|
+
// Build the simulation plan for this slot. When pipelining, this overrides L1's view of
|
|
573
|
+
// pending/archive/fee-header to "as if the proposed parent had landed", so both the
|
|
574
|
+
// mana-min-fee simulation (in the globals builder) and the pre-broadcast
|
|
575
|
+
// validateBlockHeader see the chain tip the eventual L1 send will see.
|
|
576
|
+
this.checkpointSimulationOverridesPlan = await buildCheckpointSimulationOverridesPlan({
|
|
577
|
+
checkpointNumber: this.checkpointNumber,
|
|
578
|
+
proposedCheckpointData: this.proposedCheckpointData,
|
|
579
|
+
invalidateToPendingCheckpointNumber: this.invalidateCheckpoint?.forcePendingCheckpointNumber,
|
|
580
|
+
checkpointedCheckpointNumber: this.checkpointedCheckpointNumber,
|
|
581
|
+
rollup: this.publisher.rollupContract,
|
|
582
|
+
signatureContext: this.signatureContext,
|
|
583
|
+
log: this.log,
|
|
584
|
+
});
|
|
585
|
+
|
|
231
586
|
const checkpointGlobalVariables = await this.globalsBuilder.buildCheckpointGlobalVariables(
|
|
232
587
|
coinbase,
|
|
233
588
|
feeRecipient,
|
|
234
589
|
this.targetSlot,
|
|
590
|
+
this.checkpointSimulationOverridesPlan,
|
|
235
591
|
);
|
|
236
592
|
|
|
237
593
|
// Collect L1 to L2 messages for the checkpoint and compute their hash
|
|
238
594
|
const l1ToL2Messages = await this.l1ToL2MessageSource.getL1ToL2Messages(this.checkpointNumber);
|
|
239
595
|
const inHash = computeInHashFromL1ToL2Messages(l1ToL2Messages);
|
|
240
596
|
|
|
241
|
-
// Collect the out hashes of all the checkpoints before this one in the same epoch
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
597
|
+
// Collect the out hashes of all the checkpoints before this one in the same epoch.
|
|
598
|
+
// Under pipelining the parent checkpoint may not be on L1 yet at build time, so the helper
|
|
599
|
+
// splices in the parent's checkpointOutHash from the locally-known proposed checkpoint so
|
|
600
|
+
// the resulting `epochOutHash` matches what validators (and L1) compute once the parent
|
|
601
|
+
// lands on L1.
|
|
602
|
+
const previousCheckpointOutHashes = await getPreviousCheckpointOutHashes({
|
|
603
|
+
blockSource: this.l2BlockSource,
|
|
604
|
+
epoch: this.targetEpoch,
|
|
605
|
+
checkpointNumber: this.checkpointNumber,
|
|
606
|
+
l1Constants: this.epochCache.getL1Constants(),
|
|
607
|
+
pipeliningEnabled: true,
|
|
608
|
+
proposedCheckpointData: this.proposedCheckpointData,
|
|
609
|
+
log: this.log,
|
|
610
|
+
});
|
|
245
611
|
|
|
246
|
-
//
|
|
247
|
-
|
|
612
|
+
// Anchor the modifier to the predicted parent fee header: L1 will apply it against
|
|
613
|
+
// that, not against the latest published checkpoint (which lags by one under pipelining).
|
|
614
|
+
const predictedParentEthPerFeeAssetE12 =
|
|
615
|
+
this.checkpointSimulationOverridesPlan?.pendingCheckpointState?.feeHeader?.ethPerFeeAsset;
|
|
616
|
+
const feeAssetPriceModifier = await this.publisher.getFeeAssetPriceModifier(predictedParentEthPerFeeAssetE12);
|
|
248
617
|
|
|
249
618
|
// Create a long-lived forked world state for the checkpoint builder
|
|
250
619
|
await using fork = await this.worldState.fork(this.syncedToBlockNumber, { closeDelayMs: 12_000 });
|
|
@@ -268,11 +637,12 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
268
637
|
|
|
269
638
|
const checkpointProposalOptions: CheckpointProposalOptions = {
|
|
270
639
|
publishFullTxs: !!this.config.publishTxsWithProposals,
|
|
271
|
-
broadcastInvalidCheckpointProposal:
|
|
640
|
+
broadcastInvalidCheckpointProposal:
|
|
641
|
+
this.config.broadcastInvalidCheckpointProposalOnly || this.config.broadcastInvalidBlockProposal,
|
|
272
642
|
};
|
|
273
643
|
|
|
274
644
|
let blocksInCheckpoint: L2Block[] = [];
|
|
275
|
-
let blockPendingBroadcast:
|
|
645
|
+
let blockPendingBroadcast: BlockProposal | undefined = undefined;
|
|
276
646
|
const checkpointBuildTimer = new Timer();
|
|
277
647
|
|
|
278
648
|
try {
|
|
@@ -296,23 +666,45 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
296
666
|
}
|
|
297
667
|
|
|
298
668
|
if (blocksInCheckpoint.length === 0) {
|
|
299
|
-
this.
|
|
669
|
+
this.logCheckpointEvent('build-failed', `Checkpoint build failed for slot ${this.targetSlot}`, {
|
|
670
|
+
slot: this.targetSlot,
|
|
671
|
+
checkpointNumber: this.checkpointNumber,
|
|
672
|
+
reason: 'no_blocks_built',
|
|
673
|
+
});
|
|
674
|
+
this.log.warn(`No blocks were built for slot ${this.targetSlot}`, {
|
|
675
|
+
slot: this.targetSlot,
|
|
676
|
+
checkpointNumber: this.checkpointNumber,
|
|
677
|
+
reason: 'no_blocks_built',
|
|
678
|
+
});
|
|
300
679
|
this.eventEmitter.emit('checkpoint-empty', { slot: this.targetSlot });
|
|
301
680
|
return undefined;
|
|
302
681
|
}
|
|
303
682
|
|
|
304
683
|
const minBlocksForCheckpoint = this.config.minBlocksForCheckpoint;
|
|
305
684
|
if (minBlocksForCheckpoint !== undefined && blocksInCheckpoint.length < minBlocksForCheckpoint) {
|
|
685
|
+
this.logCheckpointEvent('build-failed', `Checkpoint build failed for slot ${this.targetSlot}`, {
|
|
686
|
+
slot: this.targetSlot,
|
|
687
|
+
checkpointNumber: this.checkpointNumber,
|
|
688
|
+
blocksBuilt: blocksInCheckpoint.length,
|
|
689
|
+
minBlocksForCheckpoint,
|
|
690
|
+
reason: 'min_blocks_not_met',
|
|
691
|
+
});
|
|
306
692
|
this.log.warn(
|
|
307
693
|
`Checkpoint has fewer blocks than minimum (${blocksInCheckpoint.length} < ${minBlocksForCheckpoint}), skipping proposal`,
|
|
308
|
-
{
|
|
694
|
+
{
|
|
695
|
+
slot: this.targetSlot,
|
|
696
|
+
checkpointNumber: this.checkpointNumber,
|
|
697
|
+
blocksBuilt: blocksInCheckpoint.length,
|
|
698
|
+
minBlocksForCheckpoint,
|
|
699
|
+
reason: 'min_blocks_not_met',
|
|
700
|
+
},
|
|
309
701
|
);
|
|
310
702
|
return undefined;
|
|
311
703
|
}
|
|
312
704
|
|
|
313
705
|
// Assemble and broadcast the checkpoint proposal, including the last block that was not
|
|
314
706
|
// broadcasted yet, and wait to collect the committee attestations.
|
|
315
|
-
this.setStateFn(SequencerState.ASSEMBLING_CHECKPOINT, this.targetSlot);
|
|
707
|
+
this.setStateFn(SequencerState.ASSEMBLING_CHECKPOINT, this.targetSlot, this.slotNow);
|
|
316
708
|
const checkpoint = await checkpointBuilder.completeCheckpoint();
|
|
317
709
|
|
|
318
710
|
// Final validation: per-block limits are only checked if the operator set them explicitly.
|
|
@@ -326,21 +718,43 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
326
718
|
maxTxsPerCheckpoint: this.config.maxTxsPerCheckpoint,
|
|
327
719
|
});
|
|
328
720
|
} catch (err) {
|
|
329
|
-
this.
|
|
721
|
+
this.logCheckpointEvent('build-failed', `Checkpoint build failed for slot ${this.targetSlot}`, {
|
|
722
|
+
slot: this.targetSlot,
|
|
723
|
+
checkpointNumber: this.checkpointNumber,
|
|
724
|
+
blocksBuilt: blocksInCheckpoint.length,
|
|
725
|
+
reason: 'invalid_checkpoint',
|
|
726
|
+
checkpoint: checkpoint.header.toInspect(),
|
|
727
|
+
});
|
|
728
|
+
this.log.error(`Built an invalid checkpoint at slot ${this.slotNow} (skipping proposal)`, err, {
|
|
729
|
+
slot: this.targetSlot,
|
|
730
|
+
checkpointNumber: this.checkpointNumber,
|
|
731
|
+
blocksBuilt: blocksInCheckpoint.length,
|
|
732
|
+
reason: 'invalid_checkpoint',
|
|
330
733
|
checkpoint: checkpoint.header.toInspect(),
|
|
331
734
|
});
|
|
332
735
|
return undefined;
|
|
333
736
|
}
|
|
334
737
|
|
|
335
738
|
// Record checkpoint-level build metrics
|
|
336
|
-
this.
|
|
739
|
+
this.checkpointMetrics.recordCheckpointBuild(
|
|
337
740
|
checkpointBuildTimer.ms(),
|
|
338
741
|
blocksInCheckpoint.length,
|
|
339
742
|
checkpoint.getStats().txCount,
|
|
340
743
|
Number(checkpoint.header.totalManaUsed.toBigInt()),
|
|
341
744
|
);
|
|
745
|
+
this.logCheckpointEvent('built', `Checkpoint built for slot ${this.targetSlot}`, {
|
|
746
|
+
slot: this.targetSlot,
|
|
747
|
+
buildSlot: this.slotNow,
|
|
748
|
+
checkpointNumber: this.checkpointNumber,
|
|
749
|
+
proposer: this.proposer?.toString(),
|
|
750
|
+
attestorAddress: this.attestorAddress.toString(),
|
|
751
|
+
publisherAddress: this.publisher.getSenderAddress().toString(),
|
|
752
|
+
blocksBuilt: blocksInCheckpoint.length,
|
|
753
|
+
txCount: checkpoint.getStats().txCount,
|
|
754
|
+
totalMana: Number(checkpoint.header.totalManaUsed.toBigInt()),
|
|
755
|
+
});
|
|
342
756
|
|
|
343
|
-
//
|
|
757
|
+
// In fisherman mode, return the checkpoint without broadcasting or collecting attestations
|
|
344
758
|
if (this.config.fishermanMode) {
|
|
345
759
|
this.log.info(
|
|
346
760
|
`Built checkpoint for slot ${this.targetSlot} with ${blocksInCheckpoint.length} blocks. ` +
|
|
@@ -352,87 +766,70 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
352
766
|
},
|
|
353
767
|
);
|
|
354
768
|
this.metrics.recordCheckpointSuccess();
|
|
355
|
-
|
|
769
|
+
// Return a broadcast result with a dummy proposal — fisherman mode skips attestation collection
|
|
770
|
+
return { checkpoint, proposal: undefined!, blockProposedAt: this.dateProvider.now() };
|
|
356
771
|
}
|
|
357
772
|
|
|
358
|
-
//
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
}
|
|
773
|
+
// Validate the header against L1 state before broadcasting.
|
|
774
|
+
// If this fails the slot is aborted before any gossip work; state drift between here
|
|
775
|
+
// and the eventual L1 send is caught by the bundle simulate at send time.
|
|
776
|
+
try {
|
|
777
|
+
await this.publisher.validateBlockHeader(checkpoint.header, this.checkpointSimulationOverridesPlan);
|
|
778
|
+
} catch (err) {
|
|
779
|
+
this.log.error(`Pre-broadcast header validation failed for slot ${this.targetSlot}; aborting`, err, {
|
|
780
|
+
slot: this.targetSlot,
|
|
781
|
+
checkpointNumber: this.checkpointNumber,
|
|
782
|
+
});
|
|
783
|
+
this.metrics.recordCheckpointProposalFailed('header_validation_failed');
|
|
784
|
+
this.eventEmitter.emit('header-validation-failed', {
|
|
785
|
+
slot: this.targetSlot,
|
|
786
|
+
checkpointNumber: this.checkpointNumber,
|
|
787
|
+
reason: err instanceof Error ? err.message : String(err),
|
|
788
|
+
});
|
|
789
|
+
return undefined;
|
|
790
|
+
}
|
|
364
791
|
|
|
365
792
|
// Create the checkpoint proposal and broadcast it
|
|
366
793
|
const proposal = await this.validatorClient.createCheckpointProposal(
|
|
367
794
|
checkpoint.header,
|
|
368
795
|
checkpoint.archive.root,
|
|
796
|
+
this.checkpointNumber,
|
|
369
797
|
feeAssetPriceModifier,
|
|
370
|
-
|
|
798
|
+
blockPendingBroadcast,
|
|
371
799
|
this.proposer,
|
|
372
800
|
checkpointProposalOptions,
|
|
373
801
|
);
|
|
374
802
|
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
this.metrics.recordCheckpointAttestationDelay(blockAttestedAt - blockProposedAt);
|
|
383
|
-
|
|
384
|
-
// Proposer must sign over the attestations before pushing them to L1
|
|
385
|
-
const signer = this.proposer ?? this.publisher.getSenderAddress();
|
|
386
|
-
let attestationsSignature: Signature;
|
|
387
|
-
try {
|
|
388
|
-
attestationsSignature = await this.validatorClient.signAttestationsAndSigners(
|
|
389
|
-
attestations,
|
|
390
|
-
signer,
|
|
391
|
-
this.targetSlot,
|
|
392
|
-
this.checkpointNumber,
|
|
393
|
-
);
|
|
394
|
-
} catch (err) {
|
|
395
|
-
// We shouldn't really get here since we yield to another HA node
|
|
396
|
-
// as soon as we see these errors when creating block or checkpoint proposals.
|
|
397
|
-
if (this.handleHASigningError(err, 'Attestations signature')) {
|
|
398
|
-
return undefined;
|
|
399
|
-
}
|
|
400
|
-
throw err;
|
|
401
|
-
}
|
|
402
|
-
|
|
403
|
-
// Enqueue publishing the checkpoint to L1
|
|
404
|
-
this.setStateFn(SequencerState.PUBLISHING_CHECKPOINT, this.targetSlot);
|
|
405
|
-
const aztecSlotDuration = this.l1Constants.slotDuration;
|
|
406
|
-
const submissionSlotStart = Number(getTimestampForSlot(this.targetSlot, this.l1Constants));
|
|
407
|
-
const txTimeoutAt = new Date((submissionSlotStart + aztecSlotDuration) * 1000);
|
|
803
|
+
// Advance our own optimistic proposed-checkpoint tip locally before gossiping. Gossipsub
|
|
804
|
+
// doesn't echo our own messages back, so this is how the proposer makes its own proposed
|
|
805
|
+
// checkpoint visible for pipelining the next slot. Built from local checkpoint data — never
|
|
806
|
+
// from the broadcast proposal archive, which may be deliberately corrupted under test flags.
|
|
807
|
+
// Fail closed: if this throws, the outer catch aborts the slot before gossiping.
|
|
808
|
+
await this.syncProposedCheckpointToArchiver(checkpoint, blocksInCheckpoint.length, feeAssetPriceModifier);
|
|
408
809
|
|
|
409
|
-
|
|
410
|
-
if (
|
|
411
|
-
|
|
412
|
-
this
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
if (
|
|
416
|
-
this.
|
|
417
|
-
`Skipping publishing proposal for checkpoint ${checkpoint.number}. Configured percentage: ${this.config.skipPublishingCheckpointsPercent}, generated value: ${result}`,
|
|
418
|
-
);
|
|
419
|
-
return checkpoint;
|
|
810
|
+
const blockProposedAt = this.dateProvider.now();
|
|
811
|
+
if (this.config.skipBroadcastCheckpointProposal) {
|
|
812
|
+
// Test-only: suppress the CheckpointProposal so peers never see a proposed checkpoint for
|
|
813
|
+
// this slot, but still broadcast the held last block standalone so peers' archivers ingest
|
|
814
|
+
// it as a proposed-but-uncheckpointed tip — the exact orphan-block state that
|
|
815
|
+
// pruneOrphanProposedBlocks / checkSync must handle.
|
|
816
|
+
if (blockPendingBroadcast && !this.config.skipBroadcastProposals) {
|
|
817
|
+
await this.p2pClient.broadcastProposal(blockPendingBroadcast);
|
|
420
818
|
}
|
|
819
|
+
} else if (!this.config.skipBroadcastProposals) {
|
|
820
|
+
await this.p2pClient.broadcastCheckpointProposal(proposal);
|
|
821
|
+
this.checkpointMetrics.noteCheckpointBroadcast(this.dateProvider.now());
|
|
421
822
|
}
|
|
422
823
|
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
forcePendingCheckpointNumber: this.invalidateCheckpoint?.forcePendingCheckpointNumber,
|
|
426
|
-
});
|
|
427
|
-
|
|
428
|
-
return checkpoint;
|
|
824
|
+
// Return immediately after broadcast — attestation collection happens in the background
|
|
825
|
+
return { checkpoint, proposal, blockProposedAt };
|
|
429
826
|
} catch (err) {
|
|
430
827
|
if (err && (err instanceof DutyAlreadySignedError || err instanceof SlashingProtectionError)) {
|
|
431
828
|
// swallow this error. It's already been logged by a function deeper in the stack
|
|
432
829
|
return undefined;
|
|
433
830
|
}
|
|
434
831
|
|
|
435
|
-
this.log.error(`Error building checkpoint at slot ${this.
|
|
832
|
+
this.log.error(`Error building checkpoint at slot ${this.targetSlot}`, err);
|
|
436
833
|
return undefined;
|
|
437
834
|
}
|
|
438
835
|
}
|
|
@@ -448,20 +845,29 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
448
845
|
blockProposalOptions: BlockProposalOptions,
|
|
449
846
|
): Promise<{
|
|
450
847
|
blocksInCheckpoint: L2Block[];
|
|
451
|
-
blockPendingBroadcast:
|
|
848
|
+
blockPendingBroadcast: BlockProposal | undefined;
|
|
452
849
|
}> {
|
|
453
850
|
const blocksInCheckpoint: L2Block[] = [];
|
|
454
851
|
const txHashesAlreadyIncluded = new Set<string>();
|
|
455
852
|
const initialBlockNumber = BlockNumber(this.syncedToBlockNumber + 1);
|
|
456
853
|
|
|
457
854
|
// Last block in the checkpoint will usually be flagged as pending broadcast, so we send it along with the checkpoint proposal
|
|
458
|
-
let blockPendingBroadcast:
|
|
855
|
+
let blockPendingBroadcast: BlockProposal | undefined = undefined;
|
|
459
856
|
|
|
460
857
|
while (true) {
|
|
461
858
|
const blocksBuilt = blocksInCheckpoint.length;
|
|
462
859
|
const indexWithinCheckpoint = IndexWithinCheckpoint(blocksBuilt);
|
|
463
860
|
const blockNumber = BlockNumber(initialBlockNumber + blocksBuilt);
|
|
464
861
|
|
|
862
|
+
if (blocksBuilt >= this.config.maxBlocksPerCheckpoint) {
|
|
863
|
+
this.log.debug(`Reached max blocks per checkpoint`, {
|
|
864
|
+
slot: this.targetSlot,
|
|
865
|
+
blocksBuilt,
|
|
866
|
+
maxBlocksPerCheckpoint: this.config.maxBlocksPerCheckpoint,
|
|
867
|
+
});
|
|
868
|
+
break;
|
|
869
|
+
}
|
|
870
|
+
|
|
465
871
|
const secondsIntoSlot = this.getSecondsIntoSlot();
|
|
466
872
|
const timingInfo = this.timetable.canStartNextBlock(secondsIntoSlot);
|
|
467
873
|
|
|
@@ -488,19 +894,20 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
488
894
|
txHashesAlreadyIncluded,
|
|
489
895
|
});
|
|
490
896
|
|
|
491
|
-
//
|
|
492
|
-
if (
|
|
493
|
-
// If
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
897
|
+
// If we failed to build the block due to insufficient txs, we try again if there is still time left in the slot
|
|
898
|
+
if ('failure' in buildResult) {
|
|
899
|
+
// If this was the last subslot, or we're running with a single block per slot, we're done
|
|
900
|
+
if (timingInfo.isLastBlock || timingInfo.deadline === undefined) {
|
|
901
|
+
break;
|
|
902
|
+
}
|
|
903
|
+
// Otherwise, if there is still time for more blocks, we wait until the next subslot and try again
|
|
497
904
|
await this.waitUntilNextSubslot(timingInfo.deadline);
|
|
498
905
|
continue;
|
|
499
|
-
}
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
906
|
+
}
|
|
907
|
+
|
|
908
|
+
// If there was an error building the block, we just exit the loop and give up the rest of the slot.
|
|
909
|
+
// We don't want to risk building more blocks if something went wrong.
|
|
910
|
+
if ('error' in buildResult) {
|
|
504
911
|
if (!(buildResult.error instanceof SequencerInterruptedError)) {
|
|
505
912
|
this.log.warn(`Halting block building for slot ${this.targetSlot}`, {
|
|
506
913
|
slot: this.targetSlot,
|
|
@@ -512,35 +919,44 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
512
919
|
}
|
|
513
920
|
|
|
514
921
|
const { block, usedTxs } = buildResult;
|
|
922
|
+
this.checkpointMetrics.noteCheckpointBlockBuilt(this.dateProvider.now(), {
|
|
923
|
+
isFirstBlock: blocksBuilt === 0,
|
|
924
|
+
isLastBlock: timingInfo.isLastBlock,
|
|
925
|
+
});
|
|
926
|
+
|
|
515
927
|
blocksInCheckpoint.push(block);
|
|
516
928
|
usedTxs.forEach(tx => txHashesAlreadyIncluded.add(tx.txHash.toString()));
|
|
517
929
|
|
|
518
|
-
//
|
|
519
|
-
|
|
930
|
+
// Sign the block proposal. This will throw if HA signing fails.
|
|
931
|
+
const proposal = await this.createBlockProposal(block, inHash, usedTxs, {
|
|
932
|
+
...blockProposalOptions,
|
|
933
|
+
broadcastInvalidBlockProposal:
|
|
934
|
+
blockProposalOptions.broadcastInvalidBlockProposal ||
|
|
935
|
+
block.indexWithinCheckpoint === this.config.invalidBlockProposalIndexWithinCheckpoint,
|
|
936
|
+
});
|
|
937
|
+
|
|
938
|
+
// Sync the proposed block to the archiver to make it available, only after we've managed to sign the proposal,
|
|
939
|
+
// so we avoid polluting our archive with a block that would fail.
|
|
940
|
+
// We wait for the sync to succeed, as this helps catch consistency errors, even if it means we lose some time for block-building.
|
|
941
|
+
// If this throws, we abort the entire checkpoint.
|
|
942
|
+
await this.syncProposedBlockToArchiver(block);
|
|
943
|
+
|
|
944
|
+
// If this is the last block, do not broadcast it, since it will be included in the checkpoint proposal.
|
|
520
945
|
if (timingInfo.isLastBlock) {
|
|
521
|
-
await this.syncProposedBlockToArchiver(block);
|
|
522
946
|
this.log.verbose(`Completed final block ${blockNumber} for slot ${this.targetSlot}`, {
|
|
523
947
|
slot: this.targetSlot,
|
|
524
948
|
blockNumber,
|
|
525
949
|
blocksBuilt,
|
|
526
950
|
});
|
|
527
|
-
|
|
951
|
+
|
|
952
|
+
blockPendingBroadcast = proposal;
|
|
528
953
|
break;
|
|
529
954
|
}
|
|
530
955
|
|
|
531
|
-
// Broadcast the block proposal (unless we're in fisherman mode) unless the block is the last one,
|
|
532
|
-
// in which case we'll broadcast it along with the checkpoint at the end of the loop.
|
|
533
|
-
// Note that we only send the block to the archiver if we manage to create the proposal, so if there's
|
|
534
|
-
// a HA error we don't pollute our archiver with a block that won't make it to the chain.
|
|
535
|
-
const proposal = await this.createBlockProposal(block, inHash, usedTxs, blockProposalOptions);
|
|
536
|
-
|
|
537
|
-
// Sync the proposed block to the archiver to make it available, only after we've managed to sign the proposal.
|
|
538
|
-
// We wait for the sync to succeed, as this helps catch consistency errors, even if it means we lose some time for block-building.
|
|
539
|
-
// If this throws, we abort the entire checkpoint.
|
|
540
|
-
await this.syncProposedBlockToArchiver(block);
|
|
541
|
-
|
|
542
956
|
// Once we have a signed proposal and the archiver agreed with our proposed block, then we broadcast it.
|
|
543
|
-
proposal &&
|
|
957
|
+
if (proposal && !this.config.skipBroadcastProposals) {
|
|
958
|
+
await this.p2pClient.broadcastProposal(proposal);
|
|
959
|
+
}
|
|
544
960
|
|
|
545
961
|
// Wait until the next block's start time
|
|
546
962
|
await this.waitUntilNextSubslot(timingInfo.deadline);
|
|
@@ -567,6 +983,7 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
567
983
|
}
|
|
568
984
|
return this.validatorClient.createBlockProposal(
|
|
569
985
|
block.header,
|
|
986
|
+
this.checkpointNumber,
|
|
570
987
|
block.indexWithinCheckpoint,
|
|
571
988
|
inHash,
|
|
572
989
|
block.archive.root,
|
|
@@ -579,7 +996,7 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
579
996
|
/** Sleeps until it is time to produce the next block in the slot */
|
|
580
997
|
@trackSpan('CheckpointProposalJob.waitUntilNextSubslot')
|
|
581
998
|
private async waitUntilNextSubslot(nextSubslotStart: number) {
|
|
582
|
-
this.setStateFn(SequencerState.WAITING_UNTIL_NEXT_BLOCK, this.targetSlot);
|
|
999
|
+
this.setStateFn(SequencerState.WAITING_UNTIL_NEXT_BLOCK, this.targetSlot, this.slotNow);
|
|
583
1000
|
this.log.verbose(`Waiting until time for the next block at ${nextSubslotStart}s into slot`, {
|
|
584
1001
|
slot: this.targetSlot,
|
|
585
1002
|
});
|
|
@@ -598,7 +1015,9 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
598
1015
|
buildDeadline: Date | undefined;
|
|
599
1016
|
txHashesAlreadyIncluded: Set<string>;
|
|
600
1017
|
},
|
|
601
|
-
): Promise<
|
|
1018
|
+
): Promise<
|
|
1019
|
+
{ block: L2Block; usedTxs: Tx[] } | { failure: 'insufficient-txs' | 'insufficient-valid-txs' } | { error: Error }
|
|
1020
|
+
> {
|
|
602
1021
|
const { blockTimestamp, forceCreate, blockNumber, indexWithinCheckpoint, buildDeadline, txHashesAlreadyIncluded } =
|
|
603
1022
|
opts;
|
|
604
1023
|
|
|
@@ -611,13 +1030,30 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
611
1030
|
// Wait until we have enough txs to build the block
|
|
612
1031
|
const { availableTxs, canStartBuilding, minTxs } = await this.waitForMinTxs(opts);
|
|
613
1032
|
if (!canStartBuilding) {
|
|
1033
|
+
this.logCheckpointEvent('block-build-failed', `Block build failed for slot ${this.targetSlot}`, {
|
|
1034
|
+
reason: 'insufficient_txs',
|
|
1035
|
+
blockNumber,
|
|
1036
|
+
slot: this.targetSlot,
|
|
1037
|
+
checkpointNumber: this.checkpointNumber,
|
|
1038
|
+
indexWithinCheckpoint,
|
|
1039
|
+
availableTxs,
|
|
1040
|
+
minTxs,
|
|
1041
|
+
});
|
|
614
1042
|
this.log.warn(
|
|
615
1043
|
`Not enough txs to build block ${blockNumber} at index ${indexWithinCheckpoint} in slot ${this.targetSlot} (got ${availableTxs} txs but needs ${minTxs})`,
|
|
616
|
-
{
|
|
1044
|
+
{
|
|
1045
|
+
reason: 'insufficient_txs',
|
|
1046
|
+
blockNumber,
|
|
1047
|
+
slot: this.targetSlot,
|
|
1048
|
+
checkpointNumber: this.checkpointNumber,
|
|
1049
|
+
indexWithinCheckpoint,
|
|
1050
|
+
availableTxs,
|
|
1051
|
+
minTxs,
|
|
1052
|
+
},
|
|
617
1053
|
);
|
|
618
1054
|
this.eventEmitter.emit('block-tx-count-check-failed', { minTxs, availableTxs, slot: this.targetSlot });
|
|
619
1055
|
this.metrics.recordBlockProposalFailed('insufficient_txs');
|
|
620
|
-
return
|
|
1056
|
+
return { failure: 'insufficient-txs' };
|
|
621
1057
|
}
|
|
622
1058
|
|
|
623
1059
|
// Create iterator to pending txs. We filter out txs already included in previous blocks in the checkpoint
|
|
@@ -631,7 +1067,7 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
631
1067
|
`Building block ${blockNumber} at index ${indexWithinCheckpoint} for slot ${this.targetSlot} with ${availableTxs} available txs`,
|
|
632
1068
|
{ slot: this.targetSlot, blockNumber, indexWithinCheckpoint },
|
|
633
1069
|
);
|
|
634
|
-
this.setStateFn(SequencerState.CREATING_BLOCK, this.targetSlot);
|
|
1070
|
+
this.setStateFn(SequencerState.CREATING_BLOCK, this.targetSlot, this.slotNow);
|
|
635
1071
|
|
|
636
1072
|
// Per-block limits are operator overrides (from SEQ_MAX_L2_BLOCK_GAS etc.) further capped
|
|
637
1073
|
// by remaining checkpoint-level budgets inside CheckpointBuilder before each block is built.
|
|
@@ -665,10 +1101,21 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
665
1101
|
await this.dropFailedTxsFromP2P(buildResult.failedTxs);
|
|
666
1102
|
|
|
667
1103
|
if (buildResult.status === 'insufficient-valid-txs') {
|
|
1104
|
+
this.logCheckpointEvent('block-build-failed', `Block build failed for slot ${this.targetSlot}`, {
|
|
1105
|
+
reason: 'insufficient_valid_txs',
|
|
1106
|
+
slot: this.targetSlot,
|
|
1107
|
+
checkpointNumber: this.checkpointNumber,
|
|
1108
|
+
blockNumber,
|
|
1109
|
+
numTxs: buildResult.processedCount,
|
|
1110
|
+
indexWithinCheckpoint,
|
|
1111
|
+
minValidTxs,
|
|
1112
|
+
});
|
|
668
1113
|
this.log.warn(
|
|
669
1114
|
`Block ${blockNumber} at index ${indexWithinCheckpoint} on slot ${this.targetSlot} has too few valid txs to be proposed`,
|
|
670
1115
|
{
|
|
1116
|
+
reason: 'insufficient_valid_txs',
|
|
671
1117
|
slot: this.targetSlot,
|
|
1118
|
+
checkpointNumber: this.checkpointNumber,
|
|
672
1119
|
blockNumber,
|
|
673
1120
|
numTxs: buildResult.processedCount,
|
|
674
1121
|
indexWithinCheckpoint,
|
|
@@ -680,7 +1127,7 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
680
1127
|
slot: this.targetSlot,
|
|
681
1128
|
});
|
|
682
1129
|
this.metrics.recordBlockProposalFailed('insufficient_valid_txs');
|
|
683
|
-
return
|
|
1130
|
+
return { failure: 'insufficient-valid-txs' };
|
|
684
1131
|
}
|
|
685
1132
|
|
|
686
1133
|
// Block creation succeeded, emit stats and metrics
|
|
@@ -702,12 +1149,17 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
702
1149
|
{ blockHash, txHashes, manaPerSec, ...blockStats },
|
|
703
1150
|
);
|
|
704
1151
|
|
|
1152
|
+
// `slot` is the target/submission slot (may be one ahead when pipelining),
|
|
1153
|
+
// `buildSlot` is the wall-clock slot during which the block was actually built.
|
|
705
1154
|
this.eventEmitter.emit('block-proposed', {
|
|
706
1155
|
blockNumber: block.number,
|
|
1156
|
+
blockHash,
|
|
1157
|
+
checkpointNumber: this.checkpointNumber,
|
|
1158
|
+
indexWithinCheckpoint: block.indexWithinCheckpoint,
|
|
707
1159
|
slot: this.targetSlot,
|
|
708
1160
|
buildSlot: this.slotNow,
|
|
709
1161
|
});
|
|
710
|
-
this.metrics.recordBuiltBlock(blockBuildDuration, block.header.totalManaUsed.toNumberUnsafe());
|
|
1162
|
+
this.metrics.recordBuiltBlock(blockBuildDuration, block.header.totalManaUsed.toNumberUnsafe(), this.targetSlot);
|
|
711
1163
|
|
|
712
1164
|
return { block, usedTxs };
|
|
713
1165
|
} catch (err: any) {
|
|
@@ -715,7 +1167,18 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
715
1167
|
reason: err.message,
|
|
716
1168
|
slot: this.targetSlot,
|
|
717
1169
|
});
|
|
718
|
-
this.
|
|
1170
|
+
this.logCheckpointEvent('block-build-failed', `Block build failed for slot ${this.targetSlot}`, {
|
|
1171
|
+
reason: err instanceof Error ? err.message : String(err),
|
|
1172
|
+
slot: this.targetSlot,
|
|
1173
|
+
checkpointNumber: this.checkpointNumber,
|
|
1174
|
+
blockNumber,
|
|
1175
|
+
});
|
|
1176
|
+
this.log.error(`Error building block`, err, {
|
|
1177
|
+
reason: err instanceof Error ? err.message : String(err),
|
|
1178
|
+
slot: this.targetSlot,
|
|
1179
|
+
checkpointNumber: this.checkpointNumber,
|
|
1180
|
+
blockNumber,
|
|
1181
|
+
});
|
|
719
1182
|
this.metrics.recordBlockProposalFailed(err.name || 'unknown_error');
|
|
720
1183
|
this.metrics.recordFailedBlock();
|
|
721
1184
|
return { error: err };
|
|
@@ -775,7 +1238,7 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
775
1238
|
}
|
|
776
1239
|
|
|
777
1240
|
// Wait a bit before checking again
|
|
778
|
-
this.setStateFn(SequencerState.WAITING_FOR_TXS, this.targetSlot);
|
|
1241
|
+
this.setStateFn(SequencerState.WAITING_FOR_TXS, this.targetSlot, this.slotNow);
|
|
779
1242
|
this.log.verbose(
|
|
780
1243
|
`Waiting for enough txs to build block ${blockNumber} at index ${indexWithinCheckpoint} in slot ${this.targetSlot} (have ${availableTxs} but need ${minTxs})`,
|
|
781
1244
|
{ blockNumber, slot: this.targetSlot, indexWithinCheckpoint },
|
|
@@ -787,15 +1250,47 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
787
1250
|
return { canStartBuilding: true, availableTxs, minTxs };
|
|
788
1251
|
}
|
|
789
1252
|
|
|
1253
|
+
private async getSignedCommitteeAttestations(
|
|
1254
|
+
broadcast: CheckpointProposalBroadcast,
|
|
1255
|
+
): Promise<{ attestations: CommitteeAttestationsAndSigners; attestationsSignature: Signature } | undefined> {
|
|
1256
|
+
const { proposal, blockProposedAt } = broadcast;
|
|
1257
|
+
this.setStateFn(SequencerState.COLLECTING_ATTESTATIONS, this.targetSlot, this.slotNow);
|
|
1258
|
+
const attestations = await this.waitForAttestations(proposal);
|
|
1259
|
+
if (!attestations) {
|
|
1260
|
+
return undefined;
|
|
1261
|
+
}
|
|
1262
|
+
this.checkpointMetrics.recordCheckpointAttestationDelay(this.dateProvider.now() - blockProposedAt);
|
|
1263
|
+
|
|
1264
|
+
// Proposer must sign over the attestations before pushing them to L1
|
|
1265
|
+
const signer = this.proposer ?? this.publisher.getSenderAddress();
|
|
1266
|
+
try {
|
|
1267
|
+
const attestationsSignature = await this.validatorClient.signAttestationsAndSigners(
|
|
1268
|
+
attestations,
|
|
1269
|
+
signer,
|
|
1270
|
+
this.targetSlot,
|
|
1271
|
+
this.checkpointNumber,
|
|
1272
|
+
);
|
|
1273
|
+
return { attestations, attestationsSignature };
|
|
1274
|
+
} catch (err) {
|
|
1275
|
+
if (this.handleHASigningError(err, 'Attestations signature')) {
|
|
1276
|
+
return;
|
|
1277
|
+
}
|
|
1278
|
+
this.log.error(`Error signing attestations for checkpoint proposal at slot ${proposal.slotNumber}`, err);
|
|
1279
|
+
return undefined;
|
|
1280
|
+
}
|
|
1281
|
+
}
|
|
1282
|
+
|
|
790
1283
|
/**
|
|
791
1284
|
* Waits for enough attestations to be collected via p2p.
|
|
792
1285
|
* This is run after all blocks for the checkpoint have been built.
|
|
793
1286
|
*/
|
|
794
1287
|
@trackSpan('CheckpointProposalJob.waitForAttestations')
|
|
795
|
-
private async waitForAttestations(
|
|
1288
|
+
private async waitForAttestations(
|
|
1289
|
+
proposal: CheckpointProposal,
|
|
1290
|
+
): Promise<CommitteeAttestationsAndSigners | undefined> {
|
|
796
1291
|
if (this.config.fishermanMode) {
|
|
797
1292
|
this.log.debug('Skipping attestation collection in fisherman mode');
|
|
798
|
-
return CommitteeAttestationsAndSigners.empty();
|
|
1293
|
+
return CommitteeAttestationsAndSigners.empty(this.getSignatureContext());
|
|
799
1294
|
}
|
|
800
1295
|
|
|
801
1296
|
const slotNumber = proposal.slotNumber;
|
|
@@ -805,21 +1300,27 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
805
1300
|
throw new Error('No committee when collecting attestations');
|
|
806
1301
|
} else if (committee.length === 0) {
|
|
807
1302
|
this.log.verbose(`Attesting committee is empty`);
|
|
808
|
-
return CommitteeAttestationsAndSigners.empty();
|
|
1303
|
+
return CommitteeAttestationsAndSigners.empty(this.getSignatureContext());
|
|
809
1304
|
} else {
|
|
810
1305
|
this.log.debug(`Attesting committee length is ${committee.length}`, { committee });
|
|
811
1306
|
}
|
|
812
1307
|
|
|
813
|
-
const numberOfRequiredAttestations =
|
|
1308
|
+
const numberOfRequiredAttestations = computeQuorum(committee.length);
|
|
814
1309
|
|
|
815
1310
|
if (this.config.skipCollectingAttestations) {
|
|
816
1311
|
this.log.warn('Skipping attestation collection as per config (attesting with own keys only)');
|
|
817
|
-
const attestations = await this.validatorClient?.collectOwnAttestations(proposal);
|
|
818
|
-
|
|
1312
|
+
const attestations = await this.validatorClient?.collectOwnAttestations(proposal, this.checkpointNumber);
|
|
1313
|
+
this.logCheckpointAttestations('collected', committee, attestations ?? [], numberOfRequiredAttestations, {
|
|
1314
|
+
reason: 'collect_own_only',
|
|
1315
|
+
});
|
|
1316
|
+
return new CommitteeAttestationsAndSigners(
|
|
1317
|
+
orderAttestations(attestations ?? [], committee),
|
|
1318
|
+
this.getSignatureContext(),
|
|
1319
|
+
);
|
|
819
1320
|
}
|
|
820
1321
|
|
|
821
1322
|
const attestationTimeAllowed = this.config.enforceTimeTable
|
|
822
|
-
? this.timetable.
|
|
1323
|
+
? this.timetable.getCheckpointAttestationDeadline()
|
|
823
1324
|
: this.l1Constants.slotDuration;
|
|
824
1325
|
const attestationDeadline = new Date((this.getSlotStartBuildTimestamp() + attestationTimeAllowed) * 1000);
|
|
825
1326
|
|
|
@@ -832,6 +1333,7 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
832
1333
|
proposal,
|
|
833
1334
|
numberOfRequiredAttestations,
|
|
834
1335
|
attestationDeadline,
|
|
1336
|
+
this.checkpointNumber,
|
|
835
1337
|
);
|
|
836
1338
|
|
|
837
1339
|
collectedAttestationsCount = attestations.length;
|
|
@@ -850,6 +1352,9 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
850
1352
|
|
|
851
1353
|
// Rollup contract requires that the signatures are provided in the order of the committee
|
|
852
1354
|
const sorted = orderAttestations(trimmed, committee);
|
|
1355
|
+
this.logCheckpointAttestations('collected', committee, attestations, numberOfRequiredAttestations, {
|
|
1356
|
+
submittedCount: trimmed.length,
|
|
1357
|
+
});
|
|
853
1358
|
|
|
854
1359
|
// Manipulate the attestations if we've been configured to do so
|
|
855
1360
|
if (
|
|
@@ -861,17 +1366,56 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
861
1366
|
return this.manipulateAttestations(proposal.slotNumber, epoch, seed, committee, sorted);
|
|
862
1367
|
}
|
|
863
1368
|
|
|
864
|
-
return new CommitteeAttestationsAndSigners(sorted);
|
|
1369
|
+
return new CommitteeAttestationsAndSigners(sorted, this.getSignatureContext());
|
|
865
1370
|
} catch (err) {
|
|
866
1371
|
if (err && err instanceof AttestationTimeoutError) {
|
|
867
1372
|
collectedAttestationsCount = err.collectedCount;
|
|
1373
|
+
this.logCheckpointAttestations('failed', committee, undefined, numberOfRequiredAttestations, {
|
|
1374
|
+
collectedCount: collectedAttestationsCount,
|
|
1375
|
+
reason: 'timeout',
|
|
1376
|
+
});
|
|
1377
|
+
this.log.error(
|
|
1378
|
+
`Timeout while waiting for attestations for checkpoint proposal at slot ${proposal.slotNumber} (collected ${collectedAttestationsCount}/${numberOfRequiredAttestations})`,
|
|
1379
|
+
err,
|
|
1380
|
+
);
|
|
1381
|
+
} else {
|
|
1382
|
+
this.logCheckpointAttestations('failed', committee, undefined, numberOfRequiredAttestations, {
|
|
1383
|
+
collectedCount: collectedAttestationsCount,
|
|
1384
|
+
reason: err instanceof Error ? err.message : String(err),
|
|
1385
|
+
});
|
|
1386
|
+
this.log.error(`Error collecting attestations for checkpoint proposal at slot ${proposal.slotNumber}`, err);
|
|
868
1387
|
}
|
|
869
|
-
|
|
1388
|
+
return undefined;
|
|
870
1389
|
} finally {
|
|
871
1390
|
this.metrics.recordCollectedAttestations(collectedAttestationsCount, collectAttestationsTimer.ms());
|
|
872
1391
|
}
|
|
873
1392
|
}
|
|
874
1393
|
|
|
1394
|
+
private logCheckpointAttestations(
|
|
1395
|
+
status: 'collected' | 'failed',
|
|
1396
|
+
committee: EthAddress[],
|
|
1397
|
+
attestations: CheckpointAttestation[] | undefined,
|
|
1398
|
+
requiredAttestations: number,
|
|
1399
|
+
opts: { collectedCount?: number; submittedCount?: number; reason?: string } = {},
|
|
1400
|
+
) {
|
|
1401
|
+
const signedValidators =
|
|
1402
|
+
attestations
|
|
1403
|
+
?.map(attestation => attestation.getSender()?.toString())
|
|
1404
|
+
.filter((address): address is `0x${string}` => address !== undefined) ?? [];
|
|
1405
|
+
const collectedCount = opts.collectedCount ?? new Set(signedValidators).size;
|
|
1406
|
+
const missingValidatorCount = status === 'failed' ? Math.max(0, requiredAttestations - collectedCount) : undefined;
|
|
1407
|
+
this.logCheckpointEvent(`attestations-${status}`, `Checkpoint attestations ${status} for slot ${this.targetSlot}`, {
|
|
1408
|
+
slot: this.targetSlot,
|
|
1409
|
+
checkpointNumber: this.checkpointNumber,
|
|
1410
|
+
committeeSize: committee.length,
|
|
1411
|
+
requiredAttestations,
|
|
1412
|
+
collectedAttestations: collectedCount,
|
|
1413
|
+
...(opts.submittedCount !== undefined && { submittedAttestations: opts.submittedCount }),
|
|
1414
|
+
...(missingValidatorCount !== undefined && { missingValidatorCount }),
|
|
1415
|
+
...(opts.reason !== undefined && { reason: opts.reason }),
|
|
1416
|
+
});
|
|
1417
|
+
}
|
|
1418
|
+
|
|
875
1419
|
/** Breaks the attestations before publishing based on attack configs */
|
|
876
1420
|
private manipulateAttestations(
|
|
877
1421
|
slotNumber: SlotNumber,
|
|
@@ -915,7 +1459,7 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
915
1459
|
unfreeze(attestations[targetIndex]).signature = generateRecoverableSignature();
|
|
916
1460
|
}
|
|
917
1461
|
}
|
|
918
|
-
return new CommitteeAttestationsAndSigners(attestations);
|
|
1462
|
+
return new CommitteeAttestationsAndSigners(attestations, this.getSignatureContext());
|
|
919
1463
|
}
|
|
920
1464
|
|
|
921
1465
|
if (this.config.shuffleAttestationOrdering) {
|
|
@@ -937,11 +1481,11 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
937
1481
|
[shuffled[i], shuffled[j]] = [shuffled[j], shuffled[i]];
|
|
938
1482
|
}
|
|
939
1483
|
|
|
940
|
-
const signers = new CommitteeAttestationsAndSigners(attestations).getSigners();
|
|
941
|
-
return new MaliciousCommitteeAttestationsAndSigners(shuffled, signers);
|
|
1484
|
+
const signers = new CommitteeAttestationsAndSigners(attestations, this.getSignatureContext()).getSigners();
|
|
1485
|
+
return new MaliciousCommitteeAttestationsAndSigners(shuffled, signers, this.getSignatureContext());
|
|
942
1486
|
}
|
|
943
1487
|
|
|
944
|
-
return new CommitteeAttestationsAndSigners(attestations);
|
|
1488
|
+
return new CommitteeAttestationsAndSigners(attestations, this.getSignatureContext());
|
|
945
1489
|
}
|
|
946
1490
|
|
|
947
1491
|
private async dropFailedTxsFromP2P(failedTxs: FailedTx[]) {
|
|
@@ -958,9 +1502,13 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
958
1502
|
* Adds the proposed block to the archiver so it's available via P2P.
|
|
959
1503
|
* Gossip doesn't echo messages back to the sender, so the proposer's archiver/world-state
|
|
960
1504
|
* would never receive its own block without this explicit sync.
|
|
1505
|
+
*
|
|
1506
|
+
* In fisherman mode we skip this push: the fisherman builds blocks locally for validation
|
|
1507
|
+
* and fee analysis only, and pushing them to the archiver causes spurious reorg cascades
|
|
1508
|
+
* whenever the real proposer's block arrives from L1.
|
|
961
1509
|
*/
|
|
962
1510
|
private async syncProposedBlockToArchiver(block: L2Block): Promise<void> {
|
|
963
|
-
if (this.config.skipPushProposedBlocksToArchiver
|
|
1511
|
+
if (this.config.skipPushProposedBlocksToArchiver || this.config.fishermanMode) {
|
|
964
1512
|
this.log.warn(`Skipping push of proposed block ${block.number} to archiver`, {
|
|
965
1513
|
blockNumber: block.number,
|
|
966
1514
|
slot: block.header.globalVariables.slotNumber,
|
|
@@ -974,6 +1522,41 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
974
1522
|
await this.blockSink.addBlock(block);
|
|
975
1523
|
}
|
|
976
1524
|
|
|
1525
|
+
/**
|
|
1526
|
+
* Adds the proposed checkpoint to the archiver so the proposer's optimistic proposed-checkpoint
|
|
1527
|
+
* tip advances locally. Gossip doesn't echo our own messages back, so without this the proposer
|
|
1528
|
+
* would never see its own proposed checkpoint and couldn't pipeline the next slot.
|
|
1529
|
+
*
|
|
1530
|
+
* Skipped whenever proposed blocks aren't pushed (`skipPushProposedBlocksToArchiver`, fisherman
|
|
1531
|
+
* mode): the archiver derives the checkpoint archive from its stored blocks, so without them the
|
|
1532
|
+
* push would fail. All blocks were already added (and awaited) during block building, so this
|
|
1533
|
+
* needs no retry — they are guaranteed present by the time we get here.
|
|
1534
|
+
*/
|
|
1535
|
+
private async syncProposedCheckpointToArchiver(
|
|
1536
|
+
checkpoint: Checkpoint,
|
|
1537
|
+
blockCount: number,
|
|
1538
|
+
feeAssetPriceModifier: bigint,
|
|
1539
|
+
): Promise<void> {
|
|
1540
|
+
if (this.config.skipPushProposedBlocksToArchiver || this.config.fishermanMode) {
|
|
1541
|
+
return;
|
|
1542
|
+
}
|
|
1543
|
+
const startBlock = BlockNumber(this.syncedToBlockNumber + 1);
|
|
1544
|
+
this.log.debug(`Syncing proposed checkpoint ${this.checkpointNumber} to archiver`, {
|
|
1545
|
+
checkpointNumber: this.checkpointNumber,
|
|
1546
|
+
slot: this.targetSlot,
|
|
1547
|
+
startBlock,
|
|
1548
|
+
blockCount,
|
|
1549
|
+
});
|
|
1550
|
+
await this.blockSink.addProposedCheckpoint({
|
|
1551
|
+
header: checkpoint.header,
|
|
1552
|
+
checkpointNumber: this.checkpointNumber,
|
|
1553
|
+
startBlock,
|
|
1554
|
+
blockCount,
|
|
1555
|
+
totalManaUsed: checkpoint.header.totalManaUsed.toBigInt(),
|
|
1556
|
+
feeAssetPriceModifier,
|
|
1557
|
+
});
|
|
1558
|
+
}
|
|
1559
|
+
|
|
977
1560
|
/** Runs fee analysis and logs checkpoint outcome as fisherman */
|
|
978
1561
|
private async handleCheckpointEndAsFisherman(checkpoint: Checkpoint | undefined) {
|
|
979
1562
|
// Perform L1 fee analysis before clearing requests
|
|
@@ -1026,16 +1609,17 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
1026
1609
|
protected async waitUntilTimeInSlot(targetSecondsIntoSlot: number): Promise<void> {
|
|
1027
1610
|
const slotStartTimestamp = this.getSlotStartBuildTimestamp();
|
|
1028
1611
|
const targetTimestamp = slotStartTimestamp + targetSecondsIntoSlot;
|
|
1029
|
-
|
|
1612
|
+
const delayMs = targetTimestamp * 1000 - this.dateProvider.nowAsDate().getTime();
|
|
1613
|
+
await this.awaitInterruptibleSleep(delayMs);
|
|
1030
1614
|
}
|
|
1031
1615
|
|
|
1032
|
-
/**
|
|
1616
|
+
/** Pause between mempool polls in waitForMinTxs. Extracted for test overriding. */
|
|
1033
1617
|
protected async waitForTxsPollingInterval(): Promise<void> {
|
|
1034
|
-
await
|
|
1618
|
+
await this.awaitInterruptibleSleep(TXS_POLLING_MS);
|
|
1035
1619
|
}
|
|
1036
1620
|
|
|
1037
1621
|
private getSlotStartBuildTimestamp(): number {
|
|
1038
|
-
return getSlotStartBuildTimestamp(this.
|
|
1622
|
+
return getSlotStartBuildTimestamp(this.slotNow, this.l1Constants);
|
|
1039
1623
|
}
|
|
1040
1624
|
|
|
1041
1625
|
private getSecondsIntoSlot(): number {
|