@aztec/sequencer-client 0.0.1-commit.684755437 → 0.0.1-commit.69c59a8b3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dest/client/sequencer-client.d.ts +6 -12
- package/dest/client/sequencer-client.d.ts.map +1 -1
- package/dest/client/sequencer-client.js +30 -80
- package/dest/config.d.ts +4 -3
- package/dest/config.d.ts.map +1 -1
- package/dest/config.js +17 -13
- package/dest/global_variable_builder/fee_predictor.d.ts +37 -0
- package/dest/global_variable_builder/fee_predictor.d.ts.map +1 -0
- package/dest/global_variable_builder/fee_predictor.js +128 -0
- package/dest/global_variable_builder/fee_provider.d.ts +21 -0
- package/dest/global_variable_builder/fee_provider.d.ts.map +1 -0
- package/dest/global_variable_builder/fee_provider.js +58 -0
- package/dest/global_variable_builder/global_builder.d.ts +15 -14
- package/dest/global_variable_builder/global_builder.d.ts.map +1 -1
- package/dest/global_variable_builder/global_builder.js +16 -50
- package/dest/global_variable_builder/index.d.ts +4 -2
- package/dest/global_variable_builder/index.d.ts.map +1 -1
- package/dest/global_variable_builder/index.js +2 -0
- package/dest/publisher/config.d.ts +13 -1
- package/dest/publisher/config.d.ts.map +1 -1
- package/dest/publisher/config.js +19 -4
- package/dest/publisher/l1_tx_failed_store/failed_tx_store.d.ts +3 -4
- package/dest/publisher/l1_tx_failed_store/failed_tx_store.d.ts.map +1 -1
- package/dest/publisher/sequencer-publisher-factory.d.ts +3 -5
- package/dest/publisher/sequencer-publisher-factory.d.ts.map +1 -1
- package/dest/publisher/sequencer-publisher-factory.js +2 -3
- package/dest/publisher/sequencer-publisher.d.ts +51 -43
- package/dest/publisher/sequencer-publisher.d.ts.map +1 -1
- package/dest/publisher/sequencer-publisher.js +135 -111
- package/dest/sequencer/chain_state_overrides.d.ts +25 -0
- package/dest/sequencer/chain_state_overrides.d.ts.map +1 -0
- package/dest/sequencer/chain_state_overrides.js +39 -0
- package/dest/sequencer/checkpoint_proposal_job.d.ts +26 -7
- package/dest/sequencer/checkpoint_proposal_job.d.ts.map +1 -1
- package/dest/sequencer/checkpoint_proposal_job.js +296 -185
- package/dest/sequencer/checkpoint_proposal_job_metrics.d.ts +34 -0
- package/dest/sequencer/checkpoint_proposal_job_metrics.d.ts.map +1 -0
- package/dest/sequencer/checkpoint_proposal_job_metrics.js +72 -0
- package/dest/sequencer/checkpoint_voter.d.ts +1 -2
- package/dest/sequencer/checkpoint_voter.d.ts.map +1 -1
- package/dest/sequencer/checkpoint_voter.js +2 -5
- package/dest/sequencer/events.d.ts +2 -1
- package/dest/sequencer/events.d.ts.map +1 -1
- package/dest/sequencer/metrics.d.ts +9 -10
- package/dest/sequencer/metrics.d.ts.map +1 -1
- package/dest/sequencer/metrics.js +23 -20
- package/dest/sequencer/sequencer.d.ts +22 -7
- package/dest/sequencer/sequencer.d.ts.map +1 -1
- package/dest/sequencer/sequencer.js +136 -73
- package/dest/sequencer/timetable.d.ts +14 -1
- package/dest/sequencer/timetable.d.ts.map +1 -1
- package/dest/sequencer/timetable.js +45 -36
- package/dest/sequencer/types.d.ts +2 -5
- package/dest/sequencer/types.d.ts.map +1 -1
- package/dest/test/mock_checkpoint_builder.d.ts +4 -4
- package/dest/test/mock_checkpoint_builder.d.ts.map +1 -1
- package/package.json +27 -28
- package/src/client/sequencer-client.ts +42 -108
- package/src/config.ts +20 -12
- package/src/global_variable_builder/README.md +44 -0
- package/src/global_variable_builder/fee_predictor.ts +172 -0
- package/src/global_variable_builder/fee_provider.ts +75 -0
- package/src/global_variable_builder/global_builder.ts +26 -62
- package/src/global_variable_builder/index.ts +3 -1
- package/src/publisher/config.ts +38 -4
- package/src/publisher/l1_tx_failed_store/failed_tx_store.ts +3 -1
- package/src/publisher/sequencer-publisher-factory.ts +3 -6
- package/src/publisher/sequencer-publisher.ts +202 -158
- package/src/sequencer/README.md +83 -13
- package/src/sequencer/chain_state_overrides.ts +87 -0
- package/src/sequencer/checkpoint_proposal_job.ts +383 -206
- package/src/sequencer/checkpoint_proposal_job_metrics.ts +128 -0
- package/src/sequencer/checkpoint_voter.ts +1 -12
- package/src/sequencer/events.ts +1 -1
- package/src/sequencer/metrics.ts +29 -25
- package/src/sequencer/sequencer.ts +197 -80
- package/src/sequencer/timetable.ts +57 -45
- package/src/sequencer/types.ts +2 -5
- package/src/test/mock_checkpoint_builder.ts +3 -3
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import type { EpochCache } from '@aztec/epoch-cache';
|
|
2
|
+
import type { SimulationOverridesPlan } from '@aztec/ethereum/contracts';
|
|
2
3
|
import {
|
|
3
4
|
BlockNumber,
|
|
4
5
|
CheckpointNumber,
|
|
@@ -30,17 +31,22 @@ import {
|
|
|
30
31
|
type L2BlockSource,
|
|
31
32
|
MaliciousCommitteeAttestationsAndSigners,
|
|
32
33
|
} from '@aztec/stdlib/block';
|
|
33
|
-
import { type Checkpoint, validateCheckpoint } from '@aztec/stdlib/checkpoint';
|
|
34
|
-
import { getSlotStartBuildTimestamp } from '@aztec/stdlib/epoch-helpers';
|
|
34
|
+
import { type Checkpoint, type ProposedCheckpointData, validateCheckpoint } from '@aztec/stdlib/checkpoint';
|
|
35
|
+
import { computeQuorum, getSlotStartBuildTimestamp, getTimestampForSlot } from '@aztec/stdlib/epoch-helpers';
|
|
35
36
|
import { Gas } from '@aztec/stdlib/gas';
|
|
36
37
|
import {
|
|
37
|
-
|
|
38
|
-
|
|
38
|
+
type BlockBuilderOptions,
|
|
39
|
+
InsufficientValidTxsError,
|
|
39
40
|
type ResolvedSequencerConfig,
|
|
40
41
|
type WorldStateSynchronizer,
|
|
41
42
|
} from '@aztec/stdlib/interfaces/server';
|
|
42
43
|
import { type L1ToL2MessageSource, computeInHashFromL1ToL2Messages } from '@aztec/stdlib/messaging';
|
|
43
|
-
import type {
|
|
44
|
+
import type {
|
|
45
|
+
BlockProposal,
|
|
46
|
+
BlockProposalOptions,
|
|
47
|
+
CheckpointProposal,
|
|
48
|
+
CheckpointProposalOptions,
|
|
49
|
+
} from '@aztec/stdlib/p2p';
|
|
44
50
|
import { orderAttestations, trimAttestations } from '@aztec/stdlib/p2p';
|
|
45
51
|
import type { L2BlockBuiltStats } from '@aztec/stdlib/stats';
|
|
46
52
|
import { type FailedTx, Tx } from '@aztec/stdlib/tx';
|
|
@@ -51,6 +57,11 @@ import { DutyAlreadySignedError, SlashingProtectionError } from '@aztec/validato
|
|
|
51
57
|
|
|
52
58
|
import type { GlobalVariableBuilder } from '../global_variable_builder/global_builder.js';
|
|
53
59
|
import type { InvalidateCheckpointRequest, SequencerPublisher } from '../publisher/sequencer-publisher.js';
|
|
60
|
+
import {
|
|
61
|
+
buildPipelinedParentSimulationOverridesPlan,
|
|
62
|
+
buildSubmissionSimulationOverridesPlan,
|
|
63
|
+
} from './chain_state_overrides.js';
|
|
64
|
+
import type { CheckpointProposalJobMetricsRecorder } from './checkpoint_proposal_job_metrics.js';
|
|
54
65
|
import { CheckpointVoter } from './checkpoint_voter.js';
|
|
55
66
|
import { SequencerInterruptedError } from './errors.js';
|
|
56
67
|
import type { SequencerEvents } from './events.js';
|
|
@@ -62,6 +73,20 @@ import { SequencerState } from './utils.js';
|
|
|
62
73
|
/** How much time to sleep while waiting for min transactions to accumulate for a block */
|
|
63
74
|
const TXS_POLLING_MS = 500;
|
|
64
75
|
|
|
76
|
+
/** Result from proposeCheckpoint when a checkpoint was successfully built and broadcast. */
|
|
77
|
+
type CheckpointProposalBroadcast = {
|
|
78
|
+
checkpoint: Checkpoint;
|
|
79
|
+
proposal: CheckpointProposal;
|
|
80
|
+
blockProposedAt: number;
|
|
81
|
+
};
|
|
82
|
+
|
|
83
|
+
/** Result after attestation collection and signing, ready for L1 submission. */
|
|
84
|
+
type CheckpointProposalResult = {
|
|
85
|
+
checkpoint: Checkpoint;
|
|
86
|
+
attestations: CommitteeAttestationsAndSigners;
|
|
87
|
+
attestationsSignature: Signature;
|
|
88
|
+
};
|
|
89
|
+
|
|
65
90
|
/**
|
|
66
91
|
* Handles the execution of a checkpoint proposal after the initial preparation phase.
|
|
67
92
|
* This includes building blocks, collecting attestations, and publishing the checkpoint to L1,
|
|
@@ -71,9 +96,16 @@ const TXS_POLLING_MS = 500;
|
|
|
71
96
|
export class CheckpointProposalJob implements Traceable {
|
|
72
97
|
protected readonly log: Logger;
|
|
73
98
|
|
|
99
|
+
/** Tracks the fire-and-forget L1 submission promise so it can be awaited during shutdown. */
|
|
100
|
+
private pendingL1Submission: Promise<void> | undefined;
|
|
101
|
+
|
|
102
|
+
/** Pipelined parent chain state used while building and later submitting this checkpoint. */
|
|
103
|
+
private pipelinedParentSimulationOverridesPlan?: SimulationOverridesPlan;
|
|
104
|
+
|
|
74
105
|
constructor(
|
|
75
|
-
private readonly
|
|
76
|
-
private readonly
|
|
106
|
+
private readonly slotNow: SlotNumber,
|
|
107
|
+
private readonly targetSlot: SlotNumber,
|
|
108
|
+
private readonly targetEpoch: EpochNumber,
|
|
77
109
|
private readonly checkpointNumber: CheckpointNumber,
|
|
78
110
|
private readonly syncedToBlockNumber: BlockNumber,
|
|
79
111
|
// TODO(palla/mbps): Can we remove the proposer in favor of attestorAddress? Need to check fisherman-node flows.
|
|
@@ -96,17 +128,31 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
96
128
|
private readonly epochCache: EpochCache,
|
|
97
129
|
private readonly dateProvider: DateProvider,
|
|
98
130
|
private readonly metrics: SequencerMetrics,
|
|
131
|
+
private readonly checkpointMetrics: CheckpointProposalJobMetricsRecorder,
|
|
99
132
|
private readonly eventEmitter: TypedEventEmitter<SequencerEvents>,
|
|
100
133
|
private readonly setStateFn: (state: SequencerState, slot?: SlotNumber) => void,
|
|
101
134
|
public readonly tracer: Tracer,
|
|
102
135
|
bindings?: LoggerBindings,
|
|
136
|
+
private readonly proposedCheckpointData?: ProposedCheckpointData,
|
|
103
137
|
) {
|
|
104
|
-
this.log = createLogger('sequencer:checkpoint-proposal', {
|
|
138
|
+
this.log = createLogger('sequencer:checkpoint-proposal', {
|
|
139
|
+
...bindings,
|
|
140
|
+
instanceId: `slot-${this.slotNow}`,
|
|
141
|
+
});
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
/** Awaits the pending L1 submission if one is in progress. Call during shutdown. */
|
|
145
|
+
public async awaitPendingSubmission(): Promise<void> {
|
|
146
|
+
this.log.info('Awaiting pending L1 payload submission');
|
|
147
|
+
await this.pendingL1Submission;
|
|
105
148
|
}
|
|
106
149
|
|
|
107
150
|
/**
|
|
108
151
|
* Executes the checkpoint proposal job.
|
|
109
|
-
*
|
|
152
|
+
* Builds blocks, assembles checkpoint, and broadcasts the proposal (blocking).
|
|
153
|
+
* Attestation collection, signing, and L1 submission are backgrounded so the
|
|
154
|
+
* work loop can return to IDLE immediately for consecutive slot proposals.
|
|
155
|
+
* Returns the built checkpoint if successful, undefined otherwise.
|
|
110
156
|
*/
|
|
111
157
|
@trackSpan('CheckpointProposalJob.execute')
|
|
112
158
|
public async execute(): Promise<Checkpoint | undefined> {
|
|
@@ -114,7 +160,7 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
114
160
|
// In fisherman mode, we simulate slashing but don't actually publish to L1
|
|
115
161
|
// These are constant for the whole slot, so we only enqueue them once
|
|
116
162
|
const votesPromises = new CheckpointVoter(
|
|
117
|
-
this.
|
|
163
|
+
this.targetSlot,
|
|
118
164
|
this.publisher,
|
|
119
165
|
this.attestorAddress,
|
|
120
166
|
this.validatorClient,
|
|
@@ -125,63 +171,195 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
125
171
|
this.log,
|
|
126
172
|
).enqueueVotes();
|
|
127
173
|
|
|
128
|
-
// Build
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
// Wait until the voting promises have resolved, so all requests are enqueued (not sent)
|
|
132
|
-
await Promise.all(votesPromises);
|
|
174
|
+
// Build blocks, assemble checkpoint, and broadcast proposal (BLOCKING).
|
|
175
|
+
// Returns after broadcast — attestation collection is deferred.
|
|
176
|
+
const broadcast = await this.proposeCheckpoint();
|
|
133
177
|
|
|
134
|
-
if (
|
|
135
|
-
|
|
178
|
+
if (!broadcast) {
|
|
179
|
+
await Promise.all(votesPromises);
|
|
180
|
+
// Still submit votes even without a checkpoint
|
|
181
|
+
if (!this.config.fishermanMode) {
|
|
182
|
+
this.pendingL1Submission = this.publisher.sendRequestsAt(this.dateProvider.nowAsDate()).then(() => {});
|
|
183
|
+
}
|
|
184
|
+
return undefined;
|
|
136
185
|
}
|
|
137
186
|
|
|
187
|
+
const { checkpoint } = broadcast;
|
|
188
|
+
this.metrics.recordCheckpointProposalSuccess();
|
|
189
|
+
|
|
138
190
|
// Do not post anything to L1 if we are fishermen, but do perform L1 fee analysis
|
|
139
191
|
if (this.config.fishermanMode) {
|
|
140
192
|
await this.handleCheckpointEndAsFisherman(checkpoint);
|
|
141
|
-
return;
|
|
193
|
+
return checkpoint;
|
|
142
194
|
}
|
|
143
195
|
|
|
144
|
-
//
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
196
|
+
// Background the attestation → signing → L1 pipeline so the work loop is unblocked
|
|
197
|
+
this.pendingL1Submission = this.waitForAttestationsAndEnqueueSubmissionAsync(broadcast, votesPromises);
|
|
198
|
+
|
|
199
|
+
// Return the built checkpoint immediately — the work loop is now unblocked
|
|
200
|
+
return checkpoint;
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
/**
|
|
204
|
+
* Background pipeline: collects attestations, signs them, enqueues the checkpoint, and submits to L1.
|
|
205
|
+
* Runs as a fire-and-forget task stored in `pendingL1Submission` so the work loop is unblocked.
|
|
206
|
+
*/
|
|
207
|
+
private async waitForAttestationsAndEnqueueSubmissionAsync(
|
|
208
|
+
broadcast: CheckpointProposalBroadcast,
|
|
209
|
+
votesPromises: Promise<unknown>[],
|
|
210
|
+
): Promise<void> {
|
|
211
|
+
const { checkpoint, proposal, blockProposedAt } = broadcast;
|
|
212
|
+
try {
|
|
213
|
+
await Promise.all(votesPromises);
|
|
214
|
+
|
|
215
|
+
this.setStateFn(SequencerState.COLLECTING_ATTESTATIONS, this.targetSlot);
|
|
216
|
+
const attestations = await this.waitForAttestations(proposal);
|
|
217
|
+
|
|
218
|
+
this.checkpointMetrics.recordCheckpointAttestationDelay(this.dateProvider.now() - blockProposedAt);
|
|
219
|
+
|
|
220
|
+
// Proposer must sign over the attestations before pushing them to L1
|
|
221
|
+
const signer = this.proposer ?? this.publisher.getSenderAddress();
|
|
222
|
+
let attestationsSignature: Signature;
|
|
223
|
+
try {
|
|
224
|
+
attestationsSignature = await this.validatorClient.signAttestationsAndSigners(
|
|
225
|
+
attestations,
|
|
226
|
+
signer,
|
|
227
|
+
this.targetSlot,
|
|
228
|
+
this.checkpointNumber,
|
|
229
|
+
);
|
|
230
|
+
} catch (err) {
|
|
231
|
+
if (this.handleHASigningError(err, 'Attestations signature')) {
|
|
232
|
+
return;
|
|
233
|
+
}
|
|
234
|
+
throw err;
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
// Enqueue the checkpoint for L1 submission
|
|
238
|
+
await this.enqueueCheckpointForSubmission({ checkpoint, attestations, attestationsSignature });
|
|
239
|
+
|
|
240
|
+
// Compute the earliest time to submit: pipeline slot start when pipelining, now otherwise.
|
|
241
|
+
const submitAfter = this.epochCache.isProposerPipeliningEnabled()
|
|
242
|
+
? new Date(Number(getTimestampForSlot(this.targetSlot, this.l1Constants)) * 1000)
|
|
243
|
+
: new Date(this.dateProvider.now());
|
|
244
|
+
|
|
245
|
+
const l1Response = await this.publisher.sendRequestsAt(submitAfter);
|
|
246
|
+
const proposedAction = l1Response?.successfulActions.find(a => a === 'propose');
|
|
247
|
+
if (proposedAction) {
|
|
248
|
+
this.eventEmitter.emit('checkpoint-published', { checkpoint: this.checkpointNumber, slot: this.targetSlot });
|
|
249
|
+
const coinbase = checkpoint.header.coinbase;
|
|
250
|
+
await this.metrics.incFilledSlot(this.publisher.getSenderAddress().toString(), coinbase);
|
|
251
|
+
} else {
|
|
252
|
+
this.eventEmitter.emit('checkpoint-publish-failed', { ...l1Response, slot: this.targetSlot });
|
|
253
|
+
if (this.epochCache.isProposerPipeliningEnabled()) {
|
|
254
|
+
this.metrics.recordPipelineDiscard();
|
|
255
|
+
}
|
|
256
|
+
}
|
|
257
|
+
} catch (err) {
|
|
258
|
+
if (err instanceof SequencerInterruptedError) {
|
|
259
|
+
return;
|
|
260
|
+
}
|
|
261
|
+
this.log.error(`Background attestation/L1 pipeline failed for slot ${this.targetSlot}`, err);
|
|
262
|
+
this.eventEmitter.emit('checkpoint-publish-failed', { slot: this.targetSlot });
|
|
263
|
+
if (this.epochCache.isProposerPipeliningEnabled()) {
|
|
264
|
+
this.metrics.recordPipelineDiscard();
|
|
265
|
+
}
|
|
155
266
|
}
|
|
156
267
|
}
|
|
157
268
|
|
|
269
|
+
/** Enqueues the checkpoint for L1 submission. Called after pipeline sleep in execute(). */
|
|
270
|
+
private async enqueueCheckpointForSubmission(result: CheckpointProposalResult): Promise<void> {
|
|
271
|
+
const { checkpoint, attestations, attestationsSignature } = result;
|
|
272
|
+
|
|
273
|
+
this.setStateFn(SequencerState.PUBLISHING_CHECKPOINT, this.targetSlot);
|
|
274
|
+
const aztecSlotDuration = this.l1Constants.slotDuration;
|
|
275
|
+
const submissionSlotStart = Number(getTimestampForSlot(this.targetSlot, this.l1Constants));
|
|
276
|
+
const txTimeoutAt = new Date((submissionSlotStart + aztecSlotDuration) * 1000);
|
|
277
|
+
|
|
278
|
+
// If we have been configured to potentially skip publishing checkpoint then roll the dice here
|
|
279
|
+
if (
|
|
280
|
+
this.config.skipPublishingCheckpointsPercent !== undefined &&
|
|
281
|
+
this.config.skipPublishingCheckpointsPercent > 0
|
|
282
|
+
) {
|
|
283
|
+
const roll = Math.max(0, randomInt(100));
|
|
284
|
+
if (roll < this.config.skipPublishingCheckpointsPercent) {
|
|
285
|
+
this.log.warn(
|
|
286
|
+
`Skipping publishing proposal for checkpoint ${checkpoint.number}. Configured percentage: ${this.config.skipPublishingCheckpointsPercent}, generated value: ${roll}`,
|
|
287
|
+
);
|
|
288
|
+
return;
|
|
289
|
+
}
|
|
290
|
+
}
|
|
291
|
+
|
|
292
|
+
const isPipelining = this.epochCache.isProposerPipeliningEnabled();
|
|
293
|
+
const submissionSimulationOverridesPlan = buildSubmissionSimulationOverridesPlan({
|
|
294
|
+
pipelinedParentPlan: this.pipelinedParentSimulationOverridesPlan,
|
|
295
|
+
invalidateToPendingCheckpointNumber: this.invalidateCheckpoint?.forcePendingCheckpointNumber,
|
|
296
|
+
lastArchiveRoot: checkpoint.header.lastArchiveRoot,
|
|
297
|
+
pipeliningEnabled: isPipelining,
|
|
298
|
+
});
|
|
299
|
+
|
|
300
|
+
await this.publisher.enqueueProposeCheckpoint(checkpoint, attestations, attestationsSignature, {
|
|
301
|
+
txTimeoutAt,
|
|
302
|
+
...(submissionSimulationOverridesPlan ? { simulationOverridesPlan: submissionSimulationOverridesPlan } : {}),
|
|
303
|
+
});
|
|
304
|
+
}
|
|
305
|
+
|
|
158
306
|
@trackSpan('CheckpointProposalJob.proposeCheckpoint', function () {
|
|
159
307
|
return {
|
|
160
308
|
// nullish operator needed for tests
|
|
161
309
|
[Attributes.COINBASE]: this.validatorClient.getCoinbaseForAttestor(this.attestorAddress)?.toString(),
|
|
162
|
-
[Attributes.SLOT_NUMBER]: this.
|
|
310
|
+
[Attributes.SLOT_NUMBER]: this.targetSlot,
|
|
163
311
|
};
|
|
164
312
|
})
|
|
165
|
-
private async proposeCheckpoint(): Promise<
|
|
313
|
+
private async proposeCheckpoint(): Promise<CheckpointProposalBroadcast | undefined> {
|
|
166
314
|
try {
|
|
315
|
+
const now = this.dateProvider.now();
|
|
316
|
+
if (this.epochCache.isProposerPipeliningEnabled() && this.proposedCheckpointData) {
|
|
317
|
+
// Measure against the wall-clock slot whose build window we are currently using.
|
|
318
|
+
// In pipelining mode `targetSlot` is intentionally one slot ahead, which makes the
|
|
319
|
+
// target-slot boundary a full slot away from the actual build start time.
|
|
320
|
+
const slotBoundaryMs = Number(getTimestampForSlot(this.slotNow, this.l1Constants)) * 1000;
|
|
321
|
+
this.checkpointMetrics.recordPipelinedCheckpointBuildStartOffsetFromSlotBoundary(now - slotBoundaryMs);
|
|
322
|
+
}
|
|
323
|
+
this.checkpointMetrics.startCheckpointTiming(now);
|
|
324
|
+
|
|
167
325
|
// Get operator configured coinbase and fee recipient for this attestor
|
|
168
326
|
const coinbase = this.validatorClient.getCoinbaseForAttestor(this.attestorAddress);
|
|
169
327
|
const feeRecipient = this.validatorClient.getFeeRecipientForAttestor(this.attestorAddress);
|
|
170
328
|
|
|
171
329
|
// Start the checkpoint
|
|
172
|
-
this.setStateFn(SequencerState.INITIALIZING_CHECKPOINT, this.
|
|
173
|
-
this.
|
|
330
|
+
this.setStateFn(SequencerState.INITIALIZING_CHECKPOINT, this.targetSlot);
|
|
331
|
+
this.log.info(`Starting checkpoint proposal`, {
|
|
332
|
+
buildSlot: this.slotNow,
|
|
333
|
+
submissionSlot: this.targetSlot,
|
|
334
|
+
pipelining: this.epochCache.isProposerPipeliningEnabled(),
|
|
335
|
+
proposer: this.proposer?.toString(),
|
|
336
|
+
coinbase: coinbase.toString(),
|
|
337
|
+
});
|
|
338
|
+
this.metrics.incOpenSlot(this.targetSlot, this.proposer?.toString() ?? 'unknown');
|
|
174
339
|
|
|
175
340
|
// Enqueues checkpoint invalidation (constant for the whole slot)
|
|
176
341
|
if (this.invalidateCheckpoint && !this.config.skipInvalidateBlockAsProposer) {
|
|
177
342
|
this.publisher.enqueueInvalidateCheckpoint(this.invalidateCheckpoint);
|
|
178
343
|
}
|
|
179
344
|
|
|
180
|
-
// Create checkpoint builder for the slot
|
|
345
|
+
// Create checkpoint builder for the slot.
|
|
346
|
+
// When pipelining, force the proposed checkpoint number and fee header to our parent so the
|
|
347
|
+
// fee computation sees the same chain tip that L1 will see once the previous pipelined checkpoint lands.
|
|
348
|
+
const isPipelining = this.epochCache.isProposerPipeliningEnabled();
|
|
349
|
+
this.pipelinedParentSimulationOverridesPlan = isPipelining
|
|
350
|
+
? await buildPipelinedParentSimulationOverridesPlan({
|
|
351
|
+
checkpointNumber: this.checkpointNumber,
|
|
352
|
+
proposedCheckpointData: this.proposedCheckpointData,
|
|
353
|
+
rollup: this.publisher.rollupContract,
|
|
354
|
+
log: this.log,
|
|
355
|
+
})
|
|
356
|
+
: undefined;
|
|
357
|
+
|
|
181
358
|
const checkpointGlobalVariables = await this.globalsBuilder.buildCheckpointGlobalVariables(
|
|
182
359
|
coinbase,
|
|
183
360
|
feeRecipient,
|
|
184
|
-
this.
|
|
361
|
+
this.targetSlot,
|
|
362
|
+
this.pipelinedParentSimulationOverridesPlan,
|
|
185
363
|
);
|
|
186
364
|
|
|
187
365
|
// Collect L1 to L2 messages for the checkpoint and compute their hash
|
|
@@ -189,7 +367,7 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
189
367
|
const inHash = computeInHashFromL1ToL2Messages(l1ToL2Messages);
|
|
190
368
|
|
|
191
369
|
// Collect the out hashes of all the checkpoints before this one in the same epoch
|
|
192
|
-
const previousCheckpointOutHashes = (await this.l2BlockSource.getCheckpointsDataForEpoch(this.
|
|
370
|
+
const previousCheckpointOutHashes = (await this.l2BlockSource.getCheckpointsDataForEpoch(this.targetEpoch))
|
|
193
371
|
.filter(c => c.checkpointNumber < this.checkpointNumber)
|
|
194
372
|
.map(c => c.checkpointOutHash);
|
|
195
373
|
|
|
@@ -222,7 +400,7 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
222
400
|
};
|
|
223
401
|
|
|
224
402
|
let blocksInCheckpoint: L2Block[] = [];
|
|
225
|
-
let blockPendingBroadcast:
|
|
403
|
+
let blockPendingBroadcast: BlockProposal | undefined = undefined;
|
|
226
404
|
const checkpointBuildTimer = new Timer();
|
|
227
405
|
|
|
228
406
|
try {
|
|
@@ -246,8 +424,8 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
246
424
|
}
|
|
247
425
|
|
|
248
426
|
if (blocksInCheckpoint.length === 0) {
|
|
249
|
-
this.log.warn(`No blocks were built for slot ${this.
|
|
250
|
-
this.eventEmitter.emit('checkpoint-empty', { slot: this.
|
|
427
|
+
this.log.warn(`No blocks were built for slot ${this.targetSlot}`, { slot: this.targetSlot });
|
|
428
|
+
this.eventEmitter.emit('checkpoint-empty', { slot: this.targetSlot });
|
|
251
429
|
return undefined;
|
|
252
430
|
}
|
|
253
431
|
|
|
@@ -255,17 +433,18 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
255
433
|
if (minBlocksForCheckpoint !== undefined && blocksInCheckpoint.length < minBlocksForCheckpoint) {
|
|
256
434
|
this.log.warn(
|
|
257
435
|
`Checkpoint has fewer blocks than minimum (${blocksInCheckpoint.length} < ${minBlocksForCheckpoint}), skipping proposal`,
|
|
258
|
-
{ slot: this.
|
|
436
|
+
{ slot: this.targetSlot, blocksBuilt: blocksInCheckpoint.length, minBlocksForCheckpoint },
|
|
259
437
|
);
|
|
260
438
|
return undefined;
|
|
261
439
|
}
|
|
262
440
|
|
|
263
441
|
// Assemble and broadcast the checkpoint proposal, including the last block that was not
|
|
264
442
|
// broadcasted yet, and wait to collect the committee attestations.
|
|
265
|
-
this.setStateFn(SequencerState.ASSEMBLING_CHECKPOINT, this.
|
|
443
|
+
this.setStateFn(SequencerState.ASSEMBLING_CHECKPOINT, this.targetSlot);
|
|
266
444
|
const checkpoint = await checkpointBuilder.completeCheckpoint();
|
|
267
445
|
|
|
268
|
-
// Final validation
|
|
446
|
+
// Final validation: per-block limits are only checked if the operator set them explicitly.
|
|
447
|
+
// Otherwise, checkpoint-level budgets were already enforced by the redistribution logic.
|
|
269
448
|
try {
|
|
270
449
|
validateCheckpoint(checkpoint, {
|
|
271
450
|
rollupManaLimit: this.l1Constants.rollupManaLimit,
|
|
@@ -275,113 +454,60 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
275
454
|
maxTxsPerCheckpoint: this.config.maxTxsPerCheckpoint,
|
|
276
455
|
});
|
|
277
456
|
} catch (err) {
|
|
278
|
-
this.log.error(`Built an invalid checkpoint at slot ${this.
|
|
457
|
+
this.log.error(`Built an invalid checkpoint at slot ${this.slotNow} (skipping proposal)`, err, {
|
|
279
458
|
checkpoint: checkpoint.header.toInspect(),
|
|
280
459
|
});
|
|
281
460
|
return undefined;
|
|
282
461
|
}
|
|
283
462
|
|
|
284
463
|
// Record checkpoint-level build metrics
|
|
285
|
-
this.
|
|
464
|
+
this.checkpointMetrics.recordCheckpointBuild(
|
|
286
465
|
checkpointBuildTimer.ms(),
|
|
287
466
|
blocksInCheckpoint.length,
|
|
288
467
|
checkpoint.getStats().txCount,
|
|
289
468
|
Number(checkpoint.header.totalManaUsed.toBigInt()),
|
|
290
469
|
);
|
|
291
470
|
|
|
292
|
-
//
|
|
471
|
+
// In fisherman mode, return the checkpoint without broadcasting or collecting attestations
|
|
293
472
|
if (this.config.fishermanMode) {
|
|
294
473
|
this.log.info(
|
|
295
|
-
`Built checkpoint for slot ${this.
|
|
474
|
+
`Built checkpoint for slot ${this.targetSlot} with ${blocksInCheckpoint.length} blocks. ` +
|
|
296
475
|
`Skipping proposal in fisherman mode.`,
|
|
297
476
|
{
|
|
298
|
-
slot: this.
|
|
477
|
+
slot: this.targetSlot,
|
|
299
478
|
checkpoint: checkpoint.header.toInspect(),
|
|
300
479
|
blocksBuilt: blocksInCheckpoint.length,
|
|
301
480
|
},
|
|
302
481
|
);
|
|
303
482
|
this.metrics.recordCheckpointSuccess();
|
|
304
|
-
|
|
483
|
+
// Return a broadcast result with a dummy proposal — fisherman mode skips attestation collection
|
|
484
|
+
return { checkpoint, proposal: undefined!, blockProposedAt: this.dateProvider.now() };
|
|
305
485
|
}
|
|
306
486
|
|
|
307
|
-
// Include the block pending broadcast in the checkpoint proposal if any
|
|
308
|
-
const lastBlock = blockPendingBroadcast && {
|
|
309
|
-
blockHeader: blockPendingBroadcast.block.header,
|
|
310
|
-
indexWithinCheckpoint: blockPendingBroadcast.block.indexWithinCheckpoint,
|
|
311
|
-
txs: blockPendingBroadcast.txs,
|
|
312
|
-
};
|
|
313
|
-
|
|
314
487
|
// Create the checkpoint proposal and broadcast it
|
|
315
488
|
const proposal = await this.validatorClient.createCheckpointProposal(
|
|
316
489
|
checkpoint.header,
|
|
317
490
|
checkpoint.archive.root,
|
|
491
|
+
this.checkpointNumber,
|
|
318
492
|
feeAssetPriceModifier,
|
|
319
|
-
|
|
493
|
+
blockPendingBroadcast,
|
|
320
494
|
this.proposer,
|
|
321
495
|
checkpointProposalOptions,
|
|
322
496
|
);
|
|
323
497
|
|
|
324
498
|
const blockProposedAt = this.dateProvider.now();
|
|
325
499
|
await this.p2pClient.broadcastCheckpointProposal(proposal);
|
|
500
|
+
this.checkpointMetrics.noteCheckpointBroadcast(this.dateProvider.now());
|
|
326
501
|
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
const blockAttestedAt = this.dateProvider.now();
|
|
330
|
-
|
|
331
|
-
this.metrics.recordCheckpointAttestationDelay(blockAttestedAt - blockProposedAt);
|
|
332
|
-
|
|
333
|
-
// Proposer must sign over the attestations before pushing them to L1
|
|
334
|
-
const signer = this.proposer ?? this.publisher.getSenderAddress();
|
|
335
|
-
let attestationsSignature: Signature;
|
|
336
|
-
try {
|
|
337
|
-
attestationsSignature = await this.validatorClient.signAttestationsAndSigners(
|
|
338
|
-
attestations,
|
|
339
|
-
signer,
|
|
340
|
-
this.slot,
|
|
341
|
-
this.checkpointNumber,
|
|
342
|
-
);
|
|
343
|
-
} catch (err) {
|
|
344
|
-
// We shouldn't really get here since we yield to another HA node
|
|
345
|
-
// as soon as we see these errors when creating block or checkpoint proposals.
|
|
346
|
-
if (this.handleHASigningError(err, 'Attestations signature')) {
|
|
347
|
-
return undefined;
|
|
348
|
-
}
|
|
349
|
-
throw err;
|
|
350
|
-
}
|
|
351
|
-
|
|
352
|
-
// Enqueue publishing the checkpoint to L1
|
|
353
|
-
this.setStateFn(SequencerState.PUBLISHING_CHECKPOINT, this.slot);
|
|
354
|
-
const aztecSlotDuration = this.l1Constants.slotDuration;
|
|
355
|
-
const slotStartBuildTimestamp = this.getSlotStartBuildTimestamp();
|
|
356
|
-
const txTimeoutAt = new Date((slotStartBuildTimestamp + aztecSlotDuration) * 1000);
|
|
357
|
-
|
|
358
|
-
// If we have been configured to potentially skip publishing checkpoint then roll the dice here
|
|
359
|
-
if (
|
|
360
|
-
this.config.skipPublishingCheckpointsPercent !== undefined &&
|
|
361
|
-
this.config.skipPublishingCheckpointsPercent > 0
|
|
362
|
-
) {
|
|
363
|
-
const result = Math.max(0, randomInt(100));
|
|
364
|
-
if (result < this.config.skipPublishingCheckpointsPercent) {
|
|
365
|
-
this.log.warn(
|
|
366
|
-
`Skipping publishing proposal for checkpoint ${checkpoint.number}. Configured percentage: ${this.config.skipPublishingCheckpointsPercent}, generated value: ${result}`,
|
|
367
|
-
);
|
|
368
|
-
return checkpoint;
|
|
369
|
-
}
|
|
370
|
-
}
|
|
371
|
-
|
|
372
|
-
await this.publisher.enqueueProposeCheckpoint(checkpoint, attestations, attestationsSignature, {
|
|
373
|
-
txTimeoutAt,
|
|
374
|
-
forcePendingCheckpointNumber: this.invalidateCheckpoint?.forcePendingCheckpointNumber,
|
|
375
|
-
});
|
|
376
|
-
|
|
377
|
-
return checkpoint;
|
|
502
|
+
// Return immediately after broadcast — attestation collection happens in the background
|
|
503
|
+
return { checkpoint, proposal, blockProposedAt };
|
|
378
504
|
} catch (err) {
|
|
379
505
|
if (err && (err instanceof DutyAlreadySignedError || err instanceof SlashingProtectionError)) {
|
|
380
506
|
// swallow this error. It's already been logged by a function deeper in the stack
|
|
381
507
|
return undefined;
|
|
382
508
|
}
|
|
383
509
|
|
|
384
|
-
this.log.error(`Error building checkpoint at slot ${this.
|
|
510
|
+
this.log.error(`Error building checkpoint at slot ${this.targetSlot}`, err);
|
|
385
511
|
return undefined;
|
|
386
512
|
}
|
|
387
513
|
}
|
|
@@ -397,14 +523,14 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
397
523
|
blockProposalOptions: BlockProposalOptions,
|
|
398
524
|
): Promise<{
|
|
399
525
|
blocksInCheckpoint: L2Block[];
|
|
400
|
-
blockPendingBroadcast:
|
|
526
|
+
blockPendingBroadcast: BlockProposal | undefined;
|
|
401
527
|
}> {
|
|
402
528
|
const blocksInCheckpoint: L2Block[] = [];
|
|
403
529
|
const txHashesAlreadyIncluded = new Set<string>();
|
|
404
530
|
const initialBlockNumber = BlockNumber(this.syncedToBlockNumber + 1);
|
|
405
531
|
|
|
406
532
|
// Last block in the checkpoint will usually be flagged as pending broadcast, so we send it along with the checkpoint proposal
|
|
407
|
-
let blockPendingBroadcast:
|
|
533
|
+
let blockPendingBroadcast: BlockProposal | undefined = undefined;
|
|
408
534
|
|
|
409
535
|
while (true) {
|
|
410
536
|
const blocksBuilt = blocksInCheckpoint.length;
|
|
@@ -416,7 +542,7 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
416
542
|
|
|
417
543
|
if (!timingInfo.canStart) {
|
|
418
544
|
this.log.debug(`Not enough time left in slot to start another block`, {
|
|
419
|
-
slot: this.
|
|
545
|
+
slot: this.targetSlot,
|
|
420
546
|
blocksBuilt,
|
|
421
547
|
secondsIntoSlot,
|
|
422
548
|
});
|
|
@@ -437,22 +563,23 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
437
563
|
txHashesAlreadyIncluded,
|
|
438
564
|
});
|
|
439
565
|
|
|
440
|
-
//
|
|
441
|
-
if (
|
|
442
|
-
// If
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
566
|
+
// If we failed to build the block due to insufficient txs, we try again if there is still time left in the slot
|
|
567
|
+
if ('failure' in buildResult) {
|
|
568
|
+
// If this was the last subslot, or we're running with a single block per slot, we're done
|
|
569
|
+
if (timingInfo.isLastBlock || timingInfo.deadline === undefined) {
|
|
570
|
+
break;
|
|
571
|
+
}
|
|
572
|
+
// Otherwise, if there is still time for more blocks, we wait until the next subslot and try again
|
|
446
573
|
await this.waitUntilNextSubslot(timingInfo.deadline);
|
|
447
574
|
continue;
|
|
448
|
-
}
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
575
|
+
}
|
|
576
|
+
|
|
577
|
+
// If there was an error building the block, we just exit the loop and give up the rest of the slot.
|
|
578
|
+
// We don't want to risk building more blocks if something went wrong.
|
|
579
|
+
if ('error' in buildResult) {
|
|
453
580
|
if (!(buildResult.error instanceof SequencerInterruptedError)) {
|
|
454
|
-
this.log.warn(`Halting block building for slot ${this.
|
|
455
|
-
slot: this.
|
|
581
|
+
this.log.warn(`Halting block building for slot ${this.targetSlot}`, {
|
|
582
|
+
slot: this.targetSlot,
|
|
456
583
|
blocksBuilt,
|
|
457
584
|
error: buildResult.error,
|
|
458
585
|
});
|
|
@@ -461,61 +588,80 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
461
588
|
}
|
|
462
589
|
|
|
463
590
|
const { block, usedTxs } = buildResult;
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
// Note that the checkpoint builder uses its own fork so it should not need to wait for this syncing
|
|
468
|
-
// Eventually we should refactor the checkpoint builder to not need a separate long-lived fork
|
|
469
|
-
// Fire and forget - don't block the critical path, but log errors
|
|
470
|
-
this.syncProposedBlockToArchiver(block).catch(err => {
|
|
471
|
-
this.log.error(`Failed to sync proposed block ${block.number} to archiver`, { blockNumber: block.number, err });
|
|
591
|
+
this.checkpointMetrics.noteCheckpointBlockBuilt(this.dateProvider.now(), {
|
|
592
|
+
isFirstBlock: blocksBuilt === 0,
|
|
593
|
+
isLastBlock: timingInfo.isLastBlock,
|
|
472
594
|
});
|
|
473
595
|
|
|
596
|
+
blocksInCheckpoint.push(block);
|
|
474
597
|
usedTxs.forEach(tx => txHashesAlreadyIncluded.add(tx.txHash.toString()));
|
|
475
598
|
|
|
476
|
-
//
|
|
599
|
+
// Sign the block proposal. This will throw if HA signing fails.
|
|
600
|
+
const proposal = await this.createBlockProposal(block, inHash, usedTxs, blockProposalOptions);
|
|
601
|
+
|
|
602
|
+
// Sync the proposed block to the archiver to make it available, only after we've managed to sign the proposal,
|
|
603
|
+
// so we avoid polluting our archive with a block that would fail.
|
|
604
|
+
// We wait for the sync to succeed, as this helps catch consistency errors, even if it means we lose some time for block-building.
|
|
605
|
+
// If this throws, we abort the entire checkpoint.
|
|
606
|
+
await this.syncProposedBlockToArchiver(block);
|
|
607
|
+
|
|
608
|
+
// If this is the last block, do not broadcast it, since it will be included in the checkpoint proposal.
|
|
477
609
|
if (timingInfo.isLastBlock) {
|
|
478
|
-
this.log.verbose(`Completed final block ${blockNumber} for slot ${this.
|
|
479
|
-
slot: this.
|
|
610
|
+
this.log.verbose(`Completed final block ${blockNumber} for slot ${this.targetSlot}`, {
|
|
611
|
+
slot: this.targetSlot,
|
|
480
612
|
blockNumber,
|
|
481
613
|
blocksBuilt,
|
|
482
614
|
});
|
|
483
|
-
|
|
615
|
+
|
|
616
|
+
blockPendingBroadcast = proposal;
|
|
484
617
|
break;
|
|
485
618
|
}
|
|
486
619
|
|
|
487
|
-
//
|
|
488
|
-
|
|
489
|
-
if (!this.config.fishermanMode) {
|
|
490
|
-
const proposal = await this.validatorClient.createBlockProposal(
|
|
491
|
-
block.header,
|
|
492
|
-
block.indexWithinCheckpoint,
|
|
493
|
-
inHash,
|
|
494
|
-
block.archive.root,
|
|
495
|
-
usedTxs,
|
|
496
|
-
this.proposer,
|
|
497
|
-
blockProposalOptions,
|
|
498
|
-
);
|
|
499
|
-
await this.p2pClient.broadcastProposal(proposal);
|
|
500
|
-
}
|
|
620
|
+
// Once we have a signed proposal and the archiver agreed with our proposed block, then we broadcast it.
|
|
621
|
+
proposal && (await this.p2pClient.broadcastProposal(proposal));
|
|
501
622
|
|
|
502
623
|
// Wait until the next block's start time
|
|
503
624
|
await this.waitUntilNextSubslot(timingInfo.deadline);
|
|
504
625
|
}
|
|
505
626
|
|
|
506
|
-
this.log.verbose(`Block building loop completed for slot ${this.
|
|
507
|
-
slot: this.
|
|
627
|
+
this.log.verbose(`Block building loop completed for slot ${this.targetSlot}`, {
|
|
628
|
+
slot: this.targetSlot,
|
|
508
629
|
blocksBuilt: blocksInCheckpoint.length,
|
|
509
630
|
});
|
|
510
631
|
|
|
511
632
|
return { blocksInCheckpoint, blockPendingBroadcast };
|
|
512
633
|
}
|
|
513
634
|
|
|
635
|
+
/** Creates a block proposal for a given block via the validator client (unless in fisherman mode) */
|
|
636
|
+
private createBlockProposal(
|
|
637
|
+
block: L2Block,
|
|
638
|
+
inHash: Fr,
|
|
639
|
+
usedTxs: Tx[],
|
|
640
|
+
blockProposalOptions: BlockProposalOptions,
|
|
641
|
+
): Promise<BlockProposal | undefined> {
|
|
642
|
+
if (this.config.fishermanMode) {
|
|
643
|
+
this.log.info(`Skipping block proposal for block ${block.number} in fisherman mode`);
|
|
644
|
+
return Promise.resolve(undefined);
|
|
645
|
+
}
|
|
646
|
+
return this.validatorClient.createBlockProposal(
|
|
647
|
+
block.header,
|
|
648
|
+
this.checkpointNumber,
|
|
649
|
+
block.indexWithinCheckpoint,
|
|
650
|
+
inHash,
|
|
651
|
+
block.archive.root,
|
|
652
|
+
usedTxs,
|
|
653
|
+
this.proposer,
|
|
654
|
+
blockProposalOptions,
|
|
655
|
+
);
|
|
656
|
+
}
|
|
657
|
+
|
|
514
658
|
/** Sleeps until it is time to produce the next block in the slot */
|
|
515
659
|
@trackSpan('CheckpointProposalJob.waitUntilNextSubslot')
|
|
516
660
|
private async waitUntilNextSubslot(nextSubslotStart: number) {
|
|
517
|
-
this.setStateFn(SequencerState.WAITING_UNTIL_NEXT_BLOCK, this.
|
|
518
|
-
this.log.verbose(`Waiting until time for the next block at ${nextSubslotStart}s into slot`, {
|
|
661
|
+
this.setStateFn(SequencerState.WAITING_UNTIL_NEXT_BLOCK, this.targetSlot);
|
|
662
|
+
this.log.verbose(`Waiting until time for the next block at ${nextSubslotStart}s into slot`, {
|
|
663
|
+
slot: this.targetSlot,
|
|
664
|
+
});
|
|
519
665
|
await this.waitUntilTimeInSlot(nextSubslotStart);
|
|
520
666
|
}
|
|
521
667
|
|
|
@@ -531,27 +677,28 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
531
677
|
buildDeadline: Date | undefined;
|
|
532
678
|
txHashesAlreadyIncluded: Set<string>;
|
|
533
679
|
},
|
|
534
|
-
): Promise<
|
|
680
|
+
): Promise<
|
|
681
|
+
{ block: L2Block; usedTxs: Tx[] } | { failure: 'insufficient-txs' | 'insufficient-valid-txs' } | { error: Error }
|
|
682
|
+
> {
|
|
535
683
|
const { blockTimestamp, forceCreate, blockNumber, indexWithinCheckpoint, buildDeadline, txHashesAlreadyIncluded } =
|
|
536
684
|
opts;
|
|
537
685
|
|
|
538
686
|
this.log.verbose(
|
|
539
|
-
`Preparing block ${blockNumber} index ${indexWithinCheckpoint} at checkpoint ${this.checkpointNumber} for slot ${this.
|
|
687
|
+
`Preparing block ${blockNumber} index ${indexWithinCheckpoint} at checkpoint ${this.checkpointNumber} for slot ${this.targetSlot}`,
|
|
540
688
|
{ ...checkpointBuilder.getConstantData(), ...opts },
|
|
541
689
|
);
|
|
542
690
|
|
|
543
691
|
try {
|
|
544
692
|
// Wait until we have enough txs to build the block
|
|
545
|
-
const minTxs = this.
|
|
546
|
-
const { availableTxs, canStartBuilding } = await this.waitForMinTxs(opts);
|
|
693
|
+
const { availableTxs, canStartBuilding, minTxs } = await this.waitForMinTxs(opts);
|
|
547
694
|
if (!canStartBuilding) {
|
|
548
695
|
this.log.warn(
|
|
549
|
-
`Not enough txs to build block ${blockNumber} at index ${indexWithinCheckpoint} in slot ${this.
|
|
550
|
-
{ blockNumber, slot: this.
|
|
696
|
+
`Not enough txs to build block ${blockNumber} at index ${indexWithinCheckpoint} in slot ${this.targetSlot} (got ${availableTxs} txs but needs ${minTxs})`,
|
|
697
|
+
{ blockNumber, slot: this.targetSlot, indexWithinCheckpoint },
|
|
551
698
|
);
|
|
552
|
-
this.eventEmitter.emit('block-tx-count-check-failed', { minTxs, availableTxs, slot: this.
|
|
699
|
+
this.eventEmitter.emit('block-tx-count-check-failed', { minTxs, availableTxs, slot: this.targetSlot });
|
|
553
700
|
this.metrics.recordBlockProposalFailed('insufficient_txs');
|
|
554
|
-
return
|
|
701
|
+
return { failure: 'insufficient-txs' };
|
|
555
702
|
}
|
|
556
703
|
|
|
557
704
|
// Create iterator to pending txs. We filter out txs already included in previous blocks in the checkpoint
|
|
@@ -562,14 +709,16 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
562
709
|
);
|
|
563
710
|
|
|
564
711
|
this.log.debug(
|
|
565
|
-
`Building block ${blockNumber} at index ${indexWithinCheckpoint} for slot ${this.
|
|
566
|
-
{ slot: this.
|
|
712
|
+
`Building block ${blockNumber} at index ${indexWithinCheckpoint} for slot ${this.targetSlot} with ${availableTxs} available txs`,
|
|
713
|
+
{ slot: this.targetSlot, blockNumber, indexWithinCheckpoint },
|
|
567
714
|
);
|
|
568
|
-
this.setStateFn(SequencerState.CREATING_BLOCK, this.
|
|
715
|
+
this.setStateFn(SequencerState.CREATING_BLOCK, this.targetSlot);
|
|
569
716
|
|
|
570
|
-
// Per-block limits
|
|
717
|
+
// Per-block limits are operator overrides (from SEQ_MAX_L2_BLOCK_GAS etc.) further capped
|
|
571
718
|
// by remaining checkpoint-level budgets inside CheckpointBuilder before each block is built.
|
|
572
|
-
|
|
719
|
+
// minValidTxs is passed into the builder so it can reject the block *before* updating state.
|
|
720
|
+
const minValidTxs = forceCreate ? 0 : (this.config.minValidTxsPerBlock ?? minTxs);
|
|
721
|
+
const blockBuilderOptions: BlockBuilderOptions = {
|
|
573
722
|
maxTransactions: this.config.maxTxsPerBlock,
|
|
574
723
|
maxBlockGas:
|
|
575
724
|
this.config.maxL2BlockGas !== undefined || this.config.maxDABlockGas !== undefined
|
|
@@ -577,9 +726,14 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
577
726
|
: undefined,
|
|
578
727
|
deadline: buildDeadline,
|
|
579
728
|
isBuildingProposal: true,
|
|
729
|
+
minValidTxs,
|
|
730
|
+
maxBlocksPerCheckpoint: this.timetable.maxNumberOfBlocks,
|
|
731
|
+
perBlockAllocationMultiplier: this.config.perBlockAllocationMultiplier,
|
|
580
732
|
};
|
|
581
733
|
|
|
582
|
-
// Actually build the block by executing txs
|
|
734
|
+
// Actually build the block by executing txs. The builder throws InsufficientValidTxsError
|
|
735
|
+
// if the number of successfully processed txs is below minValidTxs, ensuring state is not
|
|
736
|
+
// updated for blocks that will be discarded.
|
|
583
737
|
const buildResult = await this.buildSingleBlockWithCheckpointBuilder(
|
|
584
738
|
checkpointBuilder,
|
|
585
739
|
pendingTxs,
|
|
@@ -591,22 +745,27 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
591
745
|
// If any txs failed during execution, drop them from the mempool so we don't pick them up again
|
|
592
746
|
await this.dropFailedTxsFromP2P(buildResult.failedTxs);
|
|
593
747
|
|
|
594
|
-
|
|
595
|
-
// too long, then we may not get to minTxsPerBlock after executing public functions.
|
|
596
|
-
const minValidTxs = this.config.minValidTxsPerBlock ?? minTxs;
|
|
597
|
-
const numTxs = buildResult.status === 'no-valid-txs' ? 0 : buildResult.numTxs;
|
|
598
|
-
if (buildResult.status === 'no-valid-txs' || (!forceCreate && numTxs < minValidTxs)) {
|
|
748
|
+
if (buildResult.status === 'insufficient-valid-txs') {
|
|
599
749
|
this.log.warn(
|
|
600
|
-
`Block ${blockNumber} at index ${indexWithinCheckpoint} on slot ${this.
|
|
601
|
-
{
|
|
750
|
+
`Block ${blockNumber} at index ${indexWithinCheckpoint} on slot ${this.targetSlot} has too few valid txs to be proposed`,
|
|
751
|
+
{
|
|
752
|
+
slot: this.targetSlot,
|
|
753
|
+
blockNumber,
|
|
754
|
+
numTxs: buildResult.processedCount,
|
|
755
|
+
indexWithinCheckpoint,
|
|
756
|
+
minValidTxs,
|
|
757
|
+
},
|
|
602
758
|
);
|
|
603
|
-
this.eventEmitter.emit('block-build-failed', {
|
|
759
|
+
this.eventEmitter.emit('block-build-failed', {
|
|
760
|
+
reason: `Insufficient valid txs`,
|
|
761
|
+
slot: this.targetSlot,
|
|
762
|
+
});
|
|
604
763
|
this.metrics.recordBlockProposalFailed('insufficient_valid_txs');
|
|
605
|
-
return
|
|
764
|
+
return { failure: 'insufficient-valid-txs' };
|
|
606
765
|
}
|
|
607
766
|
|
|
608
767
|
// Block creation succeeded, emit stats and metrics
|
|
609
|
-
const { block, publicProcessorDuration, usedTxs, blockBuildDuration } = buildResult;
|
|
768
|
+
const { block, publicProcessorDuration, usedTxs, blockBuildDuration, numTxs } = buildResult;
|
|
610
769
|
|
|
611
770
|
const blockStats = {
|
|
612
771
|
eventName: 'l2-block-built',
|
|
@@ -620,30 +779,39 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
620
779
|
const manaPerSec = block.header.totalManaUsed.toNumberUnsafe() / (blockBuildDuration / 1000);
|
|
621
780
|
|
|
622
781
|
this.log.info(
|
|
623
|
-
`Built block ${block.number} at checkpoint ${this.checkpointNumber} for slot ${this.
|
|
782
|
+
`Built block ${block.number} at checkpoint ${this.checkpointNumber} for slot ${this.targetSlot} with ${numTxs} txs`,
|
|
624
783
|
{ blockHash, txHashes, manaPerSec, ...blockStats },
|
|
625
784
|
);
|
|
626
785
|
|
|
627
|
-
|
|
628
|
-
|
|
786
|
+
// `slot` is the target/submission slot (may be one ahead when pipelining),
|
|
787
|
+
// `buildSlot` is the wall-clock slot during which the block was actually built.
|
|
788
|
+
this.eventEmitter.emit('block-proposed', {
|
|
789
|
+
blockNumber: block.number,
|
|
790
|
+
slot: this.targetSlot,
|
|
791
|
+
buildSlot: this.slotNow,
|
|
792
|
+
});
|
|
793
|
+
this.metrics.recordBuiltBlock(blockBuildDuration, block.header.totalManaUsed.toNumberUnsafe(), this.targetSlot);
|
|
629
794
|
|
|
630
795
|
return { block, usedTxs };
|
|
631
796
|
} catch (err: any) {
|
|
632
|
-
this.eventEmitter.emit('block-build-failed', {
|
|
633
|
-
|
|
797
|
+
this.eventEmitter.emit('block-build-failed', {
|
|
798
|
+
reason: err.message,
|
|
799
|
+
slot: this.targetSlot,
|
|
800
|
+
});
|
|
801
|
+
this.log.error(`Error building block`, err, { blockNumber, slot: this.targetSlot });
|
|
634
802
|
this.metrics.recordBlockProposalFailed(err.name || 'unknown_error');
|
|
635
803
|
this.metrics.recordFailedBlock();
|
|
636
804
|
return { error: err };
|
|
637
805
|
}
|
|
638
806
|
}
|
|
639
807
|
|
|
640
|
-
/** Uses the checkpoint builder to build a block, catching
|
|
808
|
+
/** Uses the checkpoint builder to build a block, catching InsufficientValidTxsError. */
|
|
641
809
|
private async buildSingleBlockWithCheckpointBuilder(
|
|
642
810
|
checkpointBuilder: CheckpointBuilder,
|
|
643
811
|
pendingTxs: AsyncIterable<Tx>,
|
|
644
812
|
blockNumber: BlockNumber,
|
|
645
813
|
blockTimestamp: bigint,
|
|
646
|
-
blockBuilderOptions:
|
|
814
|
+
blockBuilderOptions: BlockBuilderOptions,
|
|
647
815
|
) {
|
|
648
816
|
try {
|
|
649
817
|
const workTimer = new Timer();
|
|
@@ -651,8 +819,12 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
651
819
|
const blockBuildDuration = workTimer.ms();
|
|
652
820
|
return { ...result, blockBuildDuration, status: 'success' as const };
|
|
653
821
|
} catch (err: unknown) {
|
|
654
|
-
if (isErrorClass(err,
|
|
655
|
-
return {
|
|
822
|
+
if (isErrorClass(err, InsufficientValidTxsError)) {
|
|
823
|
+
return {
|
|
824
|
+
failedTxs: err.failedTxs,
|
|
825
|
+
processedCount: err.processedCount,
|
|
826
|
+
status: 'insufficient-valid-txs' as const,
|
|
827
|
+
};
|
|
656
828
|
}
|
|
657
829
|
throw err;
|
|
658
830
|
}
|
|
@@ -665,7 +837,7 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
665
837
|
blockNumber: BlockNumber;
|
|
666
838
|
indexWithinCheckpoint: IndexWithinCheckpoint;
|
|
667
839
|
buildDeadline: Date | undefined;
|
|
668
|
-
}): Promise<{ canStartBuilding: boolean; availableTxs: number }> {
|
|
840
|
+
}): Promise<{ canStartBuilding: boolean; availableTxs: number; minTxs: number }> {
|
|
669
841
|
const { indexWithinCheckpoint, blockNumber, buildDeadline, forceCreate } = opts;
|
|
670
842
|
|
|
671
843
|
// We only allow a block with 0 txs in the first block of the checkpoint
|
|
@@ -682,20 +854,20 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
682
854
|
// If we're past deadline, or we have no deadline, give up
|
|
683
855
|
const now = this.dateProvider.nowAsDate();
|
|
684
856
|
if (startBuildingDeadline === undefined || now >= startBuildingDeadline) {
|
|
685
|
-
return { canStartBuilding: false, availableTxs
|
|
857
|
+
return { canStartBuilding: false, availableTxs, minTxs };
|
|
686
858
|
}
|
|
687
859
|
|
|
688
860
|
// Wait a bit before checking again
|
|
689
|
-
this.setStateFn(SequencerState.WAITING_FOR_TXS, this.
|
|
861
|
+
this.setStateFn(SequencerState.WAITING_FOR_TXS, this.targetSlot);
|
|
690
862
|
this.log.verbose(
|
|
691
|
-
`Waiting for enough txs to build block ${blockNumber} at index ${indexWithinCheckpoint} in slot ${this.
|
|
692
|
-
{ blockNumber, slot: this.
|
|
863
|
+
`Waiting for enough txs to build block ${blockNumber} at index ${indexWithinCheckpoint} in slot ${this.targetSlot} (have ${availableTxs} but need ${minTxs})`,
|
|
864
|
+
{ blockNumber, slot: this.targetSlot, indexWithinCheckpoint },
|
|
693
865
|
);
|
|
694
866
|
await this.waitForTxsPollingInterval();
|
|
695
867
|
availableTxs = await this.p2pClient.getPendingTxCount();
|
|
696
868
|
}
|
|
697
869
|
|
|
698
|
-
return { canStartBuilding: true, availableTxs };
|
|
870
|
+
return { canStartBuilding: true, availableTxs, minTxs };
|
|
699
871
|
}
|
|
700
872
|
|
|
701
873
|
/**
|
|
@@ -721,16 +893,16 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
721
893
|
this.log.debug(`Attesting committee length is ${committee.length}`, { committee });
|
|
722
894
|
}
|
|
723
895
|
|
|
724
|
-
const numberOfRequiredAttestations =
|
|
896
|
+
const numberOfRequiredAttestations = computeQuorum(committee.length);
|
|
725
897
|
|
|
726
898
|
if (this.config.skipCollectingAttestations) {
|
|
727
899
|
this.log.warn('Skipping attestation collection as per config (attesting with own keys only)');
|
|
728
|
-
const attestations = await this.validatorClient?.collectOwnAttestations(proposal);
|
|
900
|
+
const attestations = await this.validatorClient?.collectOwnAttestations(proposal, this.checkpointNumber);
|
|
729
901
|
return new CommitteeAttestationsAndSigners(orderAttestations(attestations ?? [], committee));
|
|
730
902
|
}
|
|
731
903
|
|
|
732
904
|
const attestationTimeAllowed = this.config.enforceTimeTable
|
|
733
|
-
? this.timetable.
|
|
905
|
+
? this.timetable.getCheckpointAttestationDeadline()
|
|
734
906
|
: this.l1Constants.slotDuration;
|
|
735
907
|
const attestationDeadline = new Date((this.getSlotStartBuildTimestamp() + attestationTimeAllowed) * 1000);
|
|
736
908
|
|
|
@@ -743,6 +915,7 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
743
915
|
proposal,
|
|
744
916
|
numberOfRequiredAttestations,
|
|
745
917
|
attestationDeadline,
|
|
918
|
+
this.checkpointNumber,
|
|
746
919
|
);
|
|
747
920
|
|
|
748
921
|
collectedAttestationsCount = attestations.length;
|
|
@@ -869,9 +1042,13 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
869
1042
|
* Adds the proposed block to the archiver so it's available via P2P.
|
|
870
1043
|
* Gossip doesn't echo messages back to the sender, so the proposer's archiver/world-state
|
|
871
1044
|
* would never receive its own block without this explicit sync.
|
|
1045
|
+
*
|
|
1046
|
+
* In fisherman mode we skip this push: the fisherman builds blocks locally for validation
|
|
1047
|
+
* and fee analysis only, and pushing them to the archiver causes spurious reorg cascades
|
|
1048
|
+
* whenever the real proposer's block arrives from L1.
|
|
872
1049
|
*/
|
|
873
1050
|
private async syncProposedBlockToArchiver(block: L2Block): Promise<void> {
|
|
874
|
-
if (this.config.skipPushProposedBlocksToArchiver
|
|
1051
|
+
if (this.config.skipPushProposedBlocksToArchiver || this.config.fishermanMode) {
|
|
875
1052
|
this.log.warn(`Skipping push of proposed block ${block.number} to archiver`, {
|
|
876
1053
|
blockNumber: block.number,
|
|
877
1054
|
slot: block.header.globalVariables.slotNumber,
|
|
@@ -889,19 +1066,19 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
889
1066
|
private async handleCheckpointEndAsFisherman(checkpoint: Checkpoint | undefined) {
|
|
890
1067
|
// Perform L1 fee analysis before clearing requests
|
|
891
1068
|
// The callback is invoked asynchronously after the next block is mined
|
|
892
|
-
const feeAnalysis = await this.publisher.analyzeL1Fees(this.
|
|
1069
|
+
const feeAnalysis = await this.publisher.analyzeL1Fees(this.targetSlot, analysis =>
|
|
893
1070
|
this.metrics.recordFishermanFeeAnalysis(analysis),
|
|
894
1071
|
);
|
|
895
1072
|
|
|
896
1073
|
if (checkpoint) {
|
|
897
|
-
this.log.info(`Validation checkpoint building SUCCEEDED for slot ${this.
|
|
1074
|
+
this.log.info(`Validation checkpoint building SUCCEEDED for slot ${this.targetSlot}`, {
|
|
898
1075
|
...checkpoint.toCheckpointInfo(),
|
|
899
1076
|
...checkpoint.getStats(),
|
|
900
1077
|
feeAnalysisId: feeAnalysis?.id,
|
|
901
1078
|
});
|
|
902
1079
|
} else {
|
|
903
|
-
this.log.warn(`Validation block building FAILED for slot ${this.
|
|
904
|
-
slot: this.
|
|
1080
|
+
this.log.warn(`Validation block building FAILED for slot ${this.targetSlot}`, {
|
|
1081
|
+
slot: this.targetSlot,
|
|
905
1082
|
feeAnalysisId: feeAnalysis?.id,
|
|
906
1083
|
});
|
|
907
1084
|
this.metrics.recordCheckpointProposalFailed('block_build_failed');
|
|
@@ -915,15 +1092,15 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
915
1092
|
*/
|
|
916
1093
|
private handleHASigningError(err: any, errorContext: string): boolean {
|
|
917
1094
|
if (err instanceof DutyAlreadySignedError) {
|
|
918
|
-
this.log.info(`${errorContext} for slot ${this.
|
|
919
|
-
slot: this.
|
|
1095
|
+
this.log.info(`${errorContext} for slot ${this.targetSlot} already signed by another HA node, yielding`, {
|
|
1096
|
+
slot: this.targetSlot,
|
|
920
1097
|
signedByNode: err.signedByNode,
|
|
921
1098
|
});
|
|
922
1099
|
return true;
|
|
923
1100
|
}
|
|
924
1101
|
if (err instanceof SlashingProtectionError) {
|
|
925
|
-
this.log.info(`${errorContext} for slot ${this.
|
|
926
|
-
slot: this.
|
|
1102
|
+
this.log.info(`${errorContext} for slot ${this.targetSlot} blocked by slashing protection, yielding`, {
|
|
1103
|
+
slot: this.targetSlot,
|
|
927
1104
|
existingMessageHash: err.existingMessageHash,
|
|
928
1105
|
attemptedMessageHash: err.attemptedMessageHash,
|
|
929
1106
|
});
|
|
@@ -946,7 +1123,7 @@ export class CheckpointProposalJob implements Traceable {
|
|
|
946
1123
|
}
|
|
947
1124
|
|
|
948
1125
|
private getSlotStartBuildTimestamp(): number {
|
|
949
|
-
return getSlotStartBuildTimestamp(this.
|
|
1126
|
+
return getSlotStartBuildTimestamp(this.slotNow, this.l1Constants);
|
|
950
1127
|
}
|
|
951
1128
|
|
|
952
1129
|
private getSecondsIntoSlot(): number {
|