@aztec/sequencer-client 0.0.1-commit.5de5ca79e → 0.0.1-commit.6201a7b05
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dest/client/sequencer-client.d.ts +6 -1
- package/dest/client/sequencer-client.d.ts.map +1 -1
- package/dest/client/sequencer-client.js +11 -17
- package/dest/config.d.ts +1 -1
- package/dest/config.d.ts.map +1 -1
- package/dest/config.js +13 -12
- package/dest/global_variable_builder/fee_predictor.d.ts +37 -0
- package/dest/global_variable_builder/fee_predictor.d.ts.map +1 -0
- package/dest/global_variable_builder/fee_predictor.js +128 -0
- package/dest/global_variable_builder/fee_provider.d.ts +21 -0
- package/dest/global_variable_builder/fee_provider.d.ts.map +1 -0
- package/dest/global_variable_builder/fee_provider.js +58 -0
- package/dest/global_variable_builder/global_builder.d.ts +15 -14
- package/dest/global_variable_builder/global_builder.d.ts.map +1 -1
- package/dest/global_variable_builder/global_builder.js +16 -51
- package/dest/global_variable_builder/index.d.ts +4 -2
- package/dest/global_variable_builder/index.d.ts.map +1 -1
- package/dest/global_variable_builder/index.js +2 -0
- package/dest/publisher/config.d.ts +13 -1
- package/dest/publisher/config.d.ts.map +1 -1
- package/dest/publisher/config.js +19 -4
- package/dest/publisher/l1_tx_failed_store/failed_tx_store.d.ts +3 -4
- package/dest/publisher/l1_tx_failed_store/failed_tx_store.d.ts.map +1 -1
- package/dest/publisher/sequencer-publisher-factory.d.ts +3 -5
- package/dest/publisher/sequencer-publisher-factory.d.ts.map +1 -1
- package/dest/publisher/sequencer-publisher-factory.js +2 -3
- package/dest/publisher/sequencer-publisher.d.ts +48 -42
- package/dest/publisher/sequencer-publisher.d.ts.map +1 -1
- package/dest/publisher/sequencer-publisher.js +119 -111
- package/dest/sequencer/chain_state_overrides.d.ts +25 -0
- package/dest/sequencer/chain_state_overrides.d.ts.map +1 -0
- package/dest/sequencer/chain_state_overrides.js +39 -0
- package/dest/sequencer/checkpoint_proposal_job.d.ts +41 -11
- package/dest/sequencer/checkpoint_proposal_job.d.ts.map +1 -1
- package/dest/sequencer/checkpoint_proposal_job.js +525 -149
- package/dest/sequencer/checkpoint_proposal_job_metrics.d.ts +34 -0
- package/dest/sequencer/checkpoint_proposal_job_metrics.d.ts.map +1 -0
- package/dest/sequencer/checkpoint_proposal_job_metrics.js +72 -0
- package/dest/sequencer/checkpoint_voter.d.ts +1 -2
- package/dest/sequencer/checkpoint_voter.d.ts.map +1 -1
- package/dest/sequencer/checkpoint_voter.js +2 -5
- package/dest/sequencer/events.d.ts +6 -1
- package/dest/sequencer/events.d.ts.map +1 -1
- package/dest/sequencer/metrics.d.ts +9 -10
- package/dest/sequencer/metrics.d.ts.map +1 -1
- package/dest/sequencer/metrics.js +34 -20
- package/dest/sequencer/sequencer.d.ts +27 -7
- package/dest/sequencer/sequencer.d.ts.map +1 -1
- package/dest/sequencer/sequencer.js +106 -28
- package/dest/sequencer/timetable.d.ts +14 -1
- package/dest/sequencer/timetable.d.ts.map +1 -1
- package/dest/sequencer/timetable.js +45 -36
- package/dest/test/utils.d.ts +1 -1
- package/dest/test/utils.d.ts.map +1 -1
- package/dest/test/utils.js +7 -6
- package/package.json +27 -27
- package/src/client/sequencer-client.ts +16 -24
- package/src/config.ts +13 -11
- package/src/global_variable_builder/README.md +44 -0
- package/src/global_variable_builder/fee_predictor.ts +172 -0
- package/src/global_variable_builder/fee_provider.ts +75 -0
- package/src/global_variable_builder/global_builder.ts +26 -63
- package/src/global_variable_builder/index.ts +3 -1
- package/src/publisher/config.ts +38 -4
- package/src/publisher/l1_tx_failed_store/failed_tx_store.ts +3 -1
- package/src/publisher/sequencer-publisher-factory.ts +3 -6
- package/src/publisher/sequencer-publisher.ts +183 -159
- package/src/sequencer/README.md +82 -13
- package/src/sequencer/chain_state_overrides.ts +87 -0
- package/src/sequencer/checkpoint_proposal_job.ts +616 -167
- package/src/sequencer/checkpoint_proposal_job_metrics.ts +128 -0
- package/src/sequencer/checkpoint_voter.ts +1 -12
- package/src/sequencer/events.ts +5 -0
- package/src/sequencer/metrics.ts +43 -24
- package/src/sequencer/sequencer.ts +148 -32
- package/src/sequencer/timetable.ts +57 -45
- package/src/test/utils.ts +28 -10
|
@@ -1,19 +1,20 @@
|
|
|
1
1
|
import { getKzg } from '@aztec/blob-lib';
|
|
2
2
|
import { INITIAL_L2_BLOCK_NUM } from '@aztec/constants';
|
|
3
3
|
import type { EpochCache } from '@aztec/epoch-cache';
|
|
4
|
-
import { NoCommitteeError, type RollupContract } from '@aztec/ethereum/contracts';
|
|
4
|
+
import { NoCommitteeError, type RollupContract, SimulationOverridesBuilder } from '@aztec/ethereum/contracts';
|
|
5
5
|
import { BlockNumber, CheckpointNumber, EpochNumber, SlotNumber } from '@aztec/foundation/branded-types';
|
|
6
6
|
import { merge, omit, pick } from '@aztec/foundation/collection';
|
|
7
7
|
import { Fr } from '@aztec/foundation/curves/bn254';
|
|
8
8
|
import { EthAddress } from '@aztec/foundation/eth-address';
|
|
9
|
-
import { createLogger } from '@aztec/foundation/log';
|
|
9
|
+
import { type Logger, createLogger } from '@aztec/foundation/log';
|
|
10
10
|
import { RunningPromise } from '@aztec/foundation/running-promise';
|
|
11
11
|
import type { DateProvider } from '@aztec/foundation/timer';
|
|
12
12
|
import type { TypedEventEmitter } from '@aztec/foundation/types';
|
|
13
13
|
import type { P2P } from '@aztec/p2p';
|
|
14
14
|
import type { SlasherClientInterface } from '@aztec/slasher';
|
|
15
15
|
import type { BlockData, L2BlockSink, L2BlockSource, ValidateCheckpointResult } from '@aztec/stdlib/block';
|
|
16
|
-
import type { Checkpoint } from '@aztec/stdlib/checkpoint';
|
|
16
|
+
import type { Checkpoint, ProposedCheckpointData } from '@aztec/stdlib/checkpoint';
|
|
17
|
+
import type { ChainConfig } from '@aztec/stdlib/config';
|
|
17
18
|
import { getSlotStartBuildTimestamp } from '@aztec/stdlib/epoch-helpers';
|
|
18
19
|
import {
|
|
19
20
|
type ResolvedSequencerConfig,
|
|
@@ -22,6 +23,7 @@ import {
|
|
|
22
23
|
type WorldStateSynchronizer,
|
|
23
24
|
} from '@aztec/stdlib/interfaces/server';
|
|
24
25
|
import type { L1ToL2MessageSource } from '@aztec/stdlib/messaging';
|
|
26
|
+
import type { CoordinationSignatureContext } from '@aztec/stdlib/p2p';
|
|
25
27
|
import { pickFromSchema } from '@aztec/stdlib/schemas';
|
|
26
28
|
import { MerkleTreeId } from '@aztec/stdlib/trees';
|
|
27
29
|
import { Attributes, type TelemetryClient, type Tracer, getTelemetryClient, trackSpan } from '@aztec/telemetry-client';
|
|
@@ -34,6 +36,7 @@ import type { GlobalVariableBuilder } from '../global_variable_builder/global_bu
|
|
|
34
36
|
import type { SequencerPublisherFactory } from '../publisher/sequencer-publisher-factory.js';
|
|
35
37
|
import type { InvalidateCheckpointRequest, SequencerPublisher } from '../publisher/sequencer-publisher.js';
|
|
36
38
|
import { CheckpointProposalJob } from './checkpoint_proposal_job.js';
|
|
39
|
+
import { CheckpointProposalJobMetrics } from './checkpoint_proposal_job_metrics.js';
|
|
37
40
|
import { CheckpointVoter } from './checkpoint_voter.js';
|
|
38
41
|
import { SequencerInterruptedError, SequencerTooSlowError } from './errors.js';
|
|
39
42
|
import type { SequencerEvents } from './events.js';
|
|
@@ -55,7 +58,11 @@ export { SequencerState };
|
|
|
55
58
|
export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<SequencerEvents>) {
|
|
56
59
|
private runningPromise?: RunningPromise;
|
|
57
60
|
private state = SequencerState.STOPPED;
|
|
61
|
+
private stateSlotNumber: SlotNumber | undefined;
|
|
62
|
+
private stateEnteredAtMs = performance.now();
|
|
58
63
|
private metrics: SequencerMetrics;
|
|
64
|
+
private checkpointProposalJobMetrics: CheckpointProposalJobMetrics;
|
|
65
|
+
private readonly stateLog: Logger;
|
|
59
66
|
|
|
60
67
|
/** The last slot for which we attempted to perform our voting duties with degraded block production */
|
|
61
68
|
private lastSlotForFallbackVote: SlotNumber | undefined;
|
|
@@ -72,11 +79,15 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
72
79
|
/** The last epoch for which we logged strategy comparison in fisherman mode. */
|
|
73
80
|
private lastEpochForStrategyComparison: EpochNumber | undefined;
|
|
74
81
|
|
|
82
|
+
/** The last checkpoint proposal job, tracked so we can await its pending L1 submission during shutdown. */
|
|
83
|
+
protected lastCheckpointProposalJob: CheckpointProposalJob | undefined;
|
|
84
|
+
|
|
75
85
|
/** The maximum number of seconds that the sequencer can be into a slot to transition to a particular state. */
|
|
76
86
|
protected timetable!: SequencerTimetable;
|
|
77
87
|
|
|
78
88
|
/** Config for the sequencer */
|
|
79
89
|
protected config: ResolvedSequencerConfig = DefaultSequencerConfig;
|
|
90
|
+
private readonly signatureContext: CoordinationSignatureContext;
|
|
80
91
|
|
|
81
92
|
constructor(
|
|
82
93
|
protected publisherFactory: SequencerPublisherFactory,
|
|
@@ -92,18 +103,24 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
92
103
|
protected dateProvider: DateProvider,
|
|
93
104
|
protected epochCache: EpochCache,
|
|
94
105
|
protected rollupContract: RollupContract,
|
|
95
|
-
config: SequencerConfig,
|
|
106
|
+
config: SequencerConfig & Pick<ChainConfig, 'l1ChainId' | 'l1Contracts'>,
|
|
96
107
|
protected telemetry: TelemetryClient = getTelemetryClient(),
|
|
97
108
|
protected log = createLogger('sequencer'),
|
|
98
109
|
) {
|
|
99
110
|
super();
|
|
111
|
+
this.stateLog = log.createChild('state');
|
|
100
112
|
|
|
101
113
|
// Add [FISHERMAN] prefix to logger if in fisherman mode
|
|
102
114
|
if (config.fishermanMode) {
|
|
103
115
|
this.log = log.createChild('[FISHERMAN]');
|
|
104
116
|
}
|
|
105
117
|
|
|
118
|
+
this.signatureContext = {
|
|
119
|
+
chainId: config.l1ChainId,
|
|
120
|
+
rollupAddress: config.l1Contracts.rollupAddress,
|
|
121
|
+
};
|
|
106
122
|
this.metrics = new SequencerMetrics(telemetry, this.rollupContract, 'Sequencer');
|
|
123
|
+
this.checkpointProposalJobMetrics = new CheckpointProposalJobMetrics(telemetry);
|
|
107
124
|
this.updateConfig(config);
|
|
108
125
|
}
|
|
109
126
|
|
|
@@ -112,14 +129,16 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
112
129
|
const filteredConfig = pickFromSchema(config, SequencerConfigSchema);
|
|
113
130
|
this.log.info(`Updated sequencer config`, omit(filteredConfig, 'txPublicSetupAllowListExtend'));
|
|
114
131
|
this.config = merge(this.config, filteredConfig);
|
|
132
|
+
const p2pPropagationTime = this.config.attestationPropagationTime;
|
|
115
133
|
this.timetable = new SequencerTimetable(
|
|
116
134
|
{
|
|
117
135
|
ethereumSlotDuration: this.l1Constants.ethereumSlotDuration,
|
|
118
136
|
aztecSlotDuration: this.aztecSlotDuration,
|
|
119
137
|
l1PublishingTime: this.l1PublishingTime,
|
|
120
|
-
p2pPropagationTime
|
|
138
|
+
p2pPropagationTime,
|
|
121
139
|
blockDurationMs: this.config.blockDurationMs,
|
|
122
140
|
enforce: this.config.enforceTimeTable,
|
|
141
|
+
pipelining: this.epochCache.isProposerPipeliningEnabled(),
|
|
123
142
|
},
|
|
124
143
|
this.metrics,
|
|
125
144
|
this.log,
|
|
@@ -143,12 +162,18 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
143
162
|
this.log.info('Started sequencer');
|
|
144
163
|
}
|
|
145
164
|
|
|
165
|
+
/** Triggers an immediate run of the sequencer, bypassing the polling interval. */
|
|
166
|
+
public trigger() {
|
|
167
|
+
return this.runningPromise?.trigger();
|
|
168
|
+
}
|
|
169
|
+
|
|
146
170
|
/** Stops the sequencer from building blocks and moves to STOPPED state. */
|
|
147
171
|
public async stop(): Promise<void> {
|
|
148
172
|
this.log.info(`Stopping sequencer`);
|
|
149
173
|
this.setState(SequencerState.STOPPING, undefined, { force: true });
|
|
150
|
-
this.publisherFactory.
|
|
174
|
+
await this.publisherFactory.stopAll();
|
|
151
175
|
await this.runningPromise?.stop();
|
|
176
|
+
await this.lastCheckpointProposalJob?.awaitPendingSubmission();
|
|
152
177
|
this.setState(SequencerState.STOPPED, undefined, { force: true });
|
|
153
178
|
this.log.info('Stopped sequencer');
|
|
154
179
|
}
|
|
@@ -208,6 +233,9 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
208
233
|
return;
|
|
209
234
|
}
|
|
210
235
|
|
|
236
|
+
// Track the job so we can await its pending L1 submission during shutdown
|
|
237
|
+
this.lastCheckpointProposalJob = checkpointProposalJob;
|
|
238
|
+
|
|
211
239
|
// Execute the checkpoint proposal job
|
|
212
240
|
const checkpoint = await checkpointProposalJob.execute();
|
|
213
241
|
|
|
@@ -234,7 +262,7 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
234
262
|
* @returns CheckpointProposalJob if successful, undefined if we are not yet synced or are not the proposer.
|
|
235
263
|
*/
|
|
236
264
|
@trackSpan('Sequencer.prepareCheckpointProposal')
|
|
237
|
-
|
|
265
|
+
protected async prepareCheckpointProposal(
|
|
238
266
|
slot: SlotNumber,
|
|
239
267
|
targetSlot: SlotNumber,
|
|
240
268
|
epoch: EpochNumber,
|
|
@@ -312,6 +340,16 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
312
340
|
return undefined;
|
|
313
341
|
}
|
|
314
342
|
|
|
343
|
+
// Guard: don't exceed 1-deep pipeline. Without a proposed checkpoint, we can only build
|
|
344
|
+
// confirmed + 1. With a proposed checkpoint, we can build confirmed + 2.
|
|
345
|
+
const confirmedCkpt = syncedTo.checkpointedCheckpointNumber;
|
|
346
|
+
if (checkpointNumber > confirmedCkpt + 2) {
|
|
347
|
+
this.log.verbose(
|
|
348
|
+
`Skipping slot ${targetSlot}: checkpoint ${checkpointNumber} exceeds max pipeline depth (confirmed=${confirmedCkpt})`,
|
|
349
|
+
);
|
|
350
|
+
return undefined;
|
|
351
|
+
}
|
|
352
|
+
|
|
315
353
|
// Check that the target slot is not taken by a block already (should never happen, since only us can propose for this slot)
|
|
316
354
|
if (syncedTo.blockData && syncedTo.blockData.header.getSlot() >= targetSlot) {
|
|
317
355
|
this.log.warn(
|
|
@@ -337,13 +375,42 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
337
375
|
}
|
|
338
376
|
|
|
339
377
|
// Prepare invalidation request if the pending chain is invalid (returns undefined if no need)
|
|
340
|
-
|
|
378
|
+
let invalidateCheckpoint = await publisher.simulateInvalidateCheckpoint(syncedTo.pendingChainValidationStatus);
|
|
379
|
+
|
|
380
|
+
// Determine the correct archive and L1 state overrides for the canProposeAt check.
|
|
381
|
+
// The L1 contract reads archives[proposedCheckpointNumber] and compares it with the provided archive.
|
|
382
|
+
// When invalidating or pipelining, the local archive may differ from L1's, so we adjust accordingly.
|
|
383
|
+
let archiveForCheck = syncedTo.archive;
|
|
384
|
+
const l1SimulationOverridesBuilder = new SimulationOverridesBuilder();
|
|
385
|
+
|
|
386
|
+
if (this.epochCache.isProposerPipeliningEnabled() && syncedTo.hasProposedCheckpoint) {
|
|
387
|
+
// Parent checkpoint hasn't landed on L1 yet. Override both the proposed checkpoint number
|
|
388
|
+
// and the archive at that checkpoint so L1 simulation sees the correct chain tip.
|
|
389
|
+
const parentCheckpointNumber = CheckpointNumber(checkpointNumber - 1);
|
|
390
|
+
l1SimulationOverridesBuilder.forPendingCheckpoint(parentCheckpointNumber).withPendingArchive(syncedTo.archive);
|
|
391
|
+
this.metrics.recordPipelineDepth(syncedTo.checkpointNumber - syncedTo.checkpointedCheckpointNumber);
|
|
392
|
+
|
|
393
|
+
this.log.verbose(
|
|
394
|
+
`Building on top of proposed checkpoint (pending=${syncedTo.proposedCheckpointData?.checkpointNumber})`,
|
|
395
|
+
);
|
|
396
|
+
// Clear the invalidation - the proposed checkpoint should handle it.
|
|
397
|
+
invalidateCheckpoint = undefined;
|
|
398
|
+
} else if (invalidateCheckpoint) {
|
|
399
|
+
// After invalidation, L1 will roll back to checkpoint N-1. The archive at N-1 already
|
|
400
|
+
// exists on L1, so we just pass the matching archive (the lastArchive of the invalid checkpoint).
|
|
401
|
+
archiveForCheck = invalidateCheckpoint.lastArchive;
|
|
402
|
+
l1SimulationOverridesBuilder.forPendingCheckpoint(invalidateCheckpoint.forcePendingCheckpointNumber);
|
|
403
|
+
this.metrics.recordPipelineDepth(0);
|
|
404
|
+
} else {
|
|
405
|
+
this.metrics.recordPipelineDepth(0);
|
|
406
|
+
}
|
|
341
407
|
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
408
|
+
const simulationOverridesPlan = l1SimulationOverridesBuilder.build();
|
|
409
|
+
const canProposeCheck = await publisher.canProposeAt(
|
|
410
|
+
archiveForCheck,
|
|
411
|
+
proposer ?? EthAddress.ZERO,
|
|
412
|
+
simulationOverridesPlan,
|
|
413
|
+
);
|
|
347
414
|
|
|
348
415
|
if (canProposeCheck === undefined) {
|
|
349
416
|
this.log.warn(
|
|
@@ -391,7 +458,6 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
391
458
|
return this.createCheckpointProposalJob(
|
|
392
459
|
slot,
|
|
393
460
|
targetSlot,
|
|
394
|
-
epoch,
|
|
395
461
|
targetEpoch,
|
|
396
462
|
checkpointNumber,
|
|
397
463
|
syncedTo.blockNumber,
|
|
@@ -399,13 +465,13 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
399
465
|
publisher,
|
|
400
466
|
attestorAddress,
|
|
401
467
|
invalidateCheckpoint,
|
|
468
|
+
syncedTo.proposedCheckpointData,
|
|
402
469
|
);
|
|
403
470
|
}
|
|
404
471
|
|
|
405
472
|
protected createCheckpointProposalJob(
|
|
406
473
|
slot: SlotNumber,
|
|
407
474
|
targetSlot: SlotNumber,
|
|
408
|
-
epoch: EpochNumber,
|
|
409
475
|
targetEpoch: EpochNumber,
|
|
410
476
|
checkpointNumber: CheckpointNumber,
|
|
411
477
|
syncedToBlockNumber: BlockNumber,
|
|
@@ -413,11 +479,11 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
413
479
|
publisher: SequencerPublisher,
|
|
414
480
|
attestorAddress: EthAddress,
|
|
415
481
|
invalidateCheckpoint: InvalidateCheckpointRequest | undefined,
|
|
482
|
+
proposedCheckpointData?: ProposedCheckpointData,
|
|
416
483
|
): CheckpointProposalJob {
|
|
417
484
|
return new CheckpointProposalJob(
|
|
418
485
|
slot,
|
|
419
486
|
targetSlot,
|
|
420
|
-
epoch,
|
|
421
487
|
targetEpoch,
|
|
422
488
|
checkpointNumber,
|
|
423
489
|
syncedToBlockNumber,
|
|
@@ -434,16 +500,19 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
434
500
|
this.checkpointsBuilder,
|
|
435
501
|
this.l2BlockSource,
|
|
436
502
|
this.l1Constants,
|
|
503
|
+
this.signatureContext,
|
|
437
504
|
this.config,
|
|
438
505
|
this.timetable,
|
|
439
506
|
this.slasherClient,
|
|
440
507
|
this.epochCache,
|
|
441
508
|
this.dateProvider,
|
|
442
509
|
this.metrics,
|
|
510
|
+
this.checkpointProposalJobMetrics.createRecorder(),
|
|
443
511
|
this,
|
|
444
512
|
this.setState.bind(this),
|
|
445
513
|
this.tracer,
|
|
446
514
|
this.log.getBindings(),
|
|
515
|
+
proposedCheckpointData,
|
|
447
516
|
);
|
|
448
517
|
}
|
|
449
518
|
|
|
@@ -479,19 +548,35 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
479
548
|
this.timetable.assertTimeLeft(proposedState, secondsIntoSlot);
|
|
480
549
|
}
|
|
481
550
|
|
|
551
|
+
const oldState = this.state;
|
|
552
|
+
const oldStateSlotNumber = this.stateSlotNumber;
|
|
553
|
+
const stateChanged = proposedState !== oldState;
|
|
554
|
+
const transitionAtMs = performance.now();
|
|
555
|
+
const stateDurationMs = transitionAtMs - this.stateEnteredAtMs;
|
|
556
|
+
|
|
482
557
|
const boringStates = [SequencerState.IDLE, SequencerState.SYNCHRONIZING];
|
|
483
558
|
const logLevel =
|
|
484
|
-
boringStates.includes(proposedState) && boringStates.includes(
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
559
|
+
boringStates.includes(proposedState) && boringStates.includes(oldState) ? ('trace' as const) : ('debug' as const);
|
|
560
|
+
this.stateLog[logLevel](`Transitioning from ${oldState} to ${proposedState}`, {
|
|
561
|
+
oldState,
|
|
562
|
+
newState: proposedState,
|
|
563
|
+
slotNumber,
|
|
564
|
+
stateSlotNumber: oldStateSlotNumber,
|
|
565
|
+
secondsIntoSlot,
|
|
566
|
+
...(stateChanged && { stateDurationMs: Math.ceil(stateDurationMs) }),
|
|
567
|
+
});
|
|
488
568
|
|
|
489
569
|
this.emit('state-changed', {
|
|
490
|
-
oldState
|
|
570
|
+
oldState,
|
|
491
571
|
newState: proposedState,
|
|
492
572
|
secondsIntoSlot,
|
|
493
573
|
slot: slotNumber,
|
|
494
574
|
});
|
|
575
|
+
if (stateChanged) {
|
|
576
|
+
this.metrics.recordStateDuration(stateDurationMs, oldState);
|
|
577
|
+
this.stateEnteredAtMs = transitionAtMs;
|
|
578
|
+
this.stateSlotNumber = slotNumber;
|
|
579
|
+
}
|
|
495
580
|
this.state = proposedState;
|
|
496
581
|
}
|
|
497
582
|
|
|
@@ -501,8 +586,8 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
501
586
|
*/
|
|
502
587
|
protected async checkSync(args: { ts: bigint; slot: SlotNumber }): Promise<SequencerSyncCheckResult | undefined> {
|
|
503
588
|
// Check that the archiver has fully synced the L2 slot before the one we want to propose in.
|
|
504
|
-
//
|
|
505
|
-
//
|
|
589
|
+
// The archiver reports sync progress via L1 block timestamps and synced checkpoint slots.
|
|
590
|
+
// See getSyncedL2SlotNumber for how missed L1 blocks are handled.
|
|
506
591
|
const syncedL2Slot = await this.l2BlockSource.getSyncedL2SlotNumber();
|
|
507
592
|
const { slot } = args;
|
|
508
593
|
if (syncedL2Slot === undefined || syncedL2Slot + 1 < slot) {
|
|
@@ -518,25 +603,43 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
518
603
|
number: syncSummary.latestBlockNumber,
|
|
519
604
|
hash: syncSummary.latestBlockHash,
|
|
520
605
|
})),
|
|
521
|
-
this.l2BlockSource
|
|
606
|
+
this.l2BlockSource
|
|
607
|
+
.getL2Tips()
|
|
608
|
+
.then(t => ({ proposed: t.proposed, checkpointed: t.checkpointed, proposedCheckpoint: t.proposedCheckpoint })),
|
|
522
609
|
this.p2pClient.getStatus().then(p2p => p2p.syncedToL2Block),
|
|
523
|
-
this.l1ToL2MessageSource.getL2Tips().then(t => t.proposed),
|
|
610
|
+
this.l1ToL2MessageSource.getL2Tips().then(t => ({ proposed: t.proposed, checkpointed: t.checkpointed })),
|
|
524
611
|
this.l2BlockSource.getPendingChainValidationStatus(),
|
|
612
|
+
this.l2BlockSource.getLastProposedCheckpoint(),
|
|
525
613
|
] as const);
|
|
526
614
|
|
|
527
|
-
const [worldState,
|
|
615
|
+
const [worldState, l2Tips, p2p, l1ToL2MessageSourceTips, pendingChainValidationStatus, proposedCheckpointData] =
|
|
616
|
+
syncedBlocks;
|
|
528
617
|
|
|
529
618
|
// Handle zero as a special case, since the block hash won't match across services if we're changing the prefilled data for the genesis block,
|
|
530
619
|
// as the world state can compute the new genesis block hash, but other components use the hardcoded constant.
|
|
531
620
|
// TODO(palla/mbps): Fix the above. All components should be able to handle dynamic genesis block hashes.
|
|
532
621
|
const result =
|
|
533
|
-
(
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
622
|
+
(l2Tips.proposed.number === 0 &&
|
|
623
|
+
l2Tips.checkpointed.block.number === 0 &&
|
|
624
|
+
l2Tips.checkpointed.checkpoint.number === 0 &&
|
|
625
|
+
worldState.number === 0 &&
|
|
626
|
+
p2p.number === 0 &&
|
|
627
|
+
l1ToL2MessageSourceTips.proposed.number === 0 &&
|
|
628
|
+
l1ToL2MessageSourceTips.checkpointed.block.number === 0 &&
|
|
629
|
+
l1ToL2MessageSourceTips.checkpointed.checkpoint.number === 0) ||
|
|
630
|
+
(worldState.hash === l2Tips.proposed.hash &&
|
|
631
|
+
p2p.hash === l2Tips.proposed.hash &&
|
|
632
|
+
l1ToL2MessageSourceTips.proposed.hash === l2Tips.proposed.hash &&
|
|
633
|
+
l1ToL2MessageSourceTips.checkpointed.block.hash === l2Tips.checkpointed.block.hash &&
|
|
634
|
+
l1ToL2MessageSourceTips.checkpointed.checkpoint.hash === l2Tips.checkpointed.checkpoint.hash);
|
|
537
635
|
|
|
538
636
|
if (!result) {
|
|
539
|
-
this.log.debug(`Sequencer sync check failed`, {
|
|
637
|
+
this.log.debug(`Sequencer sync check failed`, {
|
|
638
|
+
worldState,
|
|
639
|
+
l2BlockSource: l2Tips.proposed,
|
|
640
|
+
p2p,
|
|
641
|
+
l1ToL2MessageSourceTips,
|
|
642
|
+
});
|
|
540
643
|
return undefined;
|
|
541
644
|
}
|
|
542
645
|
|
|
@@ -546,8 +649,10 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
546
649
|
const archive = new Fr((await this.worldState.getCommitted().getTreeInfo(MerkleTreeId.ARCHIVE)).root);
|
|
547
650
|
return {
|
|
548
651
|
checkpointNumber: CheckpointNumber.ZERO,
|
|
652
|
+
checkpointedCheckpointNumber: CheckpointNumber.ZERO,
|
|
549
653
|
blockNumber: BlockNumber.ZERO,
|
|
550
654
|
archive,
|
|
655
|
+
hasProposedCheckpoint: false,
|
|
551
656
|
syncedL2Slot,
|
|
552
657
|
pendingChainValidationStatus,
|
|
553
658
|
};
|
|
@@ -560,11 +665,16 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
560
665
|
return undefined;
|
|
561
666
|
}
|
|
562
667
|
|
|
668
|
+
const hasProposedCheckpoint = l2Tips.proposedCheckpoint.checkpoint.number > l2Tips.checkpointed.checkpoint.number;
|
|
669
|
+
|
|
563
670
|
return {
|
|
564
671
|
blockData,
|
|
565
672
|
blockNumber: blockData.header.getBlockNumber(),
|
|
566
673
|
checkpointNumber: blockData.checkpointNumber,
|
|
674
|
+
checkpointedCheckpointNumber: l2Tips.checkpointed.checkpoint.number,
|
|
567
675
|
archive: blockData.archive.root,
|
|
676
|
+
hasProposedCheckpoint,
|
|
677
|
+
proposedCheckpointData,
|
|
568
678
|
syncedL2Slot,
|
|
569
679
|
pendingChainValidationStatus,
|
|
570
680
|
};
|
|
@@ -612,7 +722,10 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
612
722
|
return [false, proposer];
|
|
613
723
|
}
|
|
614
724
|
|
|
615
|
-
this.log.
|
|
725
|
+
this.log.info(`We are the proposer for pipeline slot ${targetSlot}`, {
|
|
726
|
+
targetSlot,
|
|
727
|
+
proposer,
|
|
728
|
+
});
|
|
616
729
|
return [true, proposer];
|
|
617
730
|
}
|
|
618
731
|
|
|
@@ -909,8 +1022,11 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter<Sequ
|
|
|
909
1022
|
type SequencerSyncCheckResult = {
|
|
910
1023
|
blockData?: BlockData;
|
|
911
1024
|
checkpointNumber: CheckpointNumber;
|
|
1025
|
+
checkpointedCheckpointNumber: CheckpointNumber;
|
|
912
1026
|
blockNumber: BlockNumber;
|
|
913
1027
|
archive: Fr;
|
|
1028
|
+
hasProposedCheckpoint: boolean;
|
|
1029
|
+
proposedCheckpointData?: ProposedCheckpointData;
|
|
914
1030
|
syncedL2Slot: SlotNumber;
|
|
915
1031
|
pendingChainValidationStatus: ValidateCheckpointResult;
|
|
916
1032
|
};
|
|
@@ -2,8 +2,10 @@ import type { Logger } from '@aztec/foundation/log';
|
|
|
2
2
|
import {
|
|
3
3
|
CHECKPOINT_ASSEMBLE_TIME,
|
|
4
4
|
CHECKPOINT_INITIALIZATION_TIME,
|
|
5
|
+
type CheckpointTiming,
|
|
5
6
|
DEFAULT_P2P_PROPAGATION_TIME,
|
|
6
7
|
MIN_EXECUTION_TIME,
|
|
8
|
+
createCheckpointTimingModel,
|
|
7
9
|
} from '@aztec/stdlib/timetable';
|
|
8
10
|
|
|
9
11
|
import { SequencerTooSlowError } from './errors.js';
|
|
@@ -11,6 +13,8 @@ import type { SequencerMetrics } from './metrics.js';
|
|
|
11
13
|
import { SequencerState } from './utils.js';
|
|
12
14
|
|
|
13
15
|
export class SequencerTimetable {
|
|
16
|
+
private readonly checkpointTiming: CheckpointTiming;
|
|
17
|
+
|
|
14
18
|
/**
|
|
15
19
|
* How late into the slot can we be to start working. Computed as the total time needed for assembling and publishing a block,
|
|
16
20
|
* assuming an execution time equal to `minExecutionTime`, subtracted from the slot duration. This means that, if the proposer
|
|
@@ -70,6 +74,15 @@ export class SequencerTimetable {
|
|
|
70
74
|
/** Maximum number of blocks that can be built in this slot configuration */
|
|
71
75
|
public readonly maxNumberOfBlocks: number;
|
|
72
76
|
|
|
77
|
+
/** Whether pipelining is enabled (checkpoint finalization deferred to next slot). */
|
|
78
|
+
public readonly pipelining: boolean;
|
|
79
|
+
|
|
80
|
+
/**
|
|
81
|
+
* How far into the target slot attestation collection can extend when pipelining.
|
|
82
|
+
* Covers validator re-execution (one block duration) plus one-way attestation return.
|
|
83
|
+
*/
|
|
84
|
+
public readonly pipeliningAttestationGracePeriod: number;
|
|
85
|
+
|
|
73
86
|
constructor(
|
|
74
87
|
opts: {
|
|
75
88
|
ethereumSlotDuration: number;
|
|
@@ -78,6 +91,7 @@ export class SequencerTimetable {
|
|
|
78
91
|
p2pPropagationTime?: number;
|
|
79
92
|
blockDurationMs?: number;
|
|
80
93
|
enforce: boolean;
|
|
94
|
+
pipelining?: boolean;
|
|
81
95
|
},
|
|
82
96
|
private readonly metrics?: SequencerMetrics,
|
|
83
97
|
private readonly log?: Logger,
|
|
@@ -85,52 +99,31 @@ export class SequencerTimetable {
|
|
|
85
99
|
this.ethereumSlotDuration = opts.ethereumSlotDuration;
|
|
86
100
|
this.aztecSlotDuration = opts.aztecSlotDuration;
|
|
87
101
|
this.l1PublishingTime = opts.l1PublishingTime;
|
|
88
|
-
this.p2pPropagationTime = opts.p2pPropagationTime ?? DEFAULT_P2P_PROPAGATION_TIME;
|
|
89
102
|
this.blockDuration = opts.blockDurationMs ? opts.blockDurationMs / 1000 : undefined;
|
|
90
103
|
this.enforce = opts.enforce;
|
|
104
|
+
this.pipelining = opts.pipelining ?? false;
|
|
105
|
+
|
|
106
|
+
this.checkpointTiming = createCheckpointTimingModel({
|
|
107
|
+
aztecSlotDuration: this.aztecSlotDuration,
|
|
108
|
+
ethereumSlotDuration: this.ethereumSlotDuration,
|
|
109
|
+
blockDuration: this.blockDuration,
|
|
110
|
+
l1PublishingTime: this.l1PublishingTime,
|
|
111
|
+
p2pPropagationTime: opts.p2pPropagationTime ?? DEFAULT_P2P_PROPAGATION_TIME,
|
|
112
|
+
pipelining: this.pipelining,
|
|
113
|
+
});
|
|
91
114
|
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
this.checkpointInitializationTime = 0.5;
|
|
97
|
-
this.minExecutionTime = 1;
|
|
98
|
-
}
|
|
99
|
-
|
|
100
|
-
// Min execution time cannot be less than the block duration if set
|
|
101
|
-
if (this.blockDuration !== undefined && this.minExecutionTime > this.blockDuration) {
|
|
102
|
-
this.minExecutionTime = this.blockDuration;
|
|
103
|
-
}
|
|
115
|
+
this.p2pPropagationTime = this.checkpointTiming.p2pPropagationTime;
|
|
116
|
+
this.checkpointAssembleTime = this.checkpointTiming.checkpointAssembleTime;
|
|
117
|
+
this.checkpointInitializationTime = this.checkpointTiming.checkpointInitializationTime;
|
|
118
|
+
this.minExecutionTime = this.checkpointTiming.minExecutionTime;
|
|
104
119
|
|
|
105
120
|
// Calculate initialization offset - estimate of time needed for sync + proposer check
|
|
106
121
|
// This is the baseline for all sub-slot deadlines
|
|
107
|
-
this.initializationOffset = this.checkpointInitializationTime;
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
this.
|
|
111
|
-
|
|
112
|
-
this.p2pPropagationTime * 2 + // Round-trip propagation
|
|
113
|
-
this.l1PublishingTime; // L1 publishing
|
|
114
|
-
|
|
115
|
-
// Calculate maximum number of blocks that fit in this slot
|
|
116
|
-
if (!this.blockDuration) {
|
|
117
|
-
this.maxNumberOfBlocks = 1; // Single block per slot
|
|
118
|
-
} else {
|
|
119
|
-
const timeReservedAtEnd =
|
|
120
|
-
this.blockDuration + // Last sub-slot for validator re-execution
|
|
121
|
-
this.checkpointFinalizationTime; // Checkpoint finalization
|
|
122
|
-
const timeAvailableForBlocks = this.aztecSlotDuration - this.initializationOffset - timeReservedAtEnd;
|
|
123
|
-
this.maxNumberOfBlocks = Math.floor(timeAvailableForBlocks / this.blockDuration);
|
|
124
|
-
}
|
|
125
|
-
|
|
126
|
-
// Minimum work to do within a slot for building a block with the minimum time for execution and publishing its checkpoint
|
|
127
|
-
const minWorkToDo =
|
|
128
|
-
this.initializationOffset +
|
|
129
|
-
this.minExecutionTime * 2 + // Execution and reexecution
|
|
130
|
-
this.checkpointFinalizationTime;
|
|
131
|
-
|
|
132
|
-
const initializeDeadline = this.aztecSlotDuration - minWorkToDo;
|
|
133
|
-
this.initializeDeadline = initializeDeadline;
|
|
122
|
+
this.initializationOffset = this.checkpointTiming.checkpointInitializationTime;
|
|
123
|
+
this.checkpointFinalizationTime = this.checkpointTiming.checkpointFinalizationTime;
|
|
124
|
+
this.pipeliningAttestationGracePeriod = this.checkpointTiming.pipeliningAttestationGracePeriod;
|
|
125
|
+
this.maxNumberOfBlocks = this.checkpointTiming.calculateMaxBlocksPerSlot();
|
|
126
|
+
this.initializeDeadline = this.checkpointTiming.initializeDeadline;
|
|
134
127
|
|
|
135
128
|
this.log?.info(
|
|
136
129
|
`Sequencer timetable initialized with ${this.maxNumberOfBlocks} blocks per slot (${this.enforce ? 'enforced' : 'not enforced'})`,
|
|
@@ -144,19 +137,37 @@ export class SequencerTimetable {
|
|
|
144
137
|
blockAssembleTime: this.checkpointAssembleTime,
|
|
145
138
|
initializeDeadline: this.initializeDeadline,
|
|
146
139
|
enforce: this.enforce,
|
|
147
|
-
|
|
140
|
+
pipelining: this.pipelining,
|
|
141
|
+
pipeliningAttestationGracePeriod: this.pipeliningAttestationGracePeriod,
|
|
142
|
+
minWorkToDo: this.checkpointTiming.minimumBuildSlotWork,
|
|
148
143
|
blockDuration: this.blockDuration,
|
|
149
144
|
maxNumberOfBlocks: this.maxNumberOfBlocks,
|
|
150
145
|
},
|
|
151
146
|
);
|
|
152
147
|
|
|
153
|
-
if (initializeDeadline <= 0) {
|
|
148
|
+
if (this.initializeDeadline <= 0) {
|
|
154
149
|
throw new Error(
|
|
155
|
-
`Block proposal initialize deadline cannot be negative (got ${initializeDeadline} from total time needed ${
|
|
150
|
+
`Block proposal initialize deadline cannot be negative (got ${this.initializeDeadline} from total time needed ${this.checkpointTiming.minimumBuildSlotWork} and a slot duration of ${this.aztecSlotDuration}).`,
|
|
156
151
|
);
|
|
157
152
|
}
|
|
158
153
|
}
|
|
159
154
|
|
|
155
|
+
public getCheckpointAssemblyDeadline(): number {
|
|
156
|
+
return this.checkpointTiming.checkpointAssemblyDeadline;
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
public getCheckpointAttestationDeadline(): number {
|
|
160
|
+
return this.checkpointTiming.checkpointAttestationDeadline;
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
public getCheckpointAttestationStartDeadline(): number {
|
|
164
|
+
return this.checkpointTiming.checkpointAttestationStartDeadline;
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
public getCheckpointPublishingDeadline(): number {
|
|
168
|
+
return this.checkpointTiming.checkpointPublishingDeadline;
|
|
169
|
+
}
|
|
170
|
+
|
|
160
171
|
public getMaxAllowedTime(
|
|
161
172
|
state: Extract<SequencerState, SequencerState.STOPPED | SequencerState.IDLE | SequencerState.SYNCHRONIZING>,
|
|
162
173
|
): undefined;
|
|
@@ -179,10 +190,11 @@ export class SequencerTimetable {
|
|
|
179
190
|
case SequencerState.WAITING_UNTIL_NEXT_BLOCK:
|
|
180
191
|
return this.initializeDeadline + this.checkpointInitializationTime;
|
|
181
192
|
case SequencerState.ASSEMBLING_CHECKPOINT:
|
|
193
|
+
return this.getCheckpointAssemblyDeadline();
|
|
182
194
|
case SequencerState.COLLECTING_ATTESTATIONS:
|
|
183
|
-
return this.
|
|
195
|
+
return this.getCheckpointAttestationStartDeadline();
|
|
184
196
|
case SequencerState.PUBLISHING_CHECKPOINT:
|
|
185
|
-
return this.
|
|
197
|
+
return this.getCheckpointPublishingDeadline();
|
|
186
198
|
default: {
|
|
187
199
|
const _exhaustiveCheck: never = state;
|
|
188
200
|
throw new Error(`Unexpected state: ${state}`);
|
package/src/test/utils.ts
CHANGED
|
@@ -10,7 +10,11 @@ import { PublicDataWrite } from '@aztec/stdlib/avm';
|
|
|
10
10
|
import { CommitteeAttestation, L2Block } from '@aztec/stdlib/block';
|
|
11
11
|
import { BlockProposal, CheckpointAttestation, CheckpointProposal, ConsensusPayload } from '@aztec/stdlib/p2p';
|
|
12
12
|
import { CheckpointHeader } from '@aztec/stdlib/rollup';
|
|
13
|
-
import {
|
|
13
|
+
import {
|
|
14
|
+
TEST_COORDINATION_SIGNATURE_CONTEXT,
|
|
15
|
+
makeAppendOnlyTreeSnapshot,
|
|
16
|
+
mockTxForRollup,
|
|
17
|
+
} from '@aztec/stdlib/testing';
|
|
14
18
|
import { BlockHeader, GlobalVariables, type Tx, makeProcessedTxFromPrivateOnlyTx } from '@aztec/stdlib/tx';
|
|
15
19
|
|
|
16
20
|
import type { MockProxy } from 'jest-mock-extended';
|
|
@@ -109,6 +113,7 @@ export function createBlockProposal(block: L2Block, signature: Signature): Block
|
|
|
109
113
|
block.archive.root,
|
|
110
114
|
txHashes,
|
|
111
115
|
signature,
|
|
116
|
+
TEST_COORDINATION_SIGNATURE_CONTEXT,
|
|
112
117
|
);
|
|
113
118
|
}
|
|
114
119
|
|
|
@@ -123,12 +128,19 @@ export function createCheckpointProposal(
|
|
|
123
128
|
): CheckpointProposal {
|
|
124
129
|
const txHashes = block.body.txEffects.map(tx => tx.txHash);
|
|
125
130
|
const checkpointHeader = createCheckpointHeaderFromBlock(block);
|
|
126
|
-
return new CheckpointProposal(
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
131
|
+
return new CheckpointProposal(
|
|
132
|
+
checkpointHeader,
|
|
133
|
+
block.archive.root,
|
|
134
|
+
feeAssetPriceModifier,
|
|
135
|
+
checkpointSignature,
|
|
136
|
+
TEST_COORDINATION_SIGNATURE_CONTEXT,
|
|
137
|
+
{
|
|
138
|
+
blockHeader: block.header,
|
|
139
|
+
indexWithinCheckpoint: block.indexWithinCheckpoint,
|
|
140
|
+
txHashes,
|
|
141
|
+
signature: blockSignature ?? checkpointSignature, // Use checkpoint signature as block signature if not provided
|
|
142
|
+
},
|
|
143
|
+
);
|
|
132
144
|
}
|
|
133
145
|
|
|
134
146
|
/**
|
|
@@ -143,10 +155,16 @@ export function createCheckpointAttestation(
|
|
|
143
155
|
feeAssetPriceModifier: bigint = 0n,
|
|
144
156
|
): CheckpointAttestation {
|
|
145
157
|
const checkpointHeader = createCheckpointHeaderFromBlock(block);
|
|
146
|
-
const payload = new ConsensusPayload(
|
|
158
|
+
const payload = new ConsensusPayload(
|
|
159
|
+
checkpointHeader,
|
|
160
|
+
block.archive.root,
|
|
161
|
+
feeAssetPriceModifier,
|
|
162
|
+
TEST_COORDINATION_SIGNATURE_CONTEXT,
|
|
163
|
+
);
|
|
147
164
|
const attestation = new CheckpointAttestation(payload, signature, signature);
|
|
148
|
-
//
|
|
149
|
-
(attestation as any).
|
|
165
|
+
// Bypass signature recovery for testing since we use random signatures
|
|
166
|
+
(attestation as any).getSender = () => sender;
|
|
167
|
+
(attestation as any).getProposer = () => sender;
|
|
150
168
|
return attestation;
|
|
151
169
|
}
|
|
152
170
|
|