@aztec/sequencer-client 0.0.1-commit.993d240 → 0.0.1-commit.9a89641
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +40 -41
- package/dest/client/sequencer-client.d.ts +16 -3
- package/dest/client/sequencer-client.d.ts.map +1 -1
- package/dest/client/sequencer-client.js +18 -14
- package/dest/config.d.ts +9 -4
- package/dest/config.d.ts.map +1 -1
- package/dest/config.js +29 -15
- package/dest/global_variable_builder/fee_predictor.d.ts +1 -1
- package/dest/global_variable_builder/fee_predictor.d.ts.map +1 -1
- package/dest/global_variable_builder/fee_predictor.js +11 -1
- package/dest/global_variable_builder/fee_provider.d.ts +1 -1
- package/dest/global_variable_builder/fee_provider.d.ts.map +1 -1
- package/dest/global_variable_builder/fee_provider.js +27 -5
- package/dest/global_variable_builder/global_builder.d.ts +3 -16
- package/dest/global_variable_builder/global_builder.d.ts.map +1 -1
- package/dest/global_variable_builder/global_builder.js +2 -25
- package/dest/index.d.ts +2 -2
- package/dest/index.d.ts.map +1 -1
- package/dest/index.js +1 -1
- package/dest/publisher/config.d.ts +5 -1
- package/dest/publisher/config.d.ts.map +1 -1
- package/dest/publisher/config.js +11 -1
- package/dest/publisher/l1_to_l2_messaging.d.ts +21 -0
- package/dest/publisher/l1_to_l2_messaging.d.ts.map +1 -0
- package/dest/publisher/l1_to_l2_messaging.js +70 -0
- package/dest/publisher/l1_tx_failed_store/failed_tx_store.d.ts +48 -8
- package/dest/publisher/l1_tx_failed_store/failed_tx_store.d.ts.map +1 -1
- package/dest/publisher/l1_tx_failed_store/failed_tx_store.js +68 -1
- package/dest/publisher/l1_tx_failed_store/file_store_failed_tx_store.d.ts +2 -2
- package/dest/publisher/l1_tx_failed_store/file_store_failed_tx_store.d.ts.map +1 -1
- package/dest/publisher/l1_tx_failed_store/file_store_failed_tx_store.js +4 -2
- package/dest/publisher/l1_tx_failed_store/index.d.ts +2 -2
- package/dest/publisher/l1_tx_failed_store/index.d.ts.map +1 -1
- package/dest/publisher/l1_tx_failed_store/index.js +1 -0
- package/dest/publisher/sequencer-publisher.d.ts +39 -10
- package/dest/publisher/sequencer-publisher.d.ts.map +1 -1
- package/dest/publisher/sequencer-publisher.js +403 -75
- package/dest/publisher/write_json.d.ts +11 -0
- package/dest/publisher/write_json.d.ts.map +1 -0
- package/dest/publisher/write_json.js +57 -0
- package/dest/sequencer/automine/automine_factory.d.ts +5 -3
- package/dest/sequencer/automine/automine_factory.d.ts.map +1 -1
- package/dest/sequencer/automine/automine_factory.js +2 -1
- package/dest/sequencer/automine/automine_sequencer.d.ts +43 -5
- package/dest/sequencer/automine/automine_sequencer.d.ts.map +1 -1
- package/dest/sequencer/automine/automine_sequencer.js +209 -19
- package/dest/sequencer/automine/index.d.ts +3 -0
- package/dest/sequencer/automine/index.d.ts.map +1 -0
- package/dest/sequencer/automine/index.js +2 -0
- package/dest/sequencer/checkpoint_proposal_job.d.ts +28 -15
- package/dest/sequencer/checkpoint_proposal_job.d.ts.map +1 -1
- package/dest/sequencer/checkpoint_proposal_job.js +158 -117
- package/dest/sequencer/events.d.ts +16 -4
- package/dest/sequencer/events.d.ts.map +1 -1
- package/dest/sequencer/index.d.ts +1 -3
- package/dest/sequencer/index.d.ts.map +1 -1
- package/dest/sequencer/index.js +0 -2
- package/dest/sequencer/missing_committee.d.ts +72 -0
- package/dest/sequencer/missing_committee.d.ts.map +1 -0
- package/dest/sequencer/missing_committee.js +139 -0
- package/dest/sequencer/requests_tracker.d.ts +22 -0
- package/dest/sequencer/requests_tracker.d.ts.map +1 -0
- package/dest/sequencer/requests_tracker.js +33 -0
- package/dest/sequencer/sequencer.d.ts +113 -28
- package/dest/sequencer/sequencer.d.ts.map +1 -1
- package/dest/sequencer/sequencer.js +352 -162
- package/dest/test/index.d.ts +3 -3
- package/dest/test/index.d.ts.map +1 -1
- package/dest/test/utils.d.ts +15 -1
- package/dest/test/utils.d.ts.map +1 -1
- package/dest/test/utils.js +18 -1
- package/package.json +29 -28
- package/src/client/sequencer-client.ts +20 -14
- package/src/config.ts +32 -16
- package/src/global_variable_builder/fee_predictor.ts +11 -1
- package/src/global_variable_builder/fee_provider.ts +27 -5
- package/src/global_variable_builder/global_builder.ts +2 -34
- package/src/index.ts +1 -10
- package/src/publisher/config.ts +22 -1
- package/src/publisher/l1_to_l2_messaging.ts +85 -0
- package/src/publisher/l1_tx_failed_store/failed_tx_store.ts +114 -7
- package/src/publisher/l1_tx_failed_store/file_store_failed_tx_store.ts +4 -3
- package/src/publisher/l1_tx_failed_store/index.ts +1 -1
- package/src/publisher/sequencer-publisher.ts +430 -86
- package/src/publisher/write_json.ts +78 -0
- package/src/sequencer/README.md +16 -6
- package/src/sequencer/automine/README.md +18 -4
- package/src/sequencer/automine/automine_factory.ts +8 -1
- package/src/sequencer/automine/automine_sequencer.ts +229 -21
- package/src/sequencer/automine/index.ts +6 -0
- package/src/sequencer/checkpoint_proposal_job.ts +177 -132
- package/src/sequencer/events.ts +16 -3
- package/src/sequencer/index.ts +0 -2
- package/src/sequencer/missing_committee.ts +192 -0
- package/src/sequencer/requests_tracker.ts +43 -0
- package/src/sequencer/sequencer.ts +382 -176
- package/src/test/index.ts +2 -2
- package/src/test/utils.ts +33 -0
- package/dest/sequencer/chain_state_overrides.d.ts +0 -61
- package/dest/sequencer/chain_state_overrides.d.ts.map +0 -1
- package/dest/sequencer/chain_state_overrides.js +0 -98
- package/dest/sequencer/timetable.d.ts +0 -101
- package/dest/sequencer/timetable.d.ts.map +0 -1
- package/dest/sequencer/timetable.js +0 -231
- package/src/sequencer/chain_state_overrides.ts +0 -162
- package/src/sequencer/timetable.ts +0 -295
|
@@ -373,8 +373,8 @@ function _apply_decs_2203_r(targetClass, memberDecs, classDecs, parentClass) {
|
|
|
373
373
|
var _dec, _dec1, _initProto;
|
|
374
374
|
import { Blob, getBlobsPerL1Block, getPrefixedEthBlobCommitments } from '@aztec/blob-lib';
|
|
375
375
|
import { FeeAssetPriceOracle, MULTI_CALL_3_ADDRESS, Multicall3, MulticallForwarderRevertedError, buildSimulationOverridesStateOverride } from '@aztec/ethereum/contracts';
|
|
376
|
-
import { L1FeeAnalyzer } from '@aztec/ethereum/l1-fee-analysis';
|
|
377
|
-
import { MAX_L1_TX_LIMIT, WEI_CONST } from '@aztec/ethereum/l1-tx-utils';
|
|
376
|
+
import { L1FeeAnalyzer, captureWindowBlockFees } from '@aztec/ethereum/l1-fee-analysis';
|
|
377
|
+
import { L1TxTimeoutError, MAX_L1_TX_LIMIT, WEI_CONST } from '@aztec/ethereum/l1-tx-utils';
|
|
378
378
|
import { FormattedViemError, formatViemError, mergeAbis, tryDecodeRevertReason, tryExtractEvent } from '@aztec/ethereum/utils';
|
|
379
379
|
import { CheckpointNumber, SlotNumber } from '@aztec/foundation/branded-types';
|
|
380
380
|
import { trimmedBytesLength } from '@aztec/foundation/buffer';
|
|
@@ -408,6 +408,7 @@ import { SequencerPublisherMetrics } from './sequencer-publisher-metrics.js';
|
|
|
408
408
|
export const Actions = [
|
|
409
409
|
'invalidate-by-invalid-attestation',
|
|
410
410
|
'invalidate-by-insufficient-attestations',
|
|
411
|
+
'prune',
|
|
411
412
|
'propose',
|
|
412
413
|
'governance-signal',
|
|
413
414
|
'vote-offenses',
|
|
@@ -446,6 +447,8 @@ export class SequencerPublisher {
|
|
|
446
447
|
log;
|
|
447
448
|
ethereumSlotDuration;
|
|
448
449
|
aztecSlotDuration;
|
|
450
|
+
previousL1BlockWaitTimeoutMs;
|
|
451
|
+
previousL1BlockWaitPollIntervalMs;
|
|
449
452
|
/** Date provider for wall-clock time. */ dateProvider;
|
|
450
453
|
blobClient;
|
|
451
454
|
/** Optional callback to obtain a replacement publisher when the current one fails to send. */ getNextPublisher;
|
|
@@ -473,6 +476,8 @@ export class SequencerPublisher {
|
|
|
473
476
|
this.log = deps.log ?? createLogger('sequencer:publisher');
|
|
474
477
|
this.ethereumSlotDuration = BigInt(config.ethereumSlotDuration);
|
|
475
478
|
this.aztecSlotDuration = BigInt(config.aztecSlotDuration);
|
|
479
|
+
this.previousL1BlockWaitTimeoutMs = config.sequencerPublisherPreviousL1BlockWaitTimeoutMs;
|
|
480
|
+
this.previousL1BlockWaitPollIntervalMs = config.sequencerPublisherPreviousL1BlockWaitPollIntervalMs;
|
|
476
481
|
this.dateProvider = deps.dateProvider;
|
|
477
482
|
this.epochCache = deps.epochCache;
|
|
478
483
|
this.lastActions = deps.lastActions;
|
|
@@ -507,20 +512,87 @@ export class SequencerPublisher {
|
|
|
507
512
|
});
|
|
508
513
|
}
|
|
509
514
|
/**
|
|
510
|
-
*
|
|
511
|
-
*
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
}
|
|
515
|
+
* Logs the gas-pricing data of a failed L1 transaction at warn — so underpricing is diagnosable
|
|
516
|
+
* from logs even with no failed-tx store configured — and backs the record up to the store when
|
|
517
|
+
* one is. When captureFeeSummary is true, also records the fee data of the already-mined L1
|
|
518
|
+
* blocks in the target slot's inclusion window.
|
|
519
|
+
*/ backupFailedTx(failedTx, opts) {
|
|
516
520
|
const tx = {
|
|
517
521
|
...failedTx,
|
|
518
522
|
timestamp: Date.now()
|
|
519
523
|
};
|
|
520
524
|
// Fire and forget - don't block on backup
|
|
521
|
-
void
|
|
522
|
-
|
|
523
|
-
|
|
525
|
+
void (async ()=>{
|
|
526
|
+
try {
|
|
527
|
+
// Prefer a pre-captured summary (shared across a batch of failures in the same slot) so we
|
|
528
|
+
// don't re-read the fee window per record. A capture error must not lose the record itself.
|
|
529
|
+
const feeSummary = opts?.sharedFeeSummary ?? (opts?.captureFeeSummary ? await this.captureFeeEnvironment(opts.targetSlot).catch(()=>undefined) : undefined);
|
|
530
|
+
if (feeSummary) {
|
|
531
|
+
tx.gasInfo = {
|
|
532
|
+
...tx.gasInfo,
|
|
533
|
+
...feeSummary
|
|
534
|
+
};
|
|
535
|
+
}
|
|
536
|
+
if (tx.gasInfo) {
|
|
537
|
+
this.log.warn(`Gas pricing data for failed L1 tx (${tx.failureType})`, {
|
|
538
|
+
failureType: tx.failureType,
|
|
539
|
+
actions: tx.context.actions,
|
|
540
|
+
slot: tx.context.slot,
|
|
541
|
+
...tx.gasInfo,
|
|
542
|
+
...tx.timing
|
|
543
|
+
});
|
|
544
|
+
}
|
|
545
|
+
const store = await this.failedTxStore;
|
|
546
|
+
if (store) {
|
|
547
|
+
await store.saveFailedTx(tx);
|
|
548
|
+
}
|
|
549
|
+
} catch (err) {
|
|
550
|
+
this.log.warn(`Failed to backup failed L1 tx to store`, err);
|
|
551
|
+
}
|
|
552
|
+
})();
|
|
553
|
+
}
|
|
554
|
+
/**
|
|
555
|
+
* Captures per-block fee data for the L1 blocks in the target slot's inclusion window (the blocks the
|
|
556
|
+
* tx could have landed in) for underpricing diagnostics. Reads only already-mined blocks, so it never
|
|
557
|
+
* waits on the chain. Safe to call off the critical path: the underlying capture never throws, and this
|
|
558
|
+
* returns undefined when there is no target slot or the window is not yet mined (e.g. an early send
|
|
559
|
+
* failure), in which case the record simply carries no window data.
|
|
560
|
+
*/ async captureFeeEnvironment(targetL2Slot) {
|
|
561
|
+
if (targetL2Slot === undefined) {
|
|
562
|
+
return undefined;
|
|
563
|
+
}
|
|
564
|
+
const l1Constants = this.epochCache.getL1Constants();
|
|
565
|
+
// The inclusion window is [start of slot N, start of slot N+1): all L1 blocks that can include a tx
|
|
566
|
+
// for this L2 slot. getTimestampForSlot returns seconds, matching block.timestamp.
|
|
567
|
+
const windowStartS = getTimestampForSlot(targetL2Slot, l1Constants);
|
|
568
|
+
const windowEndS = getTimestampForSlot(SlotNumber(Number(targetL2Slot) + 1), l1Constants);
|
|
569
|
+
const windowBlocks = await captureWindowBlockFees(this.l1TxUtils.client, windowStartS, windowEndS);
|
|
570
|
+
if (windowBlocks.length === 0) {
|
|
571
|
+
return undefined;
|
|
572
|
+
}
|
|
573
|
+
return {
|
|
574
|
+
windowBlocks
|
|
575
|
+
};
|
|
576
|
+
}
|
|
577
|
+
/** Computes timing info relative to the L2 slot deadline. */ computeTimingInfo(targetL2Slot) {
|
|
578
|
+
if (targetL2Slot === undefined) {
|
|
579
|
+
return undefined;
|
|
580
|
+
}
|
|
581
|
+
const l1Constants = this.epochCache.getL1Constants();
|
|
582
|
+
const slotDeadlineS = getTimestampForSlot(SlotNumber(Number(targetL2Slot) + 1), l1Constants);
|
|
583
|
+
const slotDeadlineMs = Number(slotDeadlineS) * 1000;
|
|
584
|
+
return {
|
|
585
|
+
targetL2Slot: Number(targetL2Slot),
|
|
586
|
+
slotDeadlineTimestampS: slotDeadlineS,
|
|
587
|
+
msUntilSlotDeadline: slotDeadlineMs - this.dateProvider.now()
|
|
588
|
+
};
|
|
589
|
+
}
|
|
590
|
+
/**
|
|
591
|
+
* Builds an id for a synthetic failure record (send-error/timeout) that has no on-chain tx hash.
|
|
592
|
+
* Includes the failure time so each attempt — including retries of the same slot — is stored as its
|
|
593
|
+
* own record rather than overwriting the previous one.
|
|
594
|
+
*/ failureRecordId(actions, targetSlot) {
|
|
595
|
+
return keccak256(toHex(`${actions.join(',')}:${targetSlot ?? ''}:${Date.now()}`));
|
|
524
596
|
}
|
|
525
597
|
getRollupContract() {
|
|
526
598
|
return this.rollupContract;
|
|
@@ -600,10 +672,10 @@ export class SequencerPublisher {
|
|
|
600
672
|
}
|
|
601
673
|
/**
|
|
602
674
|
* Sends all requests that are still valid.
|
|
603
|
-
* @param targetSlot - The target L2 slot for this send. When provided (
|
|
604
|
-
* sendRequestsAt), it is threaded into bundleSimulate so the block.timestamp override
|
|
605
|
-
*
|
|
606
|
-
*
|
|
675
|
+
* @param targetSlot - The target L2 slot for this send. When provided (the production path, via
|
|
676
|
+
* sendRequestsAt), it is threaded into bundleSimulate so the block.timestamp override matches
|
|
677
|
+
* the slot the propose is built for. When omitted, falls back to getCurrentL2Slot() for the
|
|
678
|
+
* AutomineSequencer, which publishes synchronously within the current slot.
|
|
607
679
|
* @returns one of:
|
|
608
680
|
* - A receipt and stats if the tx succeeded
|
|
609
681
|
* - a receipt and errorMsg if it failed on L1
|
|
@@ -648,7 +720,7 @@ export class SequencerPublisher {
|
|
|
648
720
|
const bundleResult = await this.bundleSimulator.simulate(validRequests, currentL2Slot);
|
|
649
721
|
if (bundleResult.kind === 'aborted') {
|
|
650
722
|
this.logDroppedInSim(bundleResult.droppedRequests);
|
|
651
|
-
void this.backupDroppedInSim(bundleResult.droppedRequests);
|
|
723
|
+
void this.backupDroppedInSim(bundleResult.droppedRequests, currentL2Slot).catch((err)=>this.log.error(`Failed to backup requests dropped in simulation`, err));
|
|
652
724
|
return undefined;
|
|
653
725
|
}
|
|
654
726
|
const { requests, droppedRequests, gasLimit } = bundleResult.kind === 'fallback' ? {
|
|
@@ -668,7 +740,7 @@ export class SequencerPublisher {
|
|
|
668
740
|
requests: requests.map((request)=>request.action),
|
|
669
741
|
txConfig
|
|
670
742
|
});
|
|
671
|
-
const result = await this.forwardWithPublisherRotation(requests, txConfig, blobConfig);
|
|
743
|
+
const result = await this.forwardWithPublisherRotation(requests, txConfig, blobConfig, currentL2Slot);
|
|
672
744
|
if (result === undefined) {
|
|
673
745
|
return undefined;
|
|
674
746
|
}
|
|
@@ -687,6 +759,47 @@ export class SequencerPublisher {
|
|
|
687
759
|
} catch (err) {
|
|
688
760
|
const viemError = formatViemError(err);
|
|
689
761
|
this.log.error(`Failed to publish bundled transactions`, viemError);
|
|
762
|
+
if (err instanceof TimeoutError) {
|
|
763
|
+
const timeoutState = err instanceof L1TxTimeoutError ? err.txState : undefined;
|
|
764
|
+
void (async ()=>{
|
|
765
|
+
// The RPC is likely degraded right after a timeout, so back up without the block number
|
|
766
|
+
// rather than leaking an unhandled rejection.
|
|
767
|
+
const l1BlockNumber = await this.l1TxUtils.getBlockNumber().catch(()=>0n);
|
|
768
|
+
this.backupFailedTx({
|
|
769
|
+
id: this.failureRecordId(validRequests.map((r)=>r.action), currentL2Slot),
|
|
770
|
+
failureType: 'timeout',
|
|
771
|
+
request: {
|
|
772
|
+
to: MULTI_CALL_3_ADDRESS,
|
|
773
|
+
data: '0x'
|
|
774
|
+
},
|
|
775
|
+
l1BlockNumber,
|
|
776
|
+
error: {
|
|
777
|
+
message: viemError.message,
|
|
778
|
+
name: 'TimeoutError'
|
|
779
|
+
},
|
|
780
|
+
context: {
|
|
781
|
+
actions: validRequests.map((r)=>r.action),
|
|
782
|
+
requests: validRequests.filter((r)=>r.request.to !== null).map((r)=>({
|
|
783
|
+
action: r.action,
|
|
784
|
+
to: r.request.to,
|
|
785
|
+
data: r.request.data
|
|
786
|
+
})),
|
|
787
|
+
sender: this.getSenderAddress().toString(),
|
|
788
|
+
slot: Number(currentL2Slot)
|
|
789
|
+
},
|
|
790
|
+
timing: this.computeTimingInfo(currentL2Slot),
|
|
791
|
+
gasInfo: timeoutState ? {
|
|
792
|
+
sentGasPriceLadder: timeoutState.gasPriceHistory,
|
|
793
|
+
attempts: timeoutState.attempts,
|
|
794
|
+
gasLimit: timeoutState.gasLimit,
|
|
795
|
+
nonce: timeoutState.nonce
|
|
796
|
+
} : undefined
|
|
797
|
+
}, {
|
|
798
|
+
captureFeeSummary: true,
|
|
799
|
+
targetSlot: currentL2Slot
|
|
800
|
+
});
|
|
801
|
+
})();
|
|
802
|
+
}
|
|
690
803
|
return undefined;
|
|
691
804
|
} finally{
|
|
692
805
|
try {
|
|
@@ -707,37 +820,50 @@ export class SequencerPublisher {
|
|
|
707
820
|
});
|
|
708
821
|
}
|
|
709
822
|
}
|
|
710
|
-
/** Backs up entries dropped by bundle simulation, one record per dropped action. */ async backupDroppedInSim(dropped) {
|
|
823
|
+
/** Backs up entries dropped by bundle simulation, one record per dropped action. */ async backupDroppedInSim(dropped, targetSlot) {
|
|
711
824
|
if (dropped.length === 0) {
|
|
712
825
|
return;
|
|
713
826
|
}
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
this.
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
827
|
+
// Invoked as `void backupDroppedInSim(...)` on the publish path, so it must not throw.
|
|
828
|
+
try {
|
|
829
|
+
const l1BlockNumber = await this.l1TxUtils.getBlockNumber();
|
|
830
|
+
// Every dropped entry failed in the same slot against the same L1 fee conditions, so capture
|
|
831
|
+
// the fee environment once and share it rather than re-reading the window per entry.
|
|
832
|
+
const sharedFeeSummary = await this.captureFeeEnvironment(targetSlot).catch(()=>undefined);
|
|
833
|
+
const timing = this.computeTimingInfo(targetSlot);
|
|
834
|
+
for (const { request: req } of dropped){
|
|
835
|
+
this.backupFailedTx({
|
|
836
|
+
id: keccak256(req.request.data),
|
|
837
|
+
failureType: 'simulation',
|
|
838
|
+
request: {
|
|
839
|
+
to: req.request.to,
|
|
840
|
+
data: req.request.data
|
|
841
|
+
},
|
|
842
|
+
l1BlockNumber,
|
|
843
|
+
error: {
|
|
844
|
+
message: 'Bundle entry dropped: action reverted in sim'
|
|
845
|
+
},
|
|
846
|
+
context: {
|
|
847
|
+
actions: [
|
|
848
|
+
req.action
|
|
849
|
+
],
|
|
850
|
+
sender: this.getSenderAddress().toString(),
|
|
851
|
+
slot: targetSlot !== undefined ? Number(targetSlot) : undefined
|
|
852
|
+
},
|
|
853
|
+
timing
|
|
854
|
+
}, {
|
|
855
|
+
sharedFeeSummary
|
|
856
|
+
});
|
|
857
|
+
}
|
|
858
|
+
} catch (err) {
|
|
859
|
+
this.log.warn(`Failed to back up dropped-in-sim entries`, err);
|
|
734
860
|
}
|
|
735
861
|
}
|
|
736
862
|
/**
|
|
737
863
|
* Forwards transactions via Multicall3, rotating to the next available publisher if a send
|
|
738
864
|
* failure occurs (i.e. the tx never reached the chain).
|
|
739
865
|
* On-chain reverts and simulation errors are returned as-is without rotation.
|
|
740
|
-
*/ async forwardWithPublisherRotation(validRequests, txConfig, blobConfig) {
|
|
866
|
+
*/ async forwardWithPublisherRotation(validRequests, txConfig, blobConfig, targetSlot) {
|
|
741
867
|
if (!txConfig?.gasLimit) {
|
|
742
868
|
throw new Error('gasLimit is required for bundled transactions');
|
|
743
869
|
}
|
|
@@ -766,11 +892,13 @@ export class SequencerPublisher {
|
|
|
766
892
|
this.log.error('Forwarder transaction reverted on-chain; not rotating publisher', err, {
|
|
767
893
|
transactionHash: err.receipt.transactionHash
|
|
768
894
|
});
|
|
895
|
+
this.backupRevertFailure(validRequests, err, currentPublisher, targetSlot);
|
|
769
896
|
return undefined;
|
|
770
897
|
}
|
|
771
898
|
const viemError = formatViemError(err);
|
|
772
899
|
if (!this.getNextPublisher) {
|
|
773
900
|
this.log.error('Failed to publish bundled transactions', viemError);
|
|
901
|
+
this.backupSendFailure(validRequests, viemError, currentPublisher, targetSlot);
|
|
774
902
|
return undefined;
|
|
775
903
|
}
|
|
776
904
|
this.log.warn(`Publisher ${currentPublisher.getSenderAddress()} failed to send, rotating to next publisher`, viemError);
|
|
@@ -781,39 +909,153 @@ export class SequencerPublisher {
|
|
|
781
909
|
this.log.error(`All available publishers exhausted (tried ${triedAddresses.length}), failed to publish bundled transactions`, viemError, {
|
|
782
910
|
triedAddresses: triedAddresses.map((a)=>a.toString())
|
|
783
911
|
});
|
|
912
|
+
this.backupSendFailure(validRequests, viemError, currentPublisher, targetSlot);
|
|
784
913
|
return undefined;
|
|
785
914
|
}
|
|
786
915
|
currentPublisher = nextPublisher;
|
|
787
916
|
}
|
|
788
917
|
}
|
|
789
918
|
}
|
|
919
|
+
/** Backs up an on-chain revert failure to the failed tx store. */ backupRevertFailure(requests, err, publisher, targetSlot) {
|
|
920
|
+
this.backupFailedTx({
|
|
921
|
+
id: err.receipt.transactionHash,
|
|
922
|
+
failureType: 'revert',
|
|
923
|
+
request: {
|
|
924
|
+
to: MULTI_CALL_3_ADDRESS,
|
|
925
|
+
data: '0x'
|
|
926
|
+
},
|
|
927
|
+
l1BlockNumber: err.receipt.blockNumber,
|
|
928
|
+
receipt: {
|
|
929
|
+
transactionHash: err.receipt.transactionHash,
|
|
930
|
+
blockNumber: err.receipt.blockNumber,
|
|
931
|
+
gasUsed: err.receipt.gasUsed,
|
|
932
|
+
status: 'reverted'
|
|
933
|
+
},
|
|
934
|
+
error: {
|
|
935
|
+
message: err.message,
|
|
936
|
+
name: err.name
|
|
937
|
+
},
|
|
938
|
+
context: {
|
|
939
|
+
actions: requests.map((r)=>r.action),
|
|
940
|
+
requests: requests.filter((r)=>r.request.to !== null).map((r)=>({
|
|
941
|
+
action: r.action,
|
|
942
|
+
to: r.request.to,
|
|
943
|
+
data: r.request.data
|
|
944
|
+
})),
|
|
945
|
+
sender: publisher.getSenderAddress().toString(),
|
|
946
|
+
slot: targetSlot !== undefined ? Number(targetSlot) : undefined
|
|
947
|
+
},
|
|
948
|
+
gasInfo: err.txState ? {
|
|
949
|
+
sentGasPrice: err.txState.gasPrice,
|
|
950
|
+
gasLimit: err.txState.gasLimit,
|
|
951
|
+
nonce: err.txState.nonce
|
|
952
|
+
} : undefined,
|
|
953
|
+
timing: this.computeTimingInfo(targetSlot)
|
|
954
|
+
}, {
|
|
955
|
+
captureFeeSummary: true,
|
|
956
|
+
targetSlot
|
|
957
|
+
});
|
|
958
|
+
}
|
|
959
|
+
/** Backs up a send failure (tx never reached chain) to the failed tx store. */ backupSendFailure(requests, error, publisher, targetSlot) {
|
|
960
|
+
// If we can't get the block number, still back up without it.
|
|
961
|
+
void this.l1TxUtils.getBlockNumber().catch(()=>0n).then((l1BlockNumber)=>{
|
|
962
|
+
this.backupFailedTx({
|
|
963
|
+
id: this.failureRecordId(requests.map((r)=>r.action), targetSlot),
|
|
964
|
+
failureType: 'send-error',
|
|
965
|
+
request: {
|
|
966
|
+
to: MULTI_CALL_3_ADDRESS,
|
|
967
|
+
data: '0x'
|
|
968
|
+
},
|
|
969
|
+
l1BlockNumber,
|
|
970
|
+
error: {
|
|
971
|
+
message: error.message,
|
|
972
|
+
name: 'name' in error ? error.name : undefined
|
|
973
|
+
},
|
|
974
|
+
context: {
|
|
975
|
+
actions: requests.map((r)=>r.action),
|
|
976
|
+
requests: requests.filter((r)=>r.request.to !== null).map((r)=>({
|
|
977
|
+
action: r.action,
|
|
978
|
+
to: r.request.to,
|
|
979
|
+
data: r.request.data
|
|
980
|
+
})),
|
|
981
|
+
sender: publisher.getSenderAddress().toString(),
|
|
982
|
+
slot: targetSlot !== undefined ? Number(targetSlot) : undefined
|
|
983
|
+
},
|
|
984
|
+
timing: this.computeTimingInfo(targetSlot)
|
|
985
|
+
}, {
|
|
986
|
+
captureFeeSummary: true,
|
|
987
|
+
targetSlot
|
|
988
|
+
});
|
|
989
|
+
});
|
|
990
|
+
}
|
|
790
991
|
/*
|
|
791
992
|
* Schedules sending all enqueued requests at (or after) the start of the given L2 slot.
|
|
792
|
-
* Sleeps until one L1 slot before the L2 slot boundary so the tx has a chance of being
|
|
793
|
-
* picked up by the first L1 block of the L2 slot.
|
|
794
|
-
* NB: there is a known correctness risk — being included in the L1 block right before the
|
|
795
|
-
* L2 slot starts would revert propose with HeaderLib__InvalidSlotNumber.
|
|
796
|
-
* Uses InterruptibleSleep so it can be cancelled via interrupt().
|
|
797
993
|
*/ async sendRequestsAt(targetSlot) {
|
|
798
|
-
|
|
799
|
-
// Start of the target L2 slot, in ms (getTimestampForSlot returns seconds).
|
|
800
|
-
const startOfTargetSlotMs = Number(getTimestampForSlot(targetSlot, l1Constants)) * 1000;
|
|
801
|
-
// Aim to be in the mempool one L1 slot before the L2 slot starts, so we have a chance of
|
|
802
|
-
// being picked up by the first L1 block of the L2 slot.
|
|
803
|
-
const submitAfterMs = startOfTargetSlotMs - Number(this.ethereumSlotDuration) * 1000;
|
|
804
|
-
const sleepMs = submitAfterMs - this.dateProvider.now();
|
|
805
|
-
if (sleepMs > 0) {
|
|
806
|
-
this.log.debug(`Sleeping ${sleepMs}ms before sending requests`, {
|
|
807
|
-
targetSlot,
|
|
808
|
-
submitAfterMs
|
|
809
|
-
});
|
|
810
|
-
await this.interruptibleSleep.sleep(sleepMs);
|
|
811
|
-
}
|
|
994
|
+
await this.waitForTargetSlot(targetSlot);
|
|
812
995
|
if (this.interrupted) {
|
|
813
996
|
return undefined;
|
|
814
997
|
}
|
|
815
998
|
return this.sendRequests(targetSlot);
|
|
816
999
|
}
|
|
1000
|
+
/**
|
|
1001
|
+
* Sleeps until one L1 slot before the L2 slot boundary, and then waits for that L1 block
|
|
1002
|
+
* to be mined, so we don't risk being included in it. If that block never gets mined after
|
|
1003
|
+
* a timeout, we assume it got skipped on L1, so we send the tx anyway.
|
|
1004
|
+
*/ async waitForTargetSlot(targetSlot) {
|
|
1005
|
+
const l1Constants = this.epochCache.getL1Constants();
|
|
1006
|
+
const nowInSeconds = this.dateProvider.nowInSeconds();
|
|
1007
|
+
const startOfTargetSlotTs = getTimestampForSlot(targetSlot, l1Constants);
|
|
1008
|
+
const previousL1BlockTs = startOfTargetSlotTs - this.ethereumSlotDuration;
|
|
1009
|
+
const waitDeadlineTs = previousL1BlockTs + BigInt(this.previousL1BlockWaitTimeoutMs / 1000);
|
|
1010
|
+
const logCtx = {
|
|
1011
|
+
targetSlot,
|
|
1012
|
+
startOfTargetSlotTs,
|
|
1013
|
+
nowInSeconds,
|
|
1014
|
+
previousL1BlockTs,
|
|
1015
|
+
waitDeadlineTs
|
|
1016
|
+
};
|
|
1017
|
+
// Check if we are already past time
|
|
1018
|
+
if (nowInSeconds >= startOfTargetSlotTs) {
|
|
1019
|
+
this.log.verbose(`Target slot ${targetSlot} already started, sending requests immediately`, logCtx);
|
|
1020
|
+
return;
|
|
1021
|
+
}
|
|
1022
|
+
// Otherwise we wait
|
|
1023
|
+
this.log.debug(`Waiting for slot ${targetSlot} before sending requests`, logCtx);
|
|
1024
|
+
// Wait until previous L1 block timestamp first
|
|
1025
|
+
const sleepMs = (Number(previousL1BlockTs) - nowInSeconds) * 1000;
|
|
1026
|
+
if (sleepMs > 0 && !this.interrupted) {
|
|
1027
|
+
this.log.trace(`Sleeping ${sleepMs}ms before waiting for previous L1 block`, logCtx);
|
|
1028
|
+
await this.interruptibleSleep.sleep(sleepMs);
|
|
1029
|
+
}
|
|
1030
|
+
// Then loop until we see the previous L1 block, so we know that we cannot be included in it.
|
|
1031
|
+
// We time out after a while, once we are sure that that block is skipped in L1.
|
|
1032
|
+
while(!this.interrupted){
|
|
1033
|
+
try {
|
|
1034
|
+
const nowInSeconds = this.dateProvider.nowInSeconds();
|
|
1035
|
+
logCtx.nowInSeconds = nowInSeconds;
|
|
1036
|
+
if (nowInSeconds >= waitDeadlineTs) {
|
|
1037
|
+
this.log.warn(`Timed out waiting for previous L1 block before sending requests, proceeding`, logCtx);
|
|
1038
|
+
return;
|
|
1039
|
+
}
|
|
1040
|
+
const latestBlockTs = await this.l1TxUtils.getBlock().then((b)=>b.timestamp);
|
|
1041
|
+
if (latestBlockTs >= previousL1BlockTs) {
|
|
1042
|
+
this.log.debug(`Previous L1 block mined, proceeding to send requests`, {
|
|
1043
|
+
...logCtx,
|
|
1044
|
+
latestBlockTs
|
|
1045
|
+
});
|
|
1046
|
+
return;
|
|
1047
|
+
}
|
|
1048
|
+
this.log.trace(`Previous L1 block not mined yet, continuing to wait`, {
|
|
1049
|
+
...logCtx,
|
|
1050
|
+
latestBlockTs
|
|
1051
|
+
});
|
|
1052
|
+
} catch (err) {
|
|
1053
|
+
this.log.error(`Error while waiting for previous L1 block before sending requests; retrying`, err, logCtx);
|
|
1054
|
+
} finally{
|
|
1055
|
+
await this.interruptibleSleep.sleep(this.previousL1BlockWaitPollIntervalMs);
|
|
1056
|
+
}
|
|
1057
|
+
}
|
|
1058
|
+
}
|
|
817
1059
|
callbackBundledTransactions(requests, result) {
|
|
818
1060
|
const actionsListStr = requests.map((r)=>r.action).join(', ');
|
|
819
1061
|
this.log.verbose(`Published bundled transactions (${actionsListStr})`, {
|
|
@@ -858,8 +1100,7 @@ export class SequencerPublisher {
|
|
|
858
1100
|
'InvalidProposer',
|
|
859
1101
|
'InvalidArchive'
|
|
860
1102
|
];
|
|
861
|
-
const
|
|
862
|
-
const slotOffset = pipelined ? this.aztecSlotDuration : 0n;
|
|
1103
|
+
const slotOffset = this.aztecSlotDuration;
|
|
863
1104
|
const nextL1SlotTs = this.getNextL1SlotTimestamp() + slotOffset;
|
|
864
1105
|
return this.rollupContract.canProposeAt(tipArchive.toBuffer(), msgSender.toString(), nextL1SlotTs, await buildSimulationOverridesStateOverride(this.rollupContract, simulationOverridesPlan)).catch((err)=>{
|
|
865
1106
|
if (err instanceof FormattedViemError && expectedErrors.find((e)=>err.message.includes(e))) {
|
|
@@ -995,9 +1236,9 @@ export class SequencerPublisher {
|
|
|
995
1236
|
request: {
|
|
996
1237
|
to: request.to,
|
|
997
1238
|
data: request.data,
|
|
998
|
-
value: request.value
|
|
1239
|
+
value: request.value
|
|
999
1240
|
},
|
|
1000
|
-
l1BlockNumber
|
|
1241
|
+
l1BlockNumber,
|
|
1001
1242
|
error: {
|
|
1002
1243
|
message: viemError.message,
|
|
1003
1244
|
name: viemError.name
|
|
@@ -1008,7 +1249,11 @@ export class SequencerPublisher {
|
|
|
1008
1249
|
],
|
|
1009
1250
|
checkpointNumber,
|
|
1010
1251
|
sender: this.getSenderAddress().toString()
|
|
1011
|
-
}
|
|
1252
|
+
},
|
|
1253
|
+
timing: this.computeTimingInfo(this.getCurrentL2Slot())
|
|
1254
|
+
}, {
|
|
1255
|
+
captureFeeSummary: true,
|
|
1256
|
+
targetSlot: this.getCurrentL2Slot()
|
|
1012
1257
|
});
|
|
1013
1258
|
throw new Error(`Failed to simulate invalidate checkpoint ${checkpointNumber}`, {
|
|
1014
1259
|
cause: viemError
|
|
@@ -1025,7 +1270,11 @@ export class SequencerPublisher {
|
|
|
1025
1270
|
reason
|
|
1026
1271
|
};
|
|
1027
1272
|
this.log.debug(`Building invalidate checkpoint ${checkpoint.checkpointNumber} request`, logData);
|
|
1028
|
-
|
|
1273
|
+
// Use the exact packed tuple posted to L1 verbatim. A repack via `packAttestations` is not a
|
|
1274
|
+
// byte-faithful inverse of `fromPacked` (a canonicalized yParity byte or an all-zero signature slot
|
|
1275
|
+
// round-trips differently), so it would diverge from the stored `attestationsHash` and revert the
|
|
1276
|
+
// invalidation.
|
|
1277
|
+
const attestationsAndSigners = validationResult.verbatimAttestations;
|
|
1029
1278
|
if (reason === 'invalid-attestation') {
|
|
1030
1279
|
return this.rollupContract.buildInvalidateBadAttestationRequest(checkpoint.checkpointNumber, attestationsAndSigners, committee, validationResult.invalidIndex);
|
|
1031
1280
|
} else if (reason === 'insufficient-attestations') {
|
|
@@ -1047,6 +1296,17 @@ export class SequencerPublisher {
|
|
|
1047
1296
|
this.log.warn(`Cannot enqueue vote cast signal ${signalType} for address zero at slot ${slotNumber}`);
|
|
1048
1297
|
return false;
|
|
1049
1298
|
}
|
|
1299
|
+
const canonicalRollup = await base.getRollupAddress();
|
|
1300
|
+
if (!canonicalRollup.equals(EthAddress.fromString(this.rollupContract.address))) {
|
|
1301
|
+
this.log.warn(`Rollup ${this.rollupContract.address} is not canonical, skipping governance signal`, {
|
|
1302
|
+
slotNumber,
|
|
1303
|
+
signalType,
|
|
1304
|
+
canonicalRollup,
|
|
1305
|
+
targetRollup: this.rollupContract.address,
|
|
1306
|
+
payload: payload.toString()
|
|
1307
|
+
});
|
|
1308
|
+
return false;
|
|
1309
|
+
}
|
|
1050
1310
|
const round = await base.computeRound(slotNumber);
|
|
1051
1311
|
const roundInfo = await base.getRoundInfo(this.rollupContract.address, round);
|
|
1052
1312
|
if (roundInfo.quorumReached) {
|
|
@@ -1059,22 +1319,36 @@ export class SequencerPublisher {
|
|
|
1059
1319
|
this.log.warn(`Skipping vote cast for payload with empty code`);
|
|
1060
1320
|
return false;
|
|
1061
1321
|
}
|
|
1062
|
-
//
|
|
1063
|
-
//
|
|
1064
|
-
//
|
|
1065
|
-
|
|
1066
|
-
let proposed = false;
|
|
1322
|
+
// Classify the payload against the Governance proposal history so we stop signalling once its
|
|
1323
|
+
// proposal is live or was already executed, while still re-signalling one whose proposal was
|
|
1324
|
+
// merely rejected/dropped/expired.
|
|
1325
|
+
let status = 'none';
|
|
1067
1326
|
try {
|
|
1068
|
-
|
|
1327
|
+
status = await base.getPayloadProposalStatus(payload.toString());
|
|
1069
1328
|
} catch (err) {
|
|
1070
1329
|
// We deliberately swallow the error and proceed to signal. Failing closed (skipping the
|
|
1071
1330
|
// signal) on transient RPC errors would let a flaky L1 endpoint silence governance
|
|
1072
1331
|
// participation entirely; failing open at worst produces a duplicate signal that the
|
|
1073
1332
|
// contract will simply count alongside others in the round.
|
|
1074
|
-
this.log.error(`Failed to check
|
|
1333
|
+
this.log.error(`Failed to check governance proposal status for payload ${payload} (signalling anyway)`, err, {
|
|
1334
|
+
slotNumber,
|
|
1335
|
+
signalType
|
|
1336
|
+
});
|
|
1337
|
+
}
|
|
1338
|
+
if (status === 'live') {
|
|
1339
|
+
this.log.info(`Payload ${payload} has a live governance proposal, stopping signals`, {
|
|
1340
|
+
slotNumber,
|
|
1341
|
+
signalType,
|
|
1342
|
+
payload: payload.toString()
|
|
1343
|
+
});
|
|
1344
|
+
return false;
|
|
1075
1345
|
}
|
|
1076
|
-
if (
|
|
1077
|
-
this.log.info(`Payload ${payload}
|
|
1346
|
+
if (status === 'executed' && !this.config.governanceProposerForcePayloadVote) {
|
|
1347
|
+
this.log.info(`Payload ${payload} was executed by governance within lookback, stopping signals ` + `(set GOVERNANCE_PROPOSER_FORCE_PAYLOAD_VOTE to re-signal)`, {
|
|
1348
|
+
slotNumber,
|
|
1349
|
+
signalType,
|
|
1350
|
+
payload: payload.toString()
|
|
1351
|
+
});
|
|
1078
1352
|
return false;
|
|
1079
1353
|
}
|
|
1080
1354
|
const cachedLastVote = this.lastActions[signalType];
|
|
@@ -1123,6 +1397,56 @@ export class SequencerPublisher {
|
|
|
1123
1397
|
*/ enqueueGovernanceCastSignal(governancePayload, slotNumber, signerAddress, signer) {
|
|
1124
1398
|
return this.enqueueCastSignalHelper(slotNumber, 'governance-signal', governancePayload, this.govProposerContract, signerAddress, signer);
|
|
1125
1399
|
}
|
|
1400
|
+
/**
|
|
1401
|
+
* Enqueues a `prune()` transaction if the rollup is prunable at the given slot's L1 timestamp.
|
|
1402
|
+
* `prune()` is permissionless and idempotent — if the chain is no longer prunable by send time the
|
|
1403
|
+
* bundle simulation usually drops the entry; on a node without `eth_simulateV1` the bundle is sent
|
|
1404
|
+
* as-is and the prune reverts `Rollup__NothingToPrune` inside `aggregate3(allowFailure: true)`
|
|
1405
|
+
* (a failed action, never a whole-tx revert). Used by the failed-sync fallback so a stuck pending
|
|
1406
|
+
* chain (e.g. bad data blocking sync) can be wound back to recover.
|
|
1407
|
+
* @returns true if a prune request was enqueued, false otherwise.
|
|
1408
|
+
*/ async enqueuePruneIfPrunable(slotNumber) {
|
|
1409
|
+
if (this.lastActions['prune'] === slotNumber) {
|
|
1410
|
+
this.log.debug(`Skipping duplicate prune for slot ${slotNumber}`, {
|
|
1411
|
+
slotNumber
|
|
1412
|
+
});
|
|
1413
|
+
return false;
|
|
1414
|
+
}
|
|
1415
|
+
// Use the SAME timestamp the bundle simulator overrides block.timestamp with at send time
|
|
1416
|
+
// (sequencer-bundle-simulator.ts) so this upfront check and the send-time sim agree. Slot-start
|
|
1417
|
+
// and last-L1-slot both fall within the same L2 slot (and epoch, which is what `canPruneAtTime`
|
|
1418
|
+
// derives), so they agree today; matching the simulator keeps it robust if the contract ever uses
|
|
1419
|
+
// the timestamp more granularly.
|
|
1420
|
+
const ts = getLastL1SlotTimestampForL2Slot(slotNumber, this.epochCache.getL1Constants());
|
|
1421
|
+
const canPrune = await this.rollupContract.canPruneAtTime(ts).catch((err)=>{
|
|
1422
|
+
this.log.error(`Failed to check canPruneAtTime for slot ${slotNumber}`, err, {
|
|
1423
|
+
slotNumber
|
|
1424
|
+
});
|
|
1425
|
+
return false;
|
|
1426
|
+
});
|
|
1427
|
+
if (!canPrune) {
|
|
1428
|
+
this.log.debug(`Rollup not prunable at slot ${slotNumber}`, {
|
|
1429
|
+
slotNumber
|
|
1430
|
+
});
|
|
1431
|
+
return false;
|
|
1432
|
+
}
|
|
1433
|
+
const request = {
|
|
1434
|
+
to: this.rollupContract.address,
|
|
1435
|
+
data: encodeFunctionData({
|
|
1436
|
+
abi: RollupAbi,
|
|
1437
|
+
functionName: 'prune',
|
|
1438
|
+
args: []
|
|
1439
|
+
})
|
|
1440
|
+
};
|
|
1441
|
+
this.log.info(`Enqueuing rollup prune for slot ${slotNumber}`, {
|
|
1442
|
+
slotNumber
|
|
1443
|
+
});
|
|
1444
|
+
return this.enqueueRequest('prune', request, {
|
|
1445
|
+
address: this.rollupContract.address,
|
|
1446
|
+
abi: RollupAbi,
|
|
1447
|
+
eventName: 'PrunedPending'
|
|
1448
|
+
}, slotNumber);
|
|
1449
|
+
}
|
|
1126
1450
|
/** Enqueues all slashing actions as returned by the slasher client. */ async enqueueSlashingActions(actions, slotNumber, signerAddress, signer) {
|
|
1127
1451
|
if (actions.length === 0) {
|
|
1128
1452
|
this.log.debug(`No slashing actions to enqueue for slot ${slotNumber}`);
|
|
@@ -1341,7 +1665,7 @@ export class SequencerPublisher {
|
|
|
1341
1665
|
data: validateBlobsData
|
|
1342
1666
|
},
|
|
1343
1667
|
blobData: encodedData.blobs.map((b)=>toHex(b.data)),
|
|
1344
|
-
l1BlockNumber
|
|
1668
|
+
l1BlockNumber,
|
|
1345
1669
|
error: {
|
|
1346
1670
|
message: viemError.message,
|
|
1347
1671
|
name: viemError.name
|
|
@@ -1351,7 +1675,11 @@ export class SequencerPublisher {
|
|
|
1351
1675
|
'validate-blobs'
|
|
1352
1676
|
],
|
|
1353
1677
|
sender: this.getSenderAddress().toString()
|
|
1354
|
-
}
|
|
1678
|
+
},
|
|
1679
|
+
timing: this.computeTimingInfo(this.getCurrentL2Slot())
|
|
1680
|
+
}, {
|
|
1681
|
+
captureFeeSummary: true,
|
|
1682
|
+
targetSlot: this.getCurrentL2Slot()
|
|
1355
1683
|
});
|
|
1356
1684
|
throw new Error('Failed to validate blobs');
|
|
1357
1685
|
});
|