@lodestar/beacon-node 1.35.0-dev.c88a6ed255 → 1.35.0-dev.cec9bef1db
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/api/impl/beacon/blocks/index.js +58 -46
- package/lib/api/impl/beacon/blocks/index.js.map +1 -1
- package/lib/api/impl/config/constants.d.ts +5 -0
- package/lib/api/impl/config/constants.js +6 -1
- package/lib/api/impl/config/constants.js.map +1 -1
- package/lib/api/impl/debug/index.js +1 -1
- package/lib/api/impl/debug/index.js.map +1 -1
- package/lib/api/impl/errors.js +2 -0
- package/lib/api/impl/errors.js.map +1 -1
- package/lib/api/impl/index.d.ts +3 -3
- package/lib/api/impl/index.js +3 -3
- package/lib/api/impl/index.js.map +1 -1
- package/lib/api/impl/lodestar/index.js +1 -1
- package/lib/api/impl/lodestar/index.js.map +1 -1
- package/lib/api/impl/validator/index.js +2 -1
- package/lib/api/impl/validator/index.js.map +1 -1
- package/lib/api/rest/activeSockets.js +3 -2
- package/lib/api/rest/activeSockets.js.map +1 -1
- package/lib/api/rest/base.d.ts +1 -1
- package/lib/api/rest/base.js +6 -2
- package/lib/api/rest/base.js.map +1 -1
- package/lib/api/rest/index.js +2 -0
- package/lib/api/rest/index.js.map +1 -1
- package/lib/api/rest/swaggerUI.js +4 -2
- package/lib/api/rest/swaggerUI.js.map +1 -1
- package/lib/bun-wrappers/prometheus-gc-stats.d.ts +2 -0
- package/lib/bun-wrappers/prometheus-gc-stats.js +8 -0
- package/lib/bun-wrappers/prometheus-gc-stats.js.map +1 -0
- package/lib/chain/ColumnReconstructionTracker.d.ts +32 -0
- package/lib/chain/ColumnReconstructionTracker.js +71 -0
- package/lib/chain/ColumnReconstructionTracker.js.map +1 -0
- package/lib/chain/GetBlobsTracker.d.ts +31 -0
- package/lib/chain/GetBlobsTracker.js +82 -0
- package/lib/chain/GetBlobsTracker.js.map +1 -0
- package/lib/chain/archiveStore/archiveStore.js +49 -38
- package/lib/chain/archiveStore/archiveStore.js.map +1 -1
- package/lib/chain/archiveStore/historicalState/historicalStateRegen.js +3 -0
- package/lib/chain/archiveStore/historicalState/historicalStateRegen.js.map +1 -1
- package/lib/chain/archiveStore/index.d.ts +1 -1
- package/lib/chain/archiveStore/index.js +1 -1
- package/lib/chain/archiveStore/strategies/frequencyStateArchiveStrategy.js +5 -0
- package/lib/chain/archiveStore/strategies/frequencyStateArchiveStrategy.js.map +1 -1
- package/lib/chain/balancesCache.js +1 -3
- package/lib/chain/balancesCache.js.map +1 -1
- package/lib/chain/beaconProposerCache.js +1 -0
- package/lib/chain/beaconProposerCache.js.map +1 -1
- package/lib/chain/blocks/blockInput/blockInput.d.ts +19 -7
- package/lib/chain/blocks/blockInput/blockInput.js +132 -88
- package/lib/chain/blocks/blockInput/blockInput.js.map +1 -1
- package/lib/chain/blocks/blockInput/index.d.ts +1 -1
- package/lib/chain/blocks/blockInput/index.js +1 -1
- package/lib/chain/blocks/blockInput/types.d.ts +19 -10
- package/lib/chain/blocks/blockInput/types.js +1 -0
- package/lib/chain/blocks/blockInput/types.js.map +1 -1
- package/lib/chain/blocks/blockInput/utils.d.ts +0 -4
- package/lib/chain/blocks/blockInput/utils.js +6 -29
- package/lib/chain/blocks/blockInput/utils.js.map +1 -1
- package/lib/chain/blocks/importBlock.js +16 -12
- package/lib/chain/blocks/importBlock.js.map +1 -1
- package/lib/chain/blocks/index.d.ts +6 -5
- package/lib/chain/blocks/index.js +5 -4
- package/lib/chain/blocks/index.js.map +1 -1
- package/lib/chain/blocks/types.d.ts +3 -106
- package/lib/chain/blocks/types.js +0 -119
- package/lib/chain/blocks/types.js.map +1 -1
- package/lib/chain/blocks/utils/chainSegment.d.ts +2 -2
- package/lib/chain/blocks/utils/chainSegment.js +2 -2
- package/lib/chain/blocks/utils/chainSegment.js.map +1 -1
- package/lib/chain/blocks/verifyBlock.d.ts +3 -3
- package/lib/chain/blocks/verifyBlock.js +15 -14
- package/lib/chain/blocks/verifyBlock.js.map +1 -1
- package/lib/chain/blocks/verifyBlocksDataAvailability.d.ts +7 -22
- package/lib/chain/blocks/verifyBlocksDataAvailability.js +18 -110
- package/lib/chain/blocks/verifyBlocksDataAvailability.js.map +1 -1
- package/lib/chain/blocks/verifyBlocksExecutionPayloads.d.ts +4 -3
- package/lib/chain/blocks/verifyBlocksExecutionPayloads.js +24 -22
- package/lib/chain/blocks/verifyBlocksExecutionPayloads.js.map +1 -1
- package/lib/chain/blocks/verifyBlocksSanityChecks.d.ts +4 -3
- package/lib/chain/blocks/verifyBlocksSanityChecks.js +2 -2
- package/lib/chain/blocks/verifyBlocksSanityChecks.js.map +1 -1
- package/lib/chain/blocks/verifyBlocksStateTransitionOnly.d.ts +3 -2
- package/lib/chain/blocks/verifyBlocksStateTransitionOnly.js +2 -2
- package/lib/chain/blocks/verifyBlocksStateTransitionOnly.js.map +1 -1
- package/lib/chain/blocks/writeBlockInputToDb.d.ts +3 -3
- package/lib/chain/blocks/writeBlockInputToDb.js +61 -66
- package/lib/chain/blocks/writeBlockInputToDb.js.map +1 -1
- package/lib/chain/bls/multithread/index.js +168 -163
- package/lib/chain/bls/multithread/index.js.map +1 -1
- package/lib/chain/bls/singleThread.js +1 -0
- package/lib/chain/bls/singleThread.js.map +1 -1
- package/lib/chain/chain.d.ts +13 -11
- package/lib/chain/chain.js +79 -20
- package/lib/chain/chain.js.map +1 -1
- package/lib/chain/emitter.d.ts +43 -2
- package/lib/chain/emitter.js +18 -0
- package/lib/chain/emitter.js.map +1 -1
- package/lib/chain/errors/blobSidecarError.d.ts +24 -0
- package/lib/chain/errors/blobSidecarError.js +10 -0
- package/lib/chain/errors/blobSidecarError.js.map +1 -1
- package/lib/chain/errors/blockError.js +1 -0
- package/lib/chain/errors/blockError.js.map +1 -1
- package/lib/chain/errors/dataColumnSidecarError.d.ts +42 -0
- package/lib/chain/errors/dataColumnSidecarError.js +14 -0
- package/lib/chain/errors/dataColumnSidecarError.js.map +1 -1
- package/lib/chain/errors/gossipValidation.js +1 -0
- package/lib/chain/errors/gossipValidation.js.map +1 -1
- package/lib/chain/errors/index.d.ts +2 -2
- package/lib/chain/errors/index.js +2 -2
- package/lib/chain/errors/index.js.map +1 -1
- package/lib/chain/forkChoice/index.js.map +1 -1
- package/lib/chain/genesis/genesis.js +16 -5
- package/lib/chain/genesis/genesis.js.map +1 -1
- package/lib/chain/index.d.ts +2 -2
- package/lib/chain/index.js +2 -2
- package/lib/chain/index.js.map +1 -1
- package/lib/chain/interface.d.ts +12 -9
- package/lib/chain/interface.js.map +1 -1
- package/lib/chain/lightClient/index.js +16 -9
- package/lib/chain/lightClient/index.js.map +1 -1
- package/lib/chain/lightClient/proofs.js.map +1 -1
- package/lib/chain/opPools/aggregatedAttestationPool.js +14 -9
- package/lib/chain/opPools/aggregatedAttestationPool.js.map +1 -1
- package/lib/chain/opPools/attestationPool.js +8 -3
- package/lib/chain/opPools/attestationPool.js.map +1 -1
- package/lib/chain/opPools/index.d.ts +1 -1
- package/lib/chain/opPools/index.js +1 -1
- package/lib/chain/opPools/index.js.map +1 -1
- package/lib/chain/opPools/opPool.js +10 -12
- package/lib/chain/opPools/opPool.js.map +1 -1
- package/lib/chain/opPools/syncCommitteeMessagePool.js +9 -6
- package/lib/chain/opPools/syncCommitteeMessagePool.js.map +1 -1
- package/lib/chain/opPools/syncContributionAndProofPool.js +5 -2
- package/lib/chain/opPools/syncContributionAndProofPool.js.map +1 -1
- package/lib/chain/options.js +1 -1
- package/lib/chain/options.js.map +1 -1
- package/lib/chain/prepareNextSlot.js +141 -136
- package/lib/chain/prepareNextSlot.js.map +1 -1
- package/lib/chain/produceBlock/produceBlockBody.d.ts +1 -1
- package/lib/chain/produceBlock/produceBlockBody.js +3 -10
- package/lib/chain/produceBlock/produceBlockBody.js.map +1 -1
- package/lib/chain/regen/errors.js +1 -0
- package/lib/chain/regen/errors.js.map +1 -1
- package/lib/chain/regen/index.d.ts +1 -1
- package/lib/chain/regen/index.js +1 -1
- package/lib/chain/regen/index.js.map +1 -1
- package/lib/chain/regen/queued.js +35 -28
- package/lib/chain/regen/queued.js.map +1 -1
- package/lib/chain/regen/regen.js +1 -0
- package/lib/chain/regen/regen.js.map +1 -1
- package/lib/chain/reprocess.js +3 -1
- package/lib/chain/reprocess.js.map +1 -1
- package/lib/chain/seenCache/index.d.ts +1 -1
- package/lib/chain/seenCache/index.js +1 -1
- package/lib/chain/seenCache/index.js.map +1 -1
- package/lib/chain/seenCache/seenAggregateAndProof.js +7 -6
- package/lib/chain/seenCache/seenAggregateAndProof.js.map +1 -1
- package/lib/chain/seenCache/seenAttestationData.js +5 -2
- package/lib/chain/seenCache/seenAttestationData.js.map +1 -1
- package/lib/chain/seenCache/seenAttesters.js +2 -4
- package/lib/chain/seenCache/seenAttesters.js.map +1 -1
- package/lib/chain/seenCache/seenBlockProposers.js +2 -4
- package/lib/chain/seenCache/seenBlockProposers.js.map +1 -1
- package/lib/chain/seenCache/seenCommittee.js +1 -3
- package/lib/chain/seenCache/seenCommittee.js.map +1 -1
- package/lib/chain/seenCache/seenCommitteeContribution.js +3 -2
- package/lib/chain/seenCache/seenCommitteeContribution.js.map +1 -1
- package/lib/chain/seenCache/seenGossipBlockInput.d.ts +74 -78
- package/lib/chain/seenCache/seenGossipBlockInput.js +235 -369
- package/lib/chain/seenCache/seenGossipBlockInput.js.map +1 -1
- package/lib/chain/shufflingCache.js +5 -2
- package/lib/chain/shufflingCache.js.map +1 -1
- package/lib/chain/stateCache/blockStateCacheImpl.js +13 -7
- package/lib/chain/stateCache/blockStateCacheImpl.js.map +1 -1
- package/lib/chain/stateCache/datastore/db.js +1 -0
- package/lib/chain/stateCache/datastore/db.js.map +1 -1
- package/lib/chain/stateCache/datastore/file.js +1 -0
- package/lib/chain/stateCache/datastore/file.js.map +1 -1
- package/lib/chain/stateCache/datastore/index.d.ts +1 -1
- package/lib/chain/stateCache/datastore/index.js +1 -1
- package/lib/chain/stateCache/datastore/index.js.map +1 -1
- package/lib/chain/stateCache/fifoBlockStateCache.js +10 -0
- package/lib/chain/stateCache/fifoBlockStateCache.js.map +1 -1
- package/lib/chain/stateCache/inMemoryCheckpointsCache.js +10 -4
- package/lib/chain/stateCache/inMemoryCheckpointsCache.js.map +1 -1
- package/lib/chain/stateCache/index.d.ts +1 -1
- package/lib/chain/stateCache/index.js +1 -1
- package/lib/chain/stateCache/index.js.map +1 -1
- package/lib/chain/stateCache/mapMetrics.js +4 -4
- package/lib/chain/stateCache/mapMetrics.js.map +1 -1
- package/lib/chain/stateCache/persistentCheckpointsCache.js +14 -6
- package/lib/chain/stateCache/persistentCheckpointsCache.js.map +1 -1
- package/lib/chain/validation/blobSidecar.d.ts +8 -3
- package/lib/chain/validation/blobSidecar.js +73 -31
- package/lib/chain/validation/blobSidecar.js.map +1 -1
- package/lib/chain/validation/dataColumnSidecar.d.ts +7 -9
- package/lib/chain/validation/dataColumnSidecar.js +95 -57
- package/lib/chain/validation/dataColumnSidecar.js.map +1 -1
- package/lib/chain/validation/index.d.ts +1 -1
- package/lib/chain/validation/index.js +1 -1
- package/lib/chain/validation/index.js.map +1 -1
- package/lib/chain/validation/signatureSets/aggregateAndProof.d.ts +1 -2
- package/lib/chain/validation/signatureSets/aggregateAndProof.js.map +1 -1
- package/lib/chain/validatorMonitor.d.ts +1 -3
- package/lib/chain/validatorMonitor.js +2 -1
- package/lib/chain/validatorMonitor.js.map +1 -1
- package/lib/db/beacon.js +24 -0
- package/lib/db/beacon.js.map +1 -1
- package/lib/db/index.d.ts +1 -1
- package/lib/db/index.js.map +1 -1
- package/lib/db/repositories/blobSidecars.js.map +1 -1
- package/lib/db/repositories/blockArchive.js +2 -3
- package/lib/db/repositories/blockArchive.js.map +1 -1
- package/lib/db/repositories/dataColumnSidecar.js +1 -1
- package/lib/db/repositories/dataColumnSidecar.js.map +1 -1
- package/lib/db/repositories/dataColumnSidecarArchive.js +1 -1
- package/lib/db/repositories/dataColumnSidecarArchive.js.map +1 -1
- package/lib/db/repositories/depositDataRoot.js +1 -0
- package/lib/db/repositories/depositDataRoot.js.map +1 -1
- package/lib/db/repositories/index.d.ts +10 -10
- package/lib/db/repositories/index.js +9 -9
- package/lib/db/repositories/index.js.map +1 -1
- package/lib/db/single/preGenesisState.js +6 -0
- package/lib/db/single/preGenesisState.js.map +1 -1
- package/lib/db/single/preGenesisStateLastProcessedBlock.js +5 -0
- package/lib/db/single/preGenesisStateLastProcessedBlock.js.map +1 -1
- package/lib/eth1/eth1DataCache.js +2 -0
- package/lib/eth1/eth1DataCache.js.map +1 -1
- package/lib/eth1/eth1DepositDataTracker.js +18 -5
- package/lib/eth1/eth1DepositDataTracker.js.map +1 -1
- package/lib/eth1/eth1DepositsCache.js +3 -0
- package/lib/eth1/eth1DepositsCache.js.map +1 -1
- package/lib/eth1/eth1MergeBlockTracker.js +10 -4
- package/lib/eth1/eth1MergeBlockTracker.js.map +1 -1
- package/lib/eth1/index.js +2 -0
- package/lib/eth1/index.js.map +1 -1
- package/lib/eth1/provider/eth1Provider.d.ts +1 -2
- package/lib/eth1/provider/eth1Provider.js +6 -2
- package/lib/eth1/provider/eth1Provider.js.map +1 -1
- package/lib/eth1/provider/jsonRpcHttpClient.d.ts +1 -1
- package/lib/eth1/provider/jsonRpcHttpClient.js +16 -2
- package/lib/eth1/provider/jsonRpcHttpClient.js.map +1 -1
- package/lib/eth1/provider/jwt.js.map +1 -1
- package/lib/eth1/provider/utils.d.ts +5 -0
- package/lib/eth1/provider/utils.js +9 -1
- package/lib/eth1/provider/utils.js.map +1 -1
- package/lib/eth1/utils/deposits.js.map +1 -1
- package/lib/execution/builder/cache.js +6 -0
- package/lib/execution/builder/cache.js.map +1 -1
- package/lib/execution/builder/http.js +14 -8
- package/lib/execution/builder/http.js.map +1 -1
- package/lib/execution/builder/index.js.map +1 -1
- package/lib/execution/engine/disabled.js +2 -4
- package/lib/execution/engine/disabled.js.map +1 -1
- package/lib/execution/engine/http.d.ts +2 -2
- package/lib/execution/engine/http.js +43 -11
- package/lib/execution/engine/http.js.map +1 -1
- package/lib/execution/engine/interface.d.ts +2 -2
- package/lib/execution/engine/interface.js.map +1 -1
- package/lib/execution/engine/mock.d.ts +1 -0
- package/lib/execution/engine/mock.js +16 -12
- package/lib/execution/engine/mock.js.map +1 -1
- package/lib/execution/engine/payloadIdCache.js +1 -3
- package/lib/execution/engine/payloadIdCache.js.map +1 -1
- package/lib/execution/engine/types.d.ts +5 -1
- package/lib/execution/engine/types.js +37 -8
- package/lib/execution/engine/types.js.map +1 -1
- package/lib/execution/engine/utils.js +3 -3
- package/lib/execution/engine/utils.js.map +1 -1
- package/lib/execution/index.d.ts +2 -2
- package/lib/execution/index.js +2 -2
- package/lib/execution/index.js.map +1 -1
- package/lib/index.d.ts +6 -6
- package/lib/index.js +5 -5
- package/lib/index.js.map +1 -1
- package/lib/metrics/index.d.ts +1 -1
- package/lib/metrics/index.js +1 -1
- package/lib/metrics/index.js.map +1 -1
- package/lib/metrics/metrics/beacon.d.ts +11 -5
- package/lib/metrics/metrics/beacon.js +40 -20
- package/lib/metrics/metrics/beacon.js.map +1 -1
- package/lib/metrics/metrics/lodestar.d.ts +55 -13
- package/lib/metrics/metrics/lodestar.js +84 -7
- package/lib/metrics/metrics/lodestar.js.map +1 -1
- package/lib/metrics/metrics.d.ts +1 -1
- package/lib/metrics/metrics.js.map +1 -1
- package/lib/metrics/nodeJsMetrics.js +1 -1
- package/lib/metrics/nodeJsMetrics.js.map +1 -1
- package/lib/metrics/server/http.d.ts +1 -1
- package/lib/metrics/utils/avgMinMax.d.ts +1 -1
- package/lib/metrics/utils/avgMinMax.js +10 -6
- package/lib/metrics/utils/avgMinMax.js.map +1 -1
- package/lib/metrics/utils/gauge.d.ts +1 -1
- package/lib/metrics/utils/gauge.js +1 -4
- package/lib/metrics/utils/gauge.js.map +1 -1
- package/lib/metrics/utils/registryMetricCreator.d.ts +1 -1
- package/lib/metrics/utils/registryMetricCreator.js.map +1 -1
- package/lib/monitoring/properties.js +4 -0
- package/lib/monitoring/properties.js.map +1 -1
- package/lib/monitoring/service.js +13 -1
- package/lib/monitoring/service.js.map +1 -1
- package/lib/monitoring/system.js +25 -27
- package/lib/monitoring/system.js.map +1 -1
- package/lib/network/core/networkCore.d.ts +2 -2
- package/lib/network/core/networkCore.js +67 -50
- package/lib/network/core/networkCore.js.map +1 -1
- package/lib/network/core/networkCoreWorker.js +1 -1
- package/lib/network/core/networkCoreWorker.js.map +1 -1
- package/lib/network/core/networkCoreWorkerHandler.d.ts +2 -2
- package/lib/network/core/networkCoreWorkerHandler.js +14 -5
- package/lib/network/core/networkCoreWorkerHandler.js.map +1 -1
- package/lib/network/discv5/index.d.ts +2 -2
- package/lib/network/discv5/index.js +9 -3
- package/lib/network/discv5/index.js.map +1 -1
- package/lib/network/discv5/worker.js +3 -3
- package/lib/network/discv5/worker.js.map +1 -1
- package/lib/network/events.d.ts +2 -18
- package/lib/network/events.js +0 -7
- package/lib/network/events.js.map +1 -1
- package/lib/network/gossip/encoding.d.ts +1 -1
- package/lib/network/gossip/encoding.js +4 -2
- package/lib/network/gossip/encoding.js.map +1 -1
- package/lib/network/gossip/errors.js +1 -0
- package/lib/network/gossip/errors.js.map +1 -1
- package/lib/network/gossip/gossipsub.d.ts +1 -1
- package/lib/network/gossip/gossipsub.js +8 -1
- package/lib/network/gossip/gossipsub.js.map +1 -1
- package/lib/network/gossip/index.d.ts +2 -2
- package/lib/network/gossip/index.js +2 -2
- package/lib/network/gossip/index.js.map +1 -1
- package/lib/network/gossip/interface.d.ts +2 -2
- package/lib/network/gossip/topic.d.ts +125 -105
- package/lib/network/gossip/topic.js +2 -1
- package/lib/network/gossip/topic.js.map +1 -1
- package/lib/network/index.d.ts +3 -3
- package/lib/network/index.js +3 -3
- package/lib/network/index.js.map +1 -1
- package/lib/network/interface.d.ts +3 -2
- package/lib/network/libp2p/index.js +11 -3
- package/lib/network/libp2p/index.js.map +1 -1
- package/lib/network/metadata.js +4 -0
- package/lib/network/metadata.js.map +1 -1
- package/lib/network/network.d.ts +4 -2
- package/lib/network/network.js +89 -71
- package/lib/network/network.js.map +1 -1
- package/lib/network/options.js +1 -1
- package/lib/network/options.js.map +1 -1
- package/lib/network/peers/datastore.d.ts +2 -2
- package/lib/network/peers/datastore.js +10 -4
- package/lib/network/peers/datastore.js.map +1 -1
- package/lib/network/peers/discover.js +75 -66
- package/lib/network/peers/discover.js.map +1 -1
- package/lib/network/peers/peerManager.js +154 -130
- package/lib/network/peers/peerManager.js.map +1 -1
- package/lib/network/peers/peersData.d.ts +1 -1
- package/lib/network/peers/peersData.js +1 -3
- package/lib/network/peers/peersData.js.map +1 -1
- package/lib/network/peers/score/index.d.ts +1 -1
- package/lib/network/peers/score/index.js +1 -1
- package/lib/network/peers/score/index.js.map +1 -1
- package/lib/network/peers/score/score.js +6 -0
- package/lib/network/peers/score/score.js.map +1 -1
- package/lib/network/peers/score/store.js +3 -0
- package/lib/network/peers/score/store.js.map +1 -1
- package/lib/network/peers/utils/prioritizePeers.js.map +1 -1
- package/lib/network/peers/utils/subnetMap.js +2 -4
- package/lib/network/peers/utils/subnetMap.js.map +1 -1
- package/lib/network/processor/aggregatorTracker.js +1 -3
- package/lib/network/processor/aggregatorTracker.js.map +1 -1
- package/lib/network/processor/gossipHandlers.js +145 -164
- package/lib/network/processor/gossipHandlers.js.map +1 -1
- package/lib/network/processor/gossipQueues/indexed.js +11 -9
- package/lib/network/processor/gossipQueues/indexed.js.map +1 -1
- package/lib/network/processor/gossipQueues/linear.js +9 -8
- package/lib/network/processor/gossipQueues/linear.js.map +1 -1
- package/lib/network/processor/index.d.ts +2 -1
- package/lib/network/processor/index.js +22 -8
- package/lib/network/processor/index.js.map +1 -1
- package/lib/network/reqresp/ReqRespBeaconNode.d.ts +1 -1
- package/lib/network/reqresp/ReqRespBeaconNode.js +11 -2
- package/lib/network/reqresp/ReqRespBeaconNode.js.map +1 -1
- package/lib/network/reqresp/handlers/dataColumnSidecarsByRoot.js.map +1 -1
- package/lib/network/reqresp/index.d.ts +1 -3
- package/lib/network/reqresp/index.js +1 -3
- package/lib/network/reqresp/index.js.map +1 -1
- package/lib/network/reqresp/rateLimit.js +11 -5
- package/lib/network/reqresp/rateLimit.js.map +1 -1
- package/lib/network/reqresp/utils/dataColumnResponseValidation.js.map +1 -1
- package/lib/network/statusCache.js +1 -0
- package/lib/network/statusCache.js.map +1 -1
- package/lib/network/subnets/attnetsService.js +73 -65
- package/lib/network/subnets/attnetsService.js.map +1 -1
- package/lib/network/subnets/interface.js +1 -1
- package/lib/network/subnets/interface.js.map +1 -1
- package/lib/network/subnets/syncnetsService.js +29 -22
- package/lib/network/subnets/syncnetsService.js.map +1 -1
- package/lib/network/subnets/util.js +1 -2
- package/lib/network/subnets/util.js.map +1 -1
- package/lib/node/nodejs.d.ts +1 -1
- package/lib/node/nodejs.js +15 -0
- package/lib/node/nodejs.js.map +1 -1
- package/lib/node/notifier.js +1 -2
- package/lib/node/notifier.js.map +1 -1
- package/lib/node/options.js.map +1 -1
- package/lib/node/utils/interop/state.js +1 -2
- package/lib/node/utils/interop/state.js.map +1 -1
- package/lib/sync/backfill/backfill.d.ts +1 -1
- package/lib/sync/backfill/backfill.js +58 -16
- package/lib/sync/backfill/backfill.js.map +1 -1
- package/lib/sync/constants.d.ts +2 -1
- package/lib/sync/constants.js +2 -1
- package/lib/sync/constants.js.map +1 -1
- package/lib/sync/interface.d.ts +1 -59
- package/lib/sync/interface.js +0 -20
- package/lib/sync/interface.js.map +1 -1
- package/lib/sync/options.d.ts +1 -1
- package/lib/sync/range/batch.d.ts +48 -32
- package/lib/sync/range/batch.js +201 -55
- package/lib/sync/range/batch.js.map +1 -1
- package/lib/sync/range/chain.d.ts +15 -10
- package/lib/sync/range/chain.js +141 -70
- package/lib/sync/range/chain.js.map +1 -1
- package/lib/sync/range/range.d.ts +3 -3
- package/lib/sync/range/range.js +86 -54
- package/lib/sync/range/range.js.map +1 -1
- package/lib/sync/range/utils/hashBlocks.d.ts +2 -2
- package/lib/sync/range/utils/hashBlocks.js +6 -4
- package/lib/sync/range/utils/hashBlocks.js.map +1 -1
- package/lib/sync/range/utils/peerBalancer.js +18 -13
- package/lib/sync/range/utils/peerBalancer.js.map +1 -1
- package/lib/sync/sync.d.ts +1 -2
- package/lib/sync/sync.js +95 -87
- package/lib/sync/sync.js.map +1 -1
- package/lib/sync/types.d.ts +44 -0
- package/lib/sync/types.js +34 -0
- package/lib/sync/types.js.map +1 -0
- package/lib/sync/unknownBlock.d.ts +42 -28
- package/lib/sync/unknownBlock.js +404 -458
- package/lib/sync/unknownBlock.js.map +1 -1
- package/lib/sync/utils/downloadByRange.d.ts +186 -0
- package/lib/sync/utils/downloadByRange.js +457 -0
- package/lib/sync/utils/downloadByRange.js.map +1 -0
- package/lib/sync/utils/downloadByRoot.d.ts +119 -0
- package/lib/sync/utils/downloadByRoot.js +319 -0
- package/lib/sync/utils/downloadByRoot.js.map +1 -0
- package/lib/sync/utils/pendingBlocksTree.d.ts +14 -6
- package/lib/sync/utils/pendingBlocksTree.js +24 -18
- package/lib/sync/utils/pendingBlocksTree.js.map +1 -1
- package/lib/util/array.js +7 -3
- package/lib/util/array.js.map +1 -1
- package/lib/util/asyncIterableToEvents.js +6 -3
- package/lib/util/asyncIterableToEvents.js.map +1 -1
- package/lib/util/binarySearch.js +2 -0
- package/lib/util/binarySearch.js.map +1 -1
- package/lib/util/blobs.d.ts +2 -2
- package/lib/util/blobs.js +4 -4
- package/lib/util/blobs.js.map +1 -1
- package/lib/util/bufferPool.js +7 -2
- package/lib/util/bufferPool.js.map +1 -1
- package/lib/util/clock.d.ts +1 -1
- package/lib/util/clock.js +23 -18
- package/lib/util/clock.js.map +1 -1
- package/lib/util/dataColumns.d.ts +13 -10
- package/lib/util/dataColumns.js +92 -134
- package/lib/util/dataColumns.js.map +1 -1
- package/lib/util/execution.d.ts +20 -0
- package/lib/util/execution.js +148 -0
- package/lib/util/execution.js.map +1 -0
- package/lib/util/itTrigger.js +4 -4
- package/lib/util/itTrigger.js.map +1 -1
- package/lib/util/map.js +2 -0
- package/lib/util/map.js.map +1 -1
- package/lib/util/queue/index.d.ts +1 -1
- package/lib/util/queue/index.js +1 -1
- package/lib/util/queue/index.js.map +1 -1
- package/lib/util/queue/itemQueue.js +52 -49
- package/lib/util/queue/itemQueue.js.map +1 -1
- package/lib/util/serializedCache.js +1 -3
- package/lib/util/serializedCache.js.map +1 -1
- package/lib/util/set.js +2 -0
- package/lib/util/set.js.map +1 -1
- package/lib/util/sszBytes.d.ts +1 -0
- package/lib/util/sszBytes.js +1 -1
- package/lib/util/sszBytes.js.map +1 -1
- package/lib/util/timeSeries.js +3 -1
- package/lib/util/timeSeries.js.map +1 -1
- package/lib/util/types.d.ts +1 -1
- package/lib/util/types.js +2 -2
- package/lib/util/types.js.map +1 -1
- package/lib/util/wrapError.d.ts +7 -0
- package/package.json +36 -18
- package/lib/chain/seenCache/seenBlockInput.d.ts +0 -84
- package/lib/chain/seenCache/seenBlockInput.js +0 -225
- package/lib/chain/seenCache/seenBlockInput.js.map +0 -1
- package/lib/network/reqresp/beaconBlocksMaybeBlobsByRange.d.ts +0 -28
- package/lib/network/reqresp/beaconBlocksMaybeBlobsByRange.js +0 -328
- package/lib/network/reqresp/beaconBlocksMaybeBlobsByRange.js.map +0 -1
- package/lib/network/reqresp/beaconBlocksMaybeBlobsByRoot.d.ts +0 -49
- package/lib/network/reqresp/beaconBlocksMaybeBlobsByRoot.js +0 -499
- package/lib/network/reqresp/beaconBlocksMaybeBlobsByRoot.js.map +0 -1
|
@@ -3,7 +3,8 @@ import { Logger } from "@lodestar/utils";
|
|
|
3
3
|
import { Metrics } from "../../metrics/index.js";
|
|
4
4
|
import { BlockProcessOpts } from "../options.js";
|
|
5
5
|
import { ValidatorMonitor } from "../validatorMonitor.js";
|
|
6
|
-
import {
|
|
6
|
+
import { IBlockInput } from "./blockInput/index.js";
|
|
7
|
+
import { ImportBlockOpts } from "./types.js";
|
|
7
8
|
/**
|
|
8
9
|
* Verifies 1 or more blocks are fully valid running the full state transition; from a linear sequence of blocks.
|
|
9
10
|
*
|
|
@@ -12,7 +13,7 @@ import { BlockInput, ImportBlockOpts } from "./types.js";
|
|
|
12
13
|
* - STFN - per_block_processing()
|
|
13
14
|
* - Check state root matches
|
|
14
15
|
*/
|
|
15
|
-
export declare function verifyBlocksStateTransitionOnly(preState0: CachedBeaconStateAllForks, blocks:
|
|
16
|
+
export declare function verifyBlocksStateTransitionOnly(preState0: CachedBeaconStateAllForks, blocks: IBlockInput[], dataAvailabilityStatuses: DataAvailabilityStatus[], logger: Logger, metrics: Metrics | null, validatorMonitor: ValidatorMonitor | null, signal: AbortSignal, opts: BlockProcessOpts & ImportBlockOpts): Promise<{
|
|
16
17
|
postStates: CachedBeaconStateAllForks[];
|
|
17
18
|
proposerBalanceDeltas: number[];
|
|
18
19
|
verifyStateTime: number;
|
|
@@ -17,7 +17,7 @@ export async function verifyBlocksStateTransitionOnly(preState0, blocks, dataAva
|
|
|
17
17
|
const recvToValLatency = Date.now() / 1000 - (opts.seenTimestampSec ?? Date.now() / 1000);
|
|
18
18
|
for (let i = 0; i < blocks.length; i++) {
|
|
19
19
|
const { validProposerSignature, validSignatures } = opts;
|
|
20
|
-
const
|
|
20
|
+
const block = blocks[i].getBlock();
|
|
21
21
|
const preState = i === 0 ? preState0 : postStates[i - 1];
|
|
22
22
|
const dataAvailabilityStatus = dataAvailabilityStatuses[i];
|
|
23
23
|
// STFN - per_slot_processing() + per_block_processing()
|
|
@@ -65,7 +65,7 @@ export async function verifyBlocksStateTransitionOnly(preState0, blocks, dataAva
|
|
|
65
65
|
}
|
|
66
66
|
const verifyStateTime = Date.now();
|
|
67
67
|
if (blocks.length === 1 && opts.seenTimestampSec !== undefined) {
|
|
68
|
-
const slot = blocks[0].
|
|
68
|
+
const slot = blocks[0].getBlock().message.slot;
|
|
69
69
|
const recvToValidation = verifyStateTime / 1000 - opts.seenTimestampSec;
|
|
70
70
|
const validationTime = recvToValidation - recvToValLatency;
|
|
71
71
|
metrics?.gossipBlock.stateTransition.recvToValidation.observe(recvToValidation);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"verifyBlocksStateTransitionOnly.js","sourceRoot":"","sources":["../../../src/chain/blocks/verifyBlocksStateTransitionOnly.ts"],"names":[],"mappings":"AAAA,OAAO,EAGL,sBAAsB,EACtB,uBAAuB,EACvB,eAAe,GAChB,MAAM,4BAA4B,CAAC;AACpC,OAAO,EAAC,YAAY,EAAS,MAAM,iBAAiB,CAAC;AAErD,OAAO,EAAC,eAAe,EAAC,MAAM,qBAAqB,CAAC;AACpD,OAAO,EAAC,aAAa,EAAC,MAAM,yBAAyB,CAAC;AACtD,OAAO,EAAC,UAAU,EAAE,cAAc,EAAC,MAAM,oBAAoB,CAAC;
|
|
1
|
+
{"version":3,"file":"verifyBlocksStateTransitionOnly.js","sourceRoot":"","sources":["../../../src/chain/blocks/verifyBlocksStateTransitionOnly.ts"],"names":[],"mappings":"AAAA,OAAO,EAGL,sBAAsB,EACtB,uBAAuB,EACvB,eAAe,GAChB,MAAM,4BAA4B,CAAC;AACpC,OAAO,EAAC,YAAY,EAAS,MAAM,iBAAiB,CAAC;AAErD,OAAO,EAAC,eAAe,EAAC,MAAM,qBAAqB,CAAC;AACpD,OAAO,EAAC,aAAa,EAAC,MAAM,yBAAyB,CAAC;AACtD,OAAO,EAAC,UAAU,EAAE,cAAc,EAAC,MAAM,oBAAoB,CAAC;AAM9D;;;;;;;GAOG;AACH,MAAM,CAAC,KAAK,UAAU,+BAA+B,CACnD,SAAoC,EACpC,MAAqB,EACrB,wBAAkD,EAClD,MAAc,EACd,OAAuB,EACvB,gBAAyC,EACzC,MAAmB,EACnB,IAAwC;IAExC,MAAM,UAAU,GAAgC,EAAE,CAAC;IACnD,MAAM,qBAAqB,GAAa,EAAE,CAAC;IAC3C,MAAM,gBAAgB,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,gBAAgB,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC;IAE1F,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACvC,MAAM,EAAC,sBAAsB,EAAE,eAAe,EAAC,GAAG,IAAI,CAAC;QACvD,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC;QACnC,MAAM,QAAQ,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QACzD,MAAM,sBAAsB,GAAG,wBAAwB,CAAC,CAAC,CAAC,CAAC;QAE3D,wDAAwD;QACxD,qGAAqG;QACrG,MAAM,iBAAiB,GAAG,CAAC,IAAI,EAAE,qBAAqB,CAAC;QACvD,MAAM,SAAS,GAAG,eAAe,CAC/B,QAAQ,EACR,KAAK,EACL;YACE,mFAAmF;YACnF,2EAA2E;YAC3E,sBAAsB,EAAE,sBAAsB,CAAC,KAAK;YACpD,sBAAsB;YACtB,6DAA6D;YAC7D,eAAe,EAAE,KAAK;YACtB,4DAA4D;YAC5D,cAAc,EAAE,CAAC,iBAAiB,IAAI,CAAC,eAAe,IAAI,CAAC,sBAAsB;YACjF,gBAAgB,EAAE,CAAC,iBAAiB,IAAI,CAAC,eAAe;SACzD,EACD,EAAC,OAAO,EAAE,gBAAgB,EAAC,CAC5B,CAAC;QAEF,MAAM,iBAAiB,GAAG,OAAO,EAAE,qBAAqB,CAAC,UAAU,CAAC;YAClE,MAAM,EAAE,uBAAuB,CAAC,eAAe;SAChD,CAAC,CAAC;QACH,MAAM,SAAS,GAAG,SAAS,CAAC,YAAY,EAAE,CAAC;QAC3C,iBAAiB,EAAE,EAAE,CAAC;QAEtB,2BAA2B;QAC3B,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC,OAAO,CAAC,SAAS,EAAE,SAAS,CAAC,EAAE,CAAC;YACzD,MAAM,IAAI,UAAU,CAAC,KAAK,EAAE;gBAC1B,IAAI,EAAE,cAAc,CAAC,kBAAkB;gBACvC,IAAI,EAAE,SAAS,CAAC,YAAY,EAAE;gBAC9B,YAAY,EAAE,KAAK,CAAC,OAAO,CAAC,SAAS;gBACrC,QAAQ;gBACR,SAAS;aACV,CAAC,CAAC;QACL,CAAC;QAED,UAAU,CAAC,CAAC,CAAC,GAAG,SAAS,CAAC;QAE1B,iCAAiC;QACjC,MAAM,aAAa,GAAG,KAAK,CAAC,OAAO,CAAC,aAAa,CAAC;QAClD,qBAAqB,CAAC,CAAC,CAAC,GAAG,SAAS,CAAC,QAAQ,CAAC,GAAG,CAAC,aAAa,CAAC,GAAG,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC;QAExG,2FAA2F;QAC3F,wDAAwD;QACxD,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;YACnB,MAAM,IAAI,YAAY,CAAC,gCAAgC,CAAC,CAAC;QAC3D,CAAC;QAED,sDAAsD;QACtD,IAAI,CAAC,GAAG,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC1B,MAAM,aAAa,EAAE,CAAC;QACxB,CAAC;IACH,CAAC;IAED,MAAM,eAAe,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACnC,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,IAAI,IAAI,CAAC,gBAAgB,KAAK,SAAS,EAAE,CAAC;QAC/D,MAAM,IAAI,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC;QAC/C,MAAM,gBAAgB,GAAG,eAAe,GAAG,IAAI,GAAG,IAAI,CAAC,gBAAgB,CAAC;QACxE,MAAM,cAAc,GAAG,gBAAgB,GAAG,gBAAgB,CAAC;QAE3D,OAAO,EAAE,WAAW,CAAC,eAAe,CAAC,gBAAgB,CAAC,OAAO,CAAC,gBAAgB,CAAC,CAAC;QAChF,OAAO,EAAE,WAAW,CAAC,eAAe,CAAC,cAAc,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;QAE5E,MAAM,CAAC,KAAK,CAAC,iCAAiC,EAAE,EAAC,IAAI,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,cAAc,EAAC,CAAC,CAAC;IAC9G,CAAC;IAED,OAAO,EAAC,UAAU,EAAE,qBAAqB,EAAE,eAAe,EAAC,CAAC;AAC9D,CAAC"}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { BeaconChain } from "../chain.js";
|
|
2
|
-
import {
|
|
2
|
+
import { IBlockInput } from "./blockInput/index.js";
|
|
3
3
|
/**
|
|
4
4
|
* Persists block input data to DB. This operation must be eventually completed if a block is imported to the fork-choice.
|
|
5
5
|
* Else the node will be in an inconsistent state that can lead to being stuck.
|
|
@@ -7,9 +7,9 @@ import { BlockInput } from "./types.js";
|
|
|
7
7
|
* This operation may be performed before, during or after importing to the fork-choice. As long as errors
|
|
8
8
|
* are handled properly for eventual consistency.
|
|
9
9
|
*/
|
|
10
|
-
export declare function writeBlockInputToDb(this: BeaconChain,
|
|
10
|
+
export declare function writeBlockInputToDb(this: BeaconChain, blocksInputs: IBlockInput[]): Promise<void>;
|
|
11
11
|
/**
|
|
12
12
|
* Prunes eagerly persisted block inputs only if not known to the fork-choice
|
|
13
13
|
*/
|
|
14
|
-
export declare function removeEagerlyPersistedBlockInputs(this: BeaconChain, blockInputs:
|
|
14
|
+
export declare function removeEagerlyPersistedBlockInputs(this: BeaconChain, blockInputs: IBlockInput[]): Promise<void>;
|
|
15
15
|
//# sourceMappingURL=writeBlockInputToDb.d.ts.map
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
1
|
+
import { prettyPrintIndices, toRootHex } from "@lodestar/utils";
|
|
2
|
+
import { isBlockInputBlobs, isBlockInputColumns } from "./blockInput/index.js";
|
|
3
|
+
import { BLOB_AVAILABILITY_TIMEOUT } from "./verifyBlocksDataAvailability.js";
|
|
4
4
|
/**
|
|
5
5
|
* Persists block input data to DB. This operation must be eventually completed if a block is imported to the fork-choice.
|
|
6
6
|
* Else the node will be in an inconsistent state that can lead to being stuck.
|
|
@@ -8,10 +8,14 @@ import { BlockInputType } from "./types.js";
|
|
|
8
8
|
* This operation may be performed before, during or after importing to the fork-choice. As long as errors
|
|
9
9
|
* are handled properly for eventual consistency.
|
|
10
10
|
*/
|
|
11
|
-
export async function writeBlockInputToDb(
|
|
11
|
+
export async function writeBlockInputToDb(blocksInputs) {
|
|
12
12
|
const fnPromises = [];
|
|
13
|
-
|
|
14
|
-
|
|
13
|
+
// track slots for logging
|
|
14
|
+
const slots = [];
|
|
15
|
+
for (const blockInput of blocksInputs) {
|
|
16
|
+
const block = blockInput.getBlock();
|
|
17
|
+
const slot = block.message.slot;
|
|
18
|
+
slots.push(slot);
|
|
15
19
|
const blockRoot = this.config.getForkTypes(block.message.slot).BeaconBlock.hashTreeRoot(block.message);
|
|
16
20
|
const blockRootHex = toRootHex(blockRoot);
|
|
17
21
|
const blockBytes = this.serializedCache.get(block);
|
|
@@ -29,53 +33,48 @@ export async function writeBlockInputToDb(blocksInput) {
|
|
|
29
33
|
root: blockRootHex,
|
|
30
34
|
inputType: blockInput.type,
|
|
31
35
|
});
|
|
32
|
-
if (blockInput.
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
else {
|
|
46
|
-
dataColumnsLen = custodyColumns.length;
|
|
47
|
-
}
|
|
48
|
-
const blockDataColumns = blockData.dataColumns;
|
|
49
|
-
const dataColumnSidecars = blockDataColumns.filter((dataColumnSidecar) => custodyColumns.includes(dataColumnSidecar.index));
|
|
50
|
-
if (dataColumnSidecars.length !== dataColumnsLen) {
|
|
51
|
-
throw Error(`Invalid dataColumnSidecars=${dataColumnSidecars.length} for custody expected custodyColumnsLen=${dataColumnsLen}`);
|
|
52
|
-
}
|
|
53
|
-
fnPromises.push(this.db.dataColumnSidecar.putMany(blockRoot, dataColumnSidecars));
|
|
54
|
-
this.logger.debug("Persisted dataColumnSidecars to hot DB", {
|
|
55
|
-
slot: block.message.slot,
|
|
56
|
-
root: blockRootHex,
|
|
57
|
-
blockDataColumns: blockDataColumns.length,
|
|
58
|
-
dataColumnSidecars: dataColumnSidecars.length,
|
|
59
|
-
numBlobs: blobsLen,
|
|
60
|
-
custodyColumns: custodyColumns.length,
|
|
61
|
-
});
|
|
36
|
+
if (!blockInput.hasAllData()) {
|
|
37
|
+
await blockInput.waitForAllData(BLOB_AVAILABILITY_TIMEOUT);
|
|
38
|
+
}
|
|
39
|
+
// NOTE: Old data is pruned on archive
|
|
40
|
+
if (isBlockInputColumns(blockInput)) {
|
|
41
|
+
const { custodyColumns } = this.custodyConfig;
|
|
42
|
+
const blobsLen = block.message.body.blobKzgCommitments.length;
|
|
43
|
+
let dataColumnsLen;
|
|
44
|
+
if (blobsLen === 0) {
|
|
45
|
+
dataColumnsLen = 0;
|
|
46
|
+
}
|
|
47
|
+
else {
|
|
48
|
+
dataColumnsLen = custodyColumns.length;
|
|
62
49
|
}
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
this.logger.debug("Persisted blobSidecars to hot DB", {
|
|
67
|
-
blobsLen: blobSidecars.length,
|
|
68
|
-
slot: block.message.slot,
|
|
69
|
-
root: blockRootHex,
|
|
70
|
-
});
|
|
50
|
+
const dataColumnSidecars = blockInput.getCustodyColumns();
|
|
51
|
+
if (dataColumnSidecars.length !== dataColumnsLen) {
|
|
52
|
+
this.logger.debug(`Invalid dataColumnSidecars=${dataColumnSidecars.length} for custody expected custodyColumnsLen=${dataColumnsLen}`);
|
|
71
53
|
}
|
|
54
|
+
fnPromises.push(this.db.dataColumnSidecar.putMany(blockRoot, dataColumnSidecars));
|
|
55
|
+
this.logger.debug("Persisted dataColumnSidecars to hot DB", {
|
|
56
|
+
slot: block.message.slot,
|
|
57
|
+
root: blockRootHex,
|
|
58
|
+
dataColumnSidecars: dataColumnSidecars.length,
|
|
59
|
+
numBlobs: blobsLen,
|
|
60
|
+
custodyColumns: custodyColumns.length,
|
|
61
|
+
});
|
|
72
62
|
}
|
|
63
|
+
else if (isBlockInputBlobs(blockInput)) {
|
|
64
|
+
const blobSidecars = blockInput.getBlobs();
|
|
65
|
+
fnPromises.push(this.db.blobSidecars.add({ blockRoot, slot: block.message.slot, blobSidecars }));
|
|
66
|
+
this.logger.debug("Persisted blobSidecars to hot DB", {
|
|
67
|
+
blobsLen: blobSidecars.length,
|
|
68
|
+
slot: block.message.slot,
|
|
69
|
+
root: blockRootHex,
|
|
70
|
+
});
|
|
71
|
+
}
|
|
72
|
+
await Promise.all(fnPromises);
|
|
73
|
+
this.logger.debug("Persisted blocksInput to db", {
|
|
74
|
+
blocksInput: blocksInputs.length,
|
|
75
|
+
slots: prettyPrintIndices(slots),
|
|
76
|
+
});
|
|
73
77
|
}
|
|
74
|
-
await Promise.all(fnPromises);
|
|
75
|
-
this.logger.debug("Persisted blocksInput to db", {
|
|
76
|
-
blocksInput: blocksInput.length,
|
|
77
|
-
slots: prettyPrintIndices(blocksInput.map((blockInput) => blockInput.block.message.slot)),
|
|
78
|
-
});
|
|
79
78
|
}
|
|
80
79
|
/**
|
|
81
80
|
* Prunes eagerly persisted block inputs only if not known to the fork-choice
|
|
@@ -85,28 +84,24 @@ export async function removeEagerlyPersistedBlockInputs(blockInputs) {
|
|
|
85
84
|
const blobsToRemove = [];
|
|
86
85
|
const dataColumnsToRemove = [];
|
|
87
86
|
for (const blockInput of blockInputs) {
|
|
88
|
-
const
|
|
87
|
+
const block = blockInput.getBlock();
|
|
89
88
|
const slot = block.message.slot;
|
|
90
89
|
const blockRoot = this.config.getForkTypes(slot).BeaconBlock.hashTreeRoot(block.message);
|
|
91
|
-
const blockRootHex =
|
|
90
|
+
const blockRootHex = toRootHex(blockRoot);
|
|
92
91
|
if (!this.forkChoice.hasBlockHex(blockRootHex)) {
|
|
93
92
|
blockToRemove.push(block);
|
|
94
|
-
if (
|
|
95
|
-
const {
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
else {
|
|
101
|
-
const { custodyConfig } = this;
|
|
102
|
-
const { custodyColumns } = custodyConfig;
|
|
103
|
-
const dataColumnsLen = custodyColumns.length;
|
|
104
|
-
const dataColumnSidecars = blockData.dataColumns.filter((dataColumnSidecar) => custodyColumns.includes(dataColumnSidecar.index));
|
|
105
|
-
if (dataColumnSidecars.length !== dataColumnsLen) {
|
|
106
|
-
throw Error(`Invalid dataColumnSidecars=${dataColumnSidecars.length} for custody expected custodyColumnsLen=${dataColumnsLen}`);
|
|
107
|
-
}
|
|
108
|
-
dataColumnsToRemove.push(blockRoot);
|
|
93
|
+
if (isBlockInputColumns(blockInput)) {
|
|
94
|
+
const { custodyColumns } = this.custodyConfig;
|
|
95
|
+
const dataColumnsLen = custodyColumns.length;
|
|
96
|
+
const dataColumnSidecars = blockInput.getCustodyColumns();
|
|
97
|
+
if (dataColumnSidecars.length !== dataColumnsLen) {
|
|
98
|
+
throw Error(`Invalid dataColumnSidecars=${dataColumnSidecars.length} for custody expected custodyColumnsLen=${dataColumnsLen}`);
|
|
109
99
|
}
|
|
100
|
+
dataColumnsToRemove.push(blockRoot);
|
|
101
|
+
}
|
|
102
|
+
else if (isBlockInputBlobs(blockInput)) {
|
|
103
|
+
const blobSidecars = blockInput.getBlobs();
|
|
104
|
+
blobsToRemove.push({ blockRoot, slot, blobSidecars });
|
|
110
105
|
}
|
|
111
106
|
}
|
|
112
107
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"writeBlockInputToDb.js","sourceRoot":"","sources":["../../../src/chain/blocks/writeBlockInputToDb.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"writeBlockInputToDb.js","sourceRoot":"","sources":["../../../src/chain/blocks/writeBlockInputToDb.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,kBAAkB,EAAE,SAAS,EAAC,MAAM,iBAAiB,CAAC;AAE9D,OAAO,EAAc,iBAAiB,EAAE,mBAAmB,EAAC,MAAM,uBAAuB,CAAC;AAC1F,OAAO,EAAC,yBAAyB,EAAC,MAAM,mCAAmC,CAAC;AAE5E;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,mBAAmB,CAAoB,YAA2B;IACtF,MAAM,UAAU,GAAoB,EAAE,CAAC;IACvC,0BAA0B;IAC1B,MAAM,KAAK,GAAa,EAAE,CAAC;IAE3B,KAAK,MAAM,UAAU,IAAI,YAAY,EAAE,CAAC;QACtC,MAAM,KAAK,GAAG,UAAU,CAAC,QAAQ,EAAE,CAAC;QACpC,MAAM,IAAI,GAAG,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC;QAChC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACjB,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,WAAW,CAAC,YAAY,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QACvG,MAAM,YAAY,GAAG,SAAS,CAAC,SAAS,CAAC,CAAC;QAC1C,MAAM,UAAU,GAAG,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QACnD,IAAI,UAAU,EAAE,CAAC;YACf,8CAA8C;YAC9C,IAAI,CAAC,OAAO,EAAE,WAAW,CAAC,mCAAmC,CAAC,GAAG,EAAE,CAAC;YACpE,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,KAAK,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,UAAU,CAAC,CAAC,CAAC;QACnF,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,OAAO,EAAE,WAAW,CAAC,iCAAiC,CAAC,GAAG,EAAE,CAAC;YAClE,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC;QAC5C,CAAC;QAED,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,yBAAyB,EAAE;YAC3C,IAAI,EAAE,KAAK,CAAC,OAAO,CAAC,IAAI;YACxB,IAAI,EAAE,YAAY;YAClB,SAAS,EAAE,UAAU,CAAC,IAAI;SAC3B,CAAC,CAAC;QAEH,IAAI,CAAC,UAAU,CAAC,UAAU,EAAE,EAAE,CAAC;YAC7B,MAAM,UAAU,CAAC,cAAc,CAAC,yBAAyB,CAAC,CAAC;QAC7D,CAAC;QAED,sCAAsC;QACtC,IAAI,mBAAmB,CAAC,UAAU,CAAC,EAAE,CAAC;YACpC,MAAM,EAAC,cAAc,EAAC,GAAG,IAAI,CAAC,aAAa,CAAC;YAC5C,MAAM,QAAQ,GAAI,KAAK,CAAC,OAA4B,CAAC,IAAI,CAAC,kBAAkB,CAAC,MAAM,CAAC;YACpF,IAAI,cAAsB,CAAC;YAC3B,IAAI,QAAQ,KAAK,CAAC,EAAE,CAAC;gBACnB,cAAc,GAAG,CAAC,CAAC;YACrB,CAAC;iBAAM,CAAC;gBACN,cAAc,GAAG,cAAc,CAAC,MAAM,CAAC;YACzC,CAAC;YAED,MAAM,kBAAkB,GAAG,UAAU,CAAC,iBAAiB,EAAE,CAAC;YAC1D,IAAI,kBAAkB,CAAC,MAAM,KAAK,cAAc,EAAE,CAAC;gBACjD,IAAI,CAAC,MAAM,CAAC,KAAK,CACf,8BAA8B,kBAAkB,CAAC,MAAM,2CAA2C,cAAc,EAAE,CACnH,CAAC;YACJ,CAAC;YAED,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,iBAAiB,CAAC,OAAO,CAAC,SAAS,EAAE,kBAAkB,CAAC,CAAC,CAAC;YAClF,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,wCAAwC,EAAE;gBAC1D,IAAI,EAAE,KAAK,CAAC,OAAO,CAAC,IAAI;gBACxB,IAAI,EAAE,YAAY;gBAClB,kBAAkB,EAAE,kBAAkB,CAAC,MAAM;gBAC7C,QAAQ,EAAE,QAAQ;gBAClB,cAAc,EAAE,cAAc,CAAC,MAAM;aACtC,CAAC,CAAC;QACL,CAAC;aAAM,IAAI,iBAAiB,CAAC,UAAU,CAAC,EAAE,CAAC;YACzC,MAAM,YAAY,GAAG,UAAU,CAAC,QAAQ,EAAE,CAAC;YAC3C,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,YAAY,CAAC,GAAG,CAAC,EAAC,SAAS,EAAE,IAAI,EAAE,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,YAAY,EAAC,CAAC,CAAC,CAAC;YAC/F,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,kCAAkC,EAAE;gBACpD,QAAQ,EAAE,YAAY,CAAC,MAAM;gBAC7B,IAAI,EAAE,KAAK,CAAC,OAAO,CAAC,IAAI;gBACxB,IAAI,EAAE,YAAY;aACnB,CAAC,CAAC;QACL,CAAC;QAED,MAAM,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QAC9B,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,6BAA6B,EAAE;YAC/C,WAAW,EAAE,YAAY,CAAC,MAAM;YAChC,KAAK,EAAE,kBAAkB,CAAC,KAAK,CAAC;SACjC,CAAC,CAAC;IACL,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,iCAAiC,CAAoB,WAA0B;IACnG,MAAM,aAAa,GAAG,EAAE,CAAC;IACzB,MAAM,aAAa,GAAG,EAAE,CAAC;IACzB,MAAM,mBAAmB,GAAG,EAAE,CAAC;IAE/B,KAAK,MAAM,UAAU,IAAI,WAAW,EAAE,CAAC;QACrC,MAAM,KAAK,GAAG,UAAU,CAAC,QAAQ,EAAE,CAAC;QACpC,MAAM,IAAI,GAAG,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC;QAChC,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,WAAW,CAAC,YAAY,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QACzF,MAAM,YAAY,GAAG,SAAS,CAAC,SAAS,CAAC,CAAC;QAC1C,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC,YAAY,CAAC,EAAE,CAAC;YAC/C,aAAa,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAE1B,IAAI,mBAAmB,CAAC,UAAU,CAAC,EAAE,CAAC;gBACpC,MAAM,EAAC,cAAc,EAAC,GAAG,IAAI,CAAC,aAAa,CAAC;gBAC5C,MAAM,cAAc,GAAG,cAAc,CAAC,MAAM,CAAC;gBAC7C,MAAM,kBAAkB,GAAG,UAAU,CAAC,iBAAiB,EAAE,CAAC;gBAC1D,IAAI,kBAAkB,CAAC,MAAM,KAAK,cAAc,EAAE,CAAC;oBACjD,MAAM,KAAK,CACT,8BAA8B,kBAAkB,CAAC,MAAM,2CAA2C,cAAc,EAAE,CACnH,CAAC;gBACJ,CAAC;gBACD,mBAAmB,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YACtC,CAAC;iBAAM,IAAI,iBAAiB,CAAC,UAAU,CAAC,EAAE,CAAC;gBACzC,MAAM,YAAY,GAAG,UAAU,CAAC,QAAQ,EAAE,CAAC;gBAC3C,aAAa,CAAC,IAAI,CAAC,EAAC,SAAS,EAAE,IAAI,EAAE,YAAY,EAAC,CAAC,CAAC;YACtD,CAAC;QACH,CAAC;IACH,CAAC;IAED,MAAM,OAAO,CAAC,GAAG,CAAC;QAChB,uEAAuE;QACvE,IAAI,CAAC,EAAE,CAAC,KAAK,CAAC,WAAW,CAAC,aAAa,CAAC;QACxC,IAAI,CAAC,EAAE,CAAC,YAAY,CAAC,WAAW,CAAC,aAAa,CAAC;QAC/C,IAAI,CAAC,EAAE,CAAC,iBAAiB,CAAC,UAAU,CAAC,mBAAmB,CAAC;KAC1D,CAAC,CAAC;AACL,CAAC"}
|
|
@@ -64,170 +64,15 @@ var WorkerStatusCode;
|
|
|
64
64
|
* sets into packages of work and send at once to a worker to distribute the latency cost
|
|
65
65
|
*/
|
|
66
66
|
export class BlsMultiThreadWorkerPool {
|
|
67
|
+
logger;
|
|
68
|
+
metrics;
|
|
69
|
+
workers;
|
|
70
|
+
jobs = new LinkedList();
|
|
71
|
+
bufferedJobs = null;
|
|
72
|
+
blsVerifyAllMultiThread;
|
|
73
|
+
closed = false;
|
|
74
|
+
workersBusy = 0;
|
|
67
75
|
constructor(options, modules) {
|
|
68
|
-
this.jobs = new LinkedList();
|
|
69
|
-
this.bufferedJobs = null;
|
|
70
|
-
this.closed = false;
|
|
71
|
-
this.workersBusy = 0;
|
|
72
|
-
/**
|
|
73
|
-
* Potentially submit jobs to an idle worker, only if there's a worker and jobs
|
|
74
|
-
*/
|
|
75
|
-
this.runJob = async () => {
|
|
76
|
-
if (this.closed) {
|
|
77
|
-
return;
|
|
78
|
-
}
|
|
79
|
-
// Find idle worker
|
|
80
|
-
const worker = this.workers.find((worker) => worker.status.code === WorkerStatusCode.idle);
|
|
81
|
-
if (!worker || worker.status.code !== WorkerStatusCode.idle) {
|
|
82
|
-
return;
|
|
83
|
-
}
|
|
84
|
-
// Prepare work package
|
|
85
|
-
const jobsInput = this.prepareWork();
|
|
86
|
-
if (jobsInput.length === 0) {
|
|
87
|
-
return;
|
|
88
|
-
}
|
|
89
|
-
// TODO: After sending the work to the worker the main thread can drop the job arguments
|
|
90
|
-
// and free-up memory, only needs to keep the job's Promise handlers.
|
|
91
|
-
// Maybe it's not useful since all data referenced in jobs is likely referenced by others
|
|
92
|
-
const workerApi = worker.status.workerApi;
|
|
93
|
-
worker.status = { code: WorkerStatusCode.running, workerApi };
|
|
94
|
-
this.workersBusy++;
|
|
95
|
-
try {
|
|
96
|
-
let startedJobsDefault = 0;
|
|
97
|
-
let startedJobsSameMessage = 0;
|
|
98
|
-
let startedSetsDefault = 0;
|
|
99
|
-
let startedSetsSameMessage = 0;
|
|
100
|
-
const workReqs = [];
|
|
101
|
-
const jobsStarted = [];
|
|
102
|
-
for (const job of jobsInput) {
|
|
103
|
-
this.metrics?.blsThreadPool.jobWaitTime.observe((Date.now() - job.addedTimeMs) / 1000);
|
|
104
|
-
let workReq;
|
|
105
|
-
try {
|
|
106
|
-
// Note: This can throw, must be handled per-job.
|
|
107
|
-
// Pubkey and signature aggregation is defered here
|
|
108
|
-
workReq = await jobItemWorkReq(job, this.metrics);
|
|
109
|
-
}
|
|
110
|
-
catch (e) {
|
|
111
|
-
this.metrics?.blsThreadPool.errorAggregateSignatureSetsCount.inc({ type: job.type });
|
|
112
|
-
switch (job.type) {
|
|
113
|
-
case JobQueueItemType.default:
|
|
114
|
-
job.reject(e);
|
|
115
|
-
break;
|
|
116
|
-
case JobQueueItemType.sameMessage:
|
|
117
|
-
// there could be an invalid pubkey/signature, retry each individually
|
|
118
|
-
this.retryJobItemSameMessage(job);
|
|
119
|
-
break;
|
|
120
|
-
}
|
|
121
|
-
continue;
|
|
122
|
-
}
|
|
123
|
-
// Re-push all jobs with matching workReq for easier accounting of results
|
|
124
|
-
workReqs.push(workReq);
|
|
125
|
-
jobsStarted.push(job);
|
|
126
|
-
if (job.type === JobQueueItemType.sameMessage) {
|
|
127
|
-
startedJobsSameMessage += 1;
|
|
128
|
-
startedSetsSameMessage += job.sets.length;
|
|
129
|
-
}
|
|
130
|
-
else {
|
|
131
|
-
startedJobsDefault += 1;
|
|
132
|
-
startedSetsDefault += job.sets.length;
|
|
133
|
-
}
|
|
134
|
-
}
|
|
135
|
-
const startedSigSets = startedSetsDefault + startedSetsSameMessage;
|
|
136
|
-
this.metrics?.blsThreadPool.totalJobsGroupsStarted.inc(1);
|
|
137
|
-
this.metrics?.blsThreadPool.totalJobsStarted.inc({ type: JobQueueItemType.default }, startedJobsDefault);
|
|
138
|
-
this.metrics?.blsThreadPool.totalJobsStarted.inc({ type: JobQueueItemType.sameMessage }, startedJobsSameMessage);
|
|
139
|
-
this.metrics?.blsThreadPool.totalSigSetsStarted.inc({ type: JobQueueItemType.default }, startedSetsDefault);
|
|
140
|
-
this.metrics?.blsThreadPool.totalSigSetsStarted.inc({ type: JobQueueItemType.sameMessage }, startedSetsSameMessage);
|
|
141
|
-
// Send work package to the worker
|
|
142
|
-
// If the job, metrics or any code below throws: the job will reject never going stale.
|
|
143
|
-
// Only downside is the job promise may be resolved twice, but that's not an issue
|
|
144
|
-
const [jobStartSec, jobStartNs] = process.hrtime();
|
|
145
|
-
const workResult = await workerApi.verifyManySignatureSets(workReqs);
|
|
146
|
-
const [jobEndSec, jobEndNs] = process.hrtime();
|
|
147
|
-
const { workerId, batchRetries, batchSigsSuccess, workerStartTime, workerEndTime, results } = workResult;
|
|
148
|
-
const [workerStartSec, workerStartNs] = workerStartTime;
|
|
149
|
-
const [workerEndSec, workerEndNs] = workerEndTime;
|
|
150
|
-
let successCount = 0;
|
|
151
|
-
let errorCount = 0;
|
|
152
|
-
// Un-wrap work package
|
|
153
|
-
for (let i = 0; i < jobsStarted.length; i++) {
|
|
154
|
-
const job = jobsStarted[i];
|
|
155
|
-
const jobResult = results[i];
|
|
156
|
-
const sigSetCount = jobItemSigSets(job);
|
|
157
|
-
// TODO: enable exhaustive switch case checks lint rule
|
|
158
|
-
switch (job.type) {
|
|
159
|
-
case JobQueueItemType.default:
|
|
160
|
-
if (!jobResult || jobResult.code !== WorkResultCode.success) {
|
|
161
|
-
job.reject(getJobResultError(jobResult, i));
|
|
162
|
-
errorCount += sigSetCount;
|
|
163
|
-
}
|
|
164
|
-
else {
|
|
165
|
-
job.resolve(jobResult.result);
|
|
166
|
-
successCount += sigSetCount;
|
|
167
|
-
}
|
|
168
|
-
break;
|
|
169
|
-
// handle result of the verification of aggregated signature against aggregated pubkeys
|
|
170
|
-
case JobQueueItemType.sameMessage:
|
|
171
|
-
if (!jobResult || jobResult.code !== WorkResultCode.success) {
|
|
172
|
-
job.reject(getJobResultError(jobResult, i));
|
|
173
|
-
errorCount += 1;
|
|
174
|
-
}
|
|
175
|
-
else {
|
|
176
|
-
if (jobResult.result) {
|
|
177
|
-
// All are valid, most of the time it goes here
|
|
178
|
-
job.resolve(job.sets.map(() => true));
|
|
179
|
-
}
|
|
180
|
-
else {
|
|
181
|
-
// Retry each individually
|
|
182
|
-
this.retryJobItemSameMessage(job);
|
|
183
|
-
}
|
|
184
|
-
successCount += 1;
|
|
185
|
-
}
|
|
186
|
-
break;
|
|
187
|
-
}
|
|
188
|
-
}
|
|
189
|
-
const workerJobTimeSec = workerEndSec - workerStartSec + (workerEndNs - workerStartNs) / 1e9;
|
|
190
|
-
const latencyToWorkerSec = workerStartSec - jobStartSec + (workerStartNs - jobStartNs) / 1e9;
|
|
191
|
-
const latencyFromWorkerSec = jobEndSec - workerEndSec + Number(jobEndNs - workerEndNs) / 1e9;
|
|
192
|
-
this.metrics?.blsThreadPool.timePerSigSet.observe(workerJobTimeSec / startedSigSets);
|
|
193
|
-
this.metrics?.blsThreadPool.jobsWorkerTime.inc({ workerId }, workerJobTimeSec);
|
|
194
|
-
this.metrics?.blsThreadPool.latencyToWorker.observe(latencyToWorkerSec);
|
|
195
|
-
this.metrics?.blsThreadPool.latencyFromWorker.observe(latencyFromWorkerSec);
|
|
196
|
-
this.metrics?.blsThreadPool.successJobsSignatureSetsCount.inc(successCount);
|
|
197
|
-
this.metrics?.blsThreadPool.errorJobsSignatureSetsCount.inc(errorCount);
|
|
198
|
-
this.metrics?.blsThreadPool.batchRetries.inc(batchRetries);
|
|
199
|
-
this.metrics?.blsThreadPool.batchSigsSuccess.inc(batchSigsSuccess);
|
|
200
|
-
}
|
|
201
|
-
catch (e) {
|
|
202
|
-
// Worker communications should never reject
|
|
203
|
-
if (!this.closed) {
|
|
204
|
-
this.logger.error("BlsMultiThreadWorkerPool error", {}, e);
|
|
205
|
-
}
|
|
206
|
-
// Reject all
|
|
207
|
-
for (const job of jobsInput) {
|
|
208
|
-
job.reject(e);
|
|
209
|
-
}
|
|
210
|
-
}
|
|
211
|
-
worker.status = { code: WorkerStatusCode.idle, workerApi };
|
|
212
|
-
this.workersBusy--;
|
|
213
|
-
// Potentially run a new job
|
|
214
|
-
callInNextEventLoop(this.runJob);
|
|
215
|
-
};
|
|
216
|
-
/**
|
|
217
|
-
* Add all buffered jobs to the job queue and potentially run them immediately
|
|
218
|
-
*/
|
|
219
|
-
this.runBufferedJobs = () => {
|
|
220
|
-
if (this.bufferedJobs) {
|
|
221
|
-
for (const job of this.bufferedJobs.jobs) {
|
|
222
|
-
this.jobs.push(job);
|
|
223
|
-
}
|
|
224
|
-
for (const job of this.bufferedJobs.prioritizedJobs) {
|
|
225
|
-
this.jobs.unshift(job);
|
|
226
|
-
}
|
|
227
|
-
this.bufferedJobs = null;
|
|
228
|
-
callInNextEventLoop(this.runJob);
|
|
229
|
-
}
|
|
230
|
-
};
|
|
231
76
|
const { logger, metrics } = modules;
|
|
232
77
|
this.logger = logger;
|
|
233
78
|
this.metrics = metrics;
|
|
@@ -330,6 +175,7 @@ export class BlsMultiThreadWorkerPool {
|
|
|
330
175
|
for (let i = 0; i < poolSize; i++) {
|
|
331
176
|
const workerData = { workerId: i };
|
|
332
177
|
const worker = new Worker(path.join(workerDir, "worker.js"), {
|
|
178
|
+
suppressTranspileTS: Boolean(globalThis.Bun),
|
|
333
179
|
workerData,
|
|
334
180
|
});
|
|
335
181
|
const workerDescriptor = {
|
|
@@ -405,6 +251,150 @@ export class BlsMultiThreadWorkerPool {
|
|
|
405
251
|
callInNextEventLoop(this.runJob);
|
|
406
252
|
}
|
|
407
253
|
}
|
|
254
|
+
/**
|
|
255
|
+
* Potentially submit jobs to an idle worker, only if there's a worker and jobs
|
|
256
|
+
*/
|
|
257
|
+
runJob = async () => {
|
|
258
|
+
if (this.closed) {
|
|
259
|
+
return;
|
|
260
|
+
}
|
|
261
|
+
// Find idle worker
|
|
262
|
+
const worker = this.workers.find((worker) => worker.status.code === WorkerStatusCode.idle);
|
|
263
|
+
if (!worker || worker.status.code !== WorkerStatusCode.idle) {
|
|
264
|
+
return;
|
|
265
|
+
}
|
|
266
|
+
// Prepare work package
|
|
267
|
+
const jobsInput = this.prepareWork();
|
|
268
|
+
if (jobsInput.length === 0) {
|
|
269
|
+
return;
|
|
270
|
+
}
|
|
271
|
+
// TODO: After sending the work to the worker the main thread can drop the job arguments
|
|
272
|
+
// and free-up memory, only needs to keep the job's Promise handlers.
|
|
273
|
+
// Maybe it's not useful since all data referenced in jobs is likely referenced by others
|
|
274
|
+
const workerApi = worker.status.workerApi;
|
|
275
|
+
worker.status = { code: WorkerStatusCode.running, workerApi };
|
|
276
|
+
this.workersBusy++;
|
|
277
|
+
try {
|
|
278
|
+
let startedJobsDefault = 0;
|
|
279
|
+
let startedJobsSameMessage = 0;
|
|
280
|
+
let startedSetsDefault = 0;
|
|
281
|
+
let startedSetsSameMessage = 0;
|
|
282
|
+
const workReqs = [];
|
|
283
|
+
const jobsStarted = [];
|
|
284
|
+
for (const job of jobsInput) {
|
|
285
|
+
this.metrics?.blsThreadPool.jobWaitTime.observe((Date.now() - job.addedTimeMs) / 1000);
|
|
286
|
+
let workReq;
|
|
287
|
+
try {
|
|
288
|
+
// Note: This can throw, must be handled per-job.
|
|
289
|
+
// Pubkey and signature aggregation is defered here
|
|
290
|
+
workReq = await jobItemWorkReq(job, this.metrics);
|
|
291
|
+
}
|
|
292
|
+
catch (e) {
|
|
293
|
+
this.metrics?.blsThreadPool.errorAggregateSignatureSetsCount.inc({ type: job.type });
|
|
294
|
+
switch (job.type) {
|
|
295
|
+
case JobQueueItemType.default:
|
|
296
|
+
job.reject(e);
|
|
297
|
+
break;
|
|
298
|
+
case JobQueueItemType.sameMessage:
|
|
299
|
+
// there could be an invalid pubkey/signature, retry each individually
|
|
300
|
+
this.retryJobItemSameMessage(job);
|
|
301
|
+
break;
|
|
302
|
+
}
|
|
303
|
+
continue;
|
|
304
|
+
}
|
|
305
|
+
// Re-push all jobs with matching workReq for easier accounting of results
|
|
306
|
+
workReqs.push(workReq);
|
|
307
|
+
jobsStarted.push(job);
|
|
308
|
+
if (job.type === JobQueueItemType.sameMessage) {
|
|
309
|
+
startedJobsSameMessage += 1;
|
|
310
|
+
startedSetsSameMessage += job.sets.length;
|
|
311
|
+
}
|
|
312
|
+
else {
|
|
313
|
+
startedJobsDefault += 1;
|
|
314
|
+
startedSetsDefault += job.sets.length;
|
|
315
|
+
}
|
|
316
|
+
}
|
|
317
|
+
const startedSigSets = startedSetsDefault + startedSetsSameMessage;
|
|
318
|
+
this.metrics?.blsThreadPool.totalJobsGroupsStarted.inc(1);
|
|
319
|
+
this.metrics?.blsThreadPool.totalJobsStarted.inc({ type: JobQueueItemType.default }, startedJobsDefault);
|
|
320
|
+
this.metrics?.blsThreadPool.totalJobsStarted.inc({ type: JobQueueItemType.sameMessage }, startedJobsSameMessage);
|
|
321
|
+
this.metrics?.blsThreadPool.totalSigSetsStarted.inc({ type: JobQueueItemType.default }, startedSetsDefault);
|
|
322
|
+
this.metrics?.blsThreadPool.totalSigSetsStarted.inc({ type: JobQueueItemType.sameMessage }, startedSetsSameMessage);
|
|
323
|
+
// Send work package to the worker
|
|
324
|
+
// If the job, metrics or any code below throws: the job will reject never going stale.
|
|
325
|
+
// Only downside is the job promise may be resolved twice, but that's not an issue
|
|
326
|
+
const [jobStartSec, jobStartNs] = process.hrtime();
|
|
327
|
+
const workResult = await workerApi.verifyManySignatureSets(workReqs);
|
|
328
|
+
const [jobEndSec, jobEndNs] = process.hrtime();
|
|
329
|
+
const { workerId, batchRetries, batchSigsSuccess, workerStartTime, workerEndTime, results } = workResult;
|
|
330
|
+
const [workerStartSec, workerStartNs] = workerStartTime;
|
|
331
|
+
const [workerEndSec, workerEndNs] = workerEndTime;
|
|
332
|
+
let successCount = 0;
|
|
333
|
+
let errorCount = 0;
|
|
334
|
+
// Un-wrap work package
|
|
335
|
+
for (let i = 0; i < jobsStarted.length; i++) {
|
|
336
|
+
const job = jobsStarted[i];
|
|
337
|
+
const jobResult = results[i];
|
|
338
|
+
const sigSetCount = jobItemSigSets(job);
|
|
339
|
+
// TODO: enable exhaustive switch case checks lint rule
|
|
340
|
+
switch (job.type) {
|
|
341
|
+
case JobQueueItemType.default:
|
|
342
|
+
if (!jobResult || jobResult.code !== WorkResultCode.success) {
|
|
343
|
+
job.reject(getJobResultError(jobResult, i));
|
|
344
|
+
errorCount += sigSetCount;
|
|
345
|
+
}
|
|
346
|
+
else {
|
|
347
|
+
job.resolve(jobResult.result);
|
|
348
|
+
successCount += sigSetCount;
|
|
349
|
+
}
|
|
350
|
+
break;
|
|
351
|
+
// handle result of the verification of aggregated signature against aggregated pubkeys
|
|
352
|
+
case JobQueueItemType.sameMessage:
|
|
353
|
+
if (!jobResult || jobResult.code !== WorkResultCode.success) {
|
|
354
|
+
job.reject(getJobResultError(jobResult, i));
|
|
355
|
+
errorCount += 1;
|
|
356
|
+
}
|
|
357
|
+
else {
|
|
358
|
+
if (jobResult.result) {
|
|
359
|
+
// All are valid, most of the time it goes here
|
|
360
|
+
job.resolve(job.sets.map(() => true));
|
|
361
|
+
}
|
|
362
|
+
else {
|
|
363
|
+
// Retry each individually
|
|
364
|
+
this.retryJobItemSameMessage(job);
|
|
365
|
+
}
|
|
366
|
+
successCount += 1;
|
|
367
|
+
}
|
|
368
|
+
break;
|
|
369
|
+
}
|
|
370
|
+
}
|
|
371
|
+
const workerJobTimeSec = workerEndSec - workerStartSec + (workerEndNs - workerStartNs) / 1e9;
|
|
372
|
+
const latencyToWorkerSec = workerStartSec - jobStartSec + (workerStartNs - jobStartNs) / 1e9;
|
|
373
|
+
const latencyFromWorkerSec = jobEndSec - workerEndSec + Number(jobEndNs - workerEndNs) / 1e9;
|
|
374
|
+
this.metrics?.blsThreadPool.timePerSigSet.observe(workerJobTimeSec / startedSigSets);
|
|
375
|
+
this.metrics?.blsThreadPool.jobsWorkerTime.inc({ workerId }, workerJobTimeSec);
|
|
376
|
+
this.metrics?.blsThreadPool.latencyToWorker.observe(latencyToWorkerSec);
|
|
377
|
+
this.metrics?.blsThreadPool.latencyFromWorker.observe(latencyFromWorkerSec);
|
|
378
|
+
this.metrics?.blsThreadPool.successJobsSignatureSetsCount.inc(successCount);
|
|
379
|
+
this.metrics?.blsThreadPool.errorJobsSignatureSetsCount.inc(errorCount);
|
|
380
|
+
this.metrics?.blsThreadPool.batchRetries.inc(batchRetries);
|
|
381
|
+
this.metrics?.blsThreadPool.batchSigsSuccess.inc(batchSigsSuccess);
|
|
382
|
+
}
|
|
383
|
+
catch (e) {
|
|
384
|
+
// Worker communications should never reject
|
|
385
|
+
if (!this.closed) {
|
|
386
|
+
this.logger.error("BlsMultiThreadWorkerPool error", {}, e);
|
|
387
|
+
}
|
|
388
|
+
// Reject all
|
|
389
|
+
for (const job of jobsInput) {
|
|
390
|
+
job.reject(e);
|
|
391
|
+
}
|
|
392
|
+
}
|
|
393
|
+
worker.status = { code: WorkerStatusCode.idle, workerApi };
|
|
394
|
+
this.workersBusy--;
|
|
395
|
+
// Potentially run a new job
|
|
396
|
+
callInNextEventLoop(this.runJob);
|
|
397
|
+
};
|
|
408
398
|
/**
|
|
409
399
|
* Grab pending work up to a max number of signatures
|
|
410
400
|
*/
|
|
@@ -421,6 +411,21 @@ export class BlsMultiThreadWorkerPool {
|
|
|
421
411
|
}
|
|
422
412
|
return jobs;
|
|
423
413
|
}
|
|
414
|
+
/**
|
|
415
|
+
* Add all buffered jobs to the job queue and potentially run them immediately
|
|
416
|
+
*/
|
|
417
|
+
runBufferedJobs = () => {
|
|
418
|
+
if (this.bufferedJobs) {
|
|
419
|
+
for (const job of this.bufferedJobs.jobs) {
|
|
420
|
+
this.jobs.push(job);
|
|
421
|
+
}
|
|
422
|
+
for (const job of this.bufferedJobs.prioritizedJobs) {
|
|
423
|
+
this.jobs.unshift(job);
|
|
424
|
+
}
|
|
425
|
+
this.bufferedJobs = null;
|
|
426
|
+
callInNextEventLoop(this.runJob);
|
|
427
|
+
}
|
|
428
|
+
};
|
|
424
429
|
retryJobItemSameMessage(job) {
|
|
425
430
|
// Create new jobs for each pubkey set, and Promise.all all the results
|
|
426
431
|
for (const j of jobItemSameMessageToMultiSet(job)) {
|