@aztec/archiver 0.0.0-test.1 → 0.0.1-commit.b655e406

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (102) hide show
  1. package/README.md +27 -6
  2. package/dest/archiver/archiver.d.ts +126 -46
  3. package/dest/archiver/archiver.d.ts.map +1 -1
  4. package/dest/archiver/archiver.js +683 -261
  5. package/dest/archiver/archiver_store.d.ts +84 -49
  6. package/dest/archiver/archiver_store.d.ts.map +1 -1
  7. package/dest/archiver/archiver_store_test_suite.d.ts.map +1 -1
  8. package/dest/archiver/archiver_store_test_suite.js +707 -213
  9. package/dest/archiver/config.d.ts +4 -20
  10. package/dest/archiver/config.d.ts.map +1 -1
  11. package/dest/archiver/config.js +16 -12
  12. package/dest/archiver/data_retrieval.d.ts +25 -20
  13. package/dest/archiver/data_retrieval.d.ts.map +1 -1
  14. package/dest/archiver/data_retrieval.js +147 -68
  15. package/dest/archiver/errors.d.ts +8 -0
  16. package/dest/archiver/errors.d.ts.map +1 -1
  17. package/dest/archiver/errors.js +12 -0
  18. package/dest/archiver/index.d.ts +2 -3
  19. package/dest/archiver/index.d.ts.map +1 -1
  20. package/dest/archiver/index.js +1 -2
  21. package/dest/archiver/instrumentation.d.ts +9 -3
  22. package/dest/archiver/instrumentation.d.ts.map +1 -1
  23. package/dest/archiver/instrumentation.js +58 -17
  24. package/dest/archiver/kv_archiver_store/block_store.d.ts +47 -10
  25. package/dest/archiver/kv_archiver_store/block_store.d.ts.map +1 -1
  26. package/dest/archiver/kv_archiver_store/block_store.js +216 -63
  27. package/dest/archiver/kv_archiver_store/contract_class_store.d.ts +2 -2
  28. package/dest/archiver/kv_archiver_store/contract_class_store.d.ts.map +1 -1
  29. package/dest/archiver/kv_archiver_store/contract_class_store.js +12 -18
  30. package/dest/archiver/kv_archiver_store/contract_instance_store.d.ts +10 -7
  31. package/dest/archiver/kv_archiver_store/contract_instance_store.d.ts.map +1 -1
  32. package/dest/archiver/kv_archiver_store/contract_instance_store.js +30 -16
  33. package/dest/archiver/kv_archiver_store/kv_archiver_store.d.ts +49 -34
  34. package/dest/archiver/kv_archiver_store/kv_archiver_store.d.ts.map +1 -1
  35. package/dest/archiver/kv_archiver_store/kv_archiver_store.js +88 -46
  36. package/dest/archiver/kv_archiver_store/log_store.d.ts +1 -1
  37. package/dest/archiver/kv_archiver_store/log_store.d.ts.map +1 -1
  38. package/dest/archiver/kv_archiver_store/log_store.js +18 -46
  39. package/dest/archiver/kv_archiver_store/message_store.d.ts +22 -16
  40. package/dest/archiver/kv_archiver_store/message_store.d.ts.map +1 -1
  41. package/dest/archiver/kv_archiver_store/message_store.js +150 -48
  42. package/dest/archiver/structs/inbox_message.d.ts +15 -0
  43. package/dest/archiver/structs/inbox_message.d.ts.map +1 -0
  44. package/dest/archiver/structs/inbox_message.js +38 -0
  45. package/dest/archiver/structs/published.d.ts +1 -10
  46. package/dest/archiver/structs/published.d.ts.map +1 -1
  47. package/dest/archiver/structs/published.js +1 -1
  48. package/dest/archiver/validation.d.ts +11 -0
  49. package/dest/archiver/validation.d.ts.map +1 -0
  50. package/dest/archiver/validation.js +90 -0
  51. package/dest/factory.d.ts +7 -12
  52. package/dest/factory.d.ts.map +1 -1
  53. package/dest/factory.js +18 -49
  54. package/dest/rpc/index.d.ts +1 -2
  55. package/dest/rpc/index.d.ts.map +1 -1
  56. package/dest/rpc/index.js +1 -4
  57. package/dest/test/mock_archiver.d.ts +1 -1
  58. package/dest/test/mock_l1_to_l2_message_source.d.ts +4 -2
  59. package/dest/test/mock_l1_to_l2_message_source.d.ts.map +1 -1
  60. package/dest/test/mock_l1_to_l2_message_source.js +14 -1
  61. package/dest/test/mock_l2_block_source.d.ts +32 -5
  62. package/dest/test/mock_l2_block_source.d.ts.map +1 -1
  63. package/dest/test/mock_l2_block_source.js +118 -7
  64. package/dest/test/mock_structs.d.ts +9 -0
  65. package/dest/test/mock_structs.d.ts.map +1 -0
  66. package/dest/test/mock_structs.js +37 -0
  67. package/package.json +25 -27
  68. package/src/archiver/archiver.ts +858 -317
  69. package/src/archiver/archiver_store.ts +97 -55
  70. package/src/archiver/archiver_store_test_suite.ts +663 -210
  71. package/src/archiver/config.ts +23 -41
  72. package/src/archiver/data_retrieval.ts +215 -92
  73. package/src/archiver/errors.ts +21 -0
  74. package/src/archiver/index.ts +2 -3
  75. package/src/archiver/instrumentation.ts +75 -20
  76. package/src/archiver/kv_archiver_store/block_store.ts +270 -72
  77. package/src/archiver/kv_archiver_store/contract_class_store.ts +13 -23
  78. package/src/archiver/kv_archiver_store/contract_instance_store.ts +35 -27
  79. package/src/archiver/kv_archiver_store/kv_archiver_store.ts +127 -63
  80. package/src/archiver/kv_archiver_store/log_store.ts +24 -62
  81. package/src/archiver/kv_archiver_store/message_store.ts +209 -53
  82. package/src/archiver/structs/inbox_message.ts +41 -0
  83. package/src/archiver/structs/published.ts +1 -11
  84. package/src/archiver/validation.ts +99 -0
  85. package/src/factory.ts +24 -66
  86. package/src/rpc/index.ts +1 -5
  87. package/src/test/mock_archiver.ts +1 -1
  88. package/src/test/mock_l1_to_l2_message_source.ts +14 -3
  89. package/src/test/mock_l2_block_source.ts +152 -8
  90. package/src/test/mock_structs.ts +49 -0
  91. package/dest/archiver/kv_archiver_store/nullifier_store.d.ts +0 -12
  92. package/dest/archiver/kv_archiver_store/nullifier_store.d.ts.map +0 -1
  93. package/dest/archiver/kv_archiver_store/nullifier_store.js +0 -73
  94. package/dest/archiver/memory_archiver_store/l1_to_l2_message_store.d.ts +0 -23
  95. package/dest/archiver/memory_archiver_store/l1_to_l2_message_store.d.ts.map +0 -1
  96. package/dest/archiver/memory_archiver_store/l1_to_l2_message_store.js +0 -49
  97. package/dest/archiver/memory_archiver_store/memory_archiver_store.d.ts +0 -175
  98. package/dest/archiver/memory_archiver_store/memory_archiver_store.d.ts.map +0 -1
  99. package/dest/archiver/memory_archiver_store/memory_archiver_store.js +0 -636
  100. package/src/archiver/kv_archiver_store/nullifier_store.ts +0 -97
  101. package/src/archiver/memory_archiver_store/l1_to_l2_message_store.ts +0 -61
  102. package/src/archiver/memory_archiver_store/memory_archiver_store.ts +0 -801
@@ -4,25 +4,38 @@ function _ts_decorate(decorators, target, key, desc) {
4
4
  else for(var i = decorators.length - 1; i >= 0; i--)if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
5
5
  return c > 3 && r && Object.defineProperty(target, key, r), r;
6
6
  }
7
- import { createEthereumChain } from '@aztec/ethereum';
7
+ import { EpochCache } from '@aztec/epoch-cache';
8
+ import { BlockTagTooOldError, InboxContract, RollupContract, createEthereumChain } from '@aztec/ethereum';
9
+ import { maxBigint } from '@aztec/foundation/bigint';
10
+ import { Buffer16, Buffer32 } from '@aztec/foundation/buffer';
11
+ import { merge, pick } from '@aztec/foundation/collection';
8
12
  import { Fr } from '@aztec/foundation/fields';
9
13
  import { createLogger } from '@aztec/foundation/log';
14
+ import { promiseWithResolvers } from '@aztec/foundation/promise';
10
15
  import { RunningPromise, makeLoggingErrorHandler } from '@aztec/foundation/running-promise';
16
+ import { sleep } from '@aztec/foundation/sleep';
11
17
  import { count } from '@aztec/foundation/string';
12
- import { elapsed } from '@aztec/foundation/timer';
13
- import { InboxAbi, RollupAbi } from '@aztec/l1-artifacts';
14
- import { ContractClassRegisteredEvent, PrivateFunctionBroadcastedEvent, UnconstrainedFunctionBroadcastedEvent } from '@aztec/protocol-contracts/class-registerer';
15
- import { ContractInstanceDeployedEvent, ContractInstanceUpdatedEvent } from '@aztec/protocol-contracts/instance-deployer';
18
+ import { Timer, elapsed } from '@aztec/foundation/timer';
19
+ import { ContractClassPublishedEvent, PrivateFunctionBroadcastedEvent, UtilityFunctionBroadcastedEvent } from '@aztec/protocol-contracts/class-registry';
20
+ import { ContractInstancePublishedEvent, ContractInstanceUpdatedEvent } from '@aztec/protocol-contracts/instance-registry';
16
21
  import { L2BlockSourceEvents } from '@aztec/stdlib/block';
17
- import { computePublicBytecodeCommitment, isValidPrivateFunctionMembershipProof, isValidUnconstrainedFunctionMembershipProof } from '@aztec/stdlib/contract';
22
+ import { computePublicBytecodeCommitment, isValidPrivateFunctionMembershipProof, isValidUtilityFunctionMembershipProof } from '@aztec/stdlib/contract';
18
23
  import { getEpochAtSlot, getEpochNumberAtTimestamp, getSlotAtTimestamp, getSlotRangeForEpoch, getTimestampRangeForEpoch } from '@aztec/stdlib/epoch-helpers';
19
- import { Attributes, trackSpan } from '@aztec/telemetry-client';
24
+ import { Attributes, getTelemetryClient, trackSpan } from '@aztec/telemetry-client';
20
25
  import { EventEmitter } from 'events';
21
26
  import groupBy from 'lodash.groupby';
22
- import { createPublicClient, fallback, getContract, http } from 'viem';
23
- import { retrieveBlocksFromRollup, retrieveL1ToL2Messages } from './data_retrieval.js';
24
- import { NoBlobBodiesFoundError } from './errors.js';
27
+ import { createPublicClient, fallback, http } from 'viem';
28
+ import { retrieveBlocksFromRollup, retrieveL1ToL2Message, retrieveL1ToL2Messages, retrievedBlockToPublishedL2Block } from './data_retrieval.js';
29
+ import { InitialBlockNumberNotSequentialError, NoBlobBodiesFoundError } from './errors.js';
25
30
  import { ArchiverInstrumentation } from './instrumentation.js';
31
+ import { validateBlockAttestations } from './validation.js';
32
+ function mapArchiverConfig(config) {
33
+ return {
34
+ pollingIntervalMs: config.archiverPollingIntervalMS,
35
+ batchSize: config.archiverBatchSize,
36
+ skipValidateBlockAttestations: config.skipValidateBlockAttestations
37
+ };
38
+ }
26
39
  /**
27
40
  * Pulls L2 blocks in a non-blocking manner and provides interface for their retrieval.
28
41
  * Responsible for handling robust L1 polling so that other components do not need to
@@ -33,17 +46,18 @@ import { ArchiverInstrumentation } from './instrumentation.js';
33
46
  dataStore;
34
47
  config;
35
48
  blobSinkClient;
49
+ epochCache;
36
50
  instrumentation;
37
51
  l1constants;
38
52
  log;
39
- /**
40
- * A promise in which we will be continually fetching new L2 blocks.
41
- */ runningPromise;
53
+ /** A loop in which we will be continually fetching new L2 blocks. */ runningPromise;
42
54
  rollup;
43
55
  inbox;
44
56
  store;
45
57
  l1BlockNumber;
46
58
  l1Timestamp;
59
+ initialSyncComplete;
60
+ initialSyncPromise;
47
61
  tracer;
48
62
  /**
49
63
  * Creates a new instance of the Archiver.
@@ -54,20 +68,13 @@ import { ArchiverInstrumentation } from './instrumentation.js';
54
68
  * @param pollingIntervalMs - The interval for polling for L1 logs (in milliseconds).
55
69
  * @param store - An archiver data store for storage & retrieval of blocks, encrypted logs & contract data.
56
70
  * @param log - A logger.
57
- */ constructor(publicClient, l1Addresses, dataStore, config, blobSinkClient, instrumentation, l1constants, log = createLogger('archiver')){
58
- super(), this.publicClient = publicClient, this.l1Addresses = l1Addresses, this.dataStore = dataStore, this.config = config, this.blobSinkClient = blobSinkClient, this.instrumentation = instrumentation, this.l1constants = l1constants, this.log = log;
71
+ */ constructor(publicClient, l1Addresses, dataStore, config, blobSinkClient, epochCache, instrumentation, l1constants, log = createLogger('archiver')){
72
+ super(), this.publicClient = publicClient, this.l1Addresses = l1Addresses, this.dataStore = dataStore, this.config = config, this.blobSinkClient = blobSinkClient, this.epochCache = epochCache, this.instrumentation = instrumentation, this.l1constants = l1constants, this.log = log, this.initialSyncComplete = false;
59
73
  this.tracer = instrumentation.tracer;
60
74
  this.store = new ArchiverStoreHelper(dataStore);
61
- this.rollup = getContract({
62
- address: l1Addresses.rollupAddress.toString(),
63
- abi: RollupAbi,
64
- client: publicClient
65
- });
66
- this.inbox = getContract({
67
- address: l1Addresses.inboxAddress.toString(),
68
- abi: InboxAbi,
69
- client: publicClient
70
- });
75
+ this.rollup = new RollupContract(publicClient, l1Addresses.rollupAddress);
76
+ this.inbox = new InboxContract(publicClient, l1Addresses.inboxAddress);
77
+ this.initialSyncPromise = promiseWithResolvers();
71
78
  }
72
79
  /**
73
80
  * Creates a new instance of the Archiver and blocks until it syncs from chain.
@@ -82,29 +89,41 @@ import { ArchiverInstrumentation } from './instrumentation.js';
82
89
  transport: fallback(config.l1RpcUrls.map((url)=>http(url))),
83
90
  pollingInterval: config.viemPollingIntervalMS
84
91
  });
85
- const rollup = getContract({
86
- address: config.l1Contracts.rollupAddress.toString(),
87
- abi: RollupAbi,
88
- client: publicClient
89
- });
90
- const [l1StartBlock, l1GenesisTime] = await Promise.all([
91
- rollup.read.L1_BLOCK_AT_GENESIS(),
92
- rollup.read.getGenesisTime()
92
+ const rollup = new RollupContract(publicClient, config.l1Contracts.rollupAddress);
93
+ const [l1StartBlock, l1GenesisTime, proofSubmissionEpochs, genesisArchiveRoot] = await Promise.all([
94
+ rollup.getL1StartBlock(),
95
+ rollup.getL1GenesisTime(),
96
+ rollup.getProofSubmissionEpochs(),
97
+ rollup.getGenesisArchiveTreeRoot()
93
98
  ]);
99
+ const l1StartBlockHash = await publicClient.getBlock({
100
+ blockNumber: l1StartBlock,
101
+ includeTransactions: false
102
+ }).then((block)=>Buffer32.fromString(block.hash));
94
103
  const { aztecEpochDuration: epochDuration, aztecSlotDuration: slotDuration, ethereumSlotDuration } = config;
95
- const archiver = new Archiver(publicClient, config.l1Contracts, archiverStore, {
96
- pollingIntervalMs: config.archiverPollingIntervalMS ?? 10_000,
97
- batchSize: config.archiverBatchSize ?? 100
98
- }, deps.blobSinkClient, await ArchiverInstrumentation.new(deps.telemetry, ()=>archiverStore.estimateSize()), {
104
+ const l1Constants = {
105
+ l1StartBlockHash,
99
106
  l1StartBlock,
100
107
  l1GenesisTime,
101
108
  epochDuration,
102
109
  slotDuration,
103
- ethereumSlotDuration
104
- });
110
+ ethereumSlotDuration,
111
+ proofSubmissionEpochs: Number(proofSubmissionEpochs),
112
+ genesisArchiveRoot: Fr.fromHexString(genesisArchiveRoot)
113
+ };
114
+ const opts = merge({
115
+ pollingIntervalMs: 10_000,
116
+ batchSize: 100
117
+ }, mapArchiverConfig(config));
118
+ const epochCache = deps.epochCache ?? await EpochCache.create(config.l1Contracts.rollupAddress, config, deps);
119
+ const telemetry = deps.telemetry ?? getTelemetryClient();
120
+ const archiver = new Archiver(publicClient, config.l1Contracts, archiverStore, opts, deps.blobSinkClient, epochCache, await ArchiverInstrumentation.new(telemetry, ()=>archiverStore.estimateSize()), l1Constants);
105
121
  await archiver.start(blockUntilSynced);
106
122
  return archiver;
107
123
  }
124
+ /** Updates archiver config */ updateConfig(newConfig) {
125
+ this.config = merge(this.config, mapArchiverConfig(newConfig));
126
+ }
108
127
  /**
109
128
  * Starts sync process.
110
129
  * @param blockUntilSynced - If true, blocks until the archiver has fully synced.
@@ -112,21 +131,40 @@ import { ArchiverInstrumentation } from './instrumentation.js';
112
131
  if (this.runningPromise) {
113
132
  throw new Error('Archiver is already running');
114
133
  }
134
+ await this.blobSinkClient.testSources();
115
135
  if (blockUntilSynced) {
116
- await this.syncSafe(blockUntilSynced);
136
+ while(!await this.syncSafe(true)){
137
+ this.log.info(`Retrying initial archiver sync in ${this.config.pollingIntervalMs}ms`);
138
+ await sleep(this.config.pollingIntervalMs);
139
+ }
117
140
  }
118
141
  this.runningPromise = new RunningPromise(()=>this.sync(false), this.log, this.config.pollingIntervalMs, makeLoggingErrorHandler(this.log, // Ignored errors will not log to the console
119
142
  // We ignore NoBlobBodiesFound as the message may not have been passed to the blob sink yet
120
143
  NoBlobBodiesFoundError));
121
144
  this.runningPromise.start();
122
145
  }
146
+ syncImmediate() {
147
+ if (!this.runningPromise) {
148
+ throw new Error('Archiver is not running');
149
+ }
150
+ return this.runningPromise.trigger();
151
+ }
152
+ waitForInitialSync() {
153
+ return this.initialSyncPromise.promise;
154
+ }
123
155
  async syncSafe(initialRun) {
124
156
  try {
125
157
  await this.sync(initialRun);
158
+ return true;
126
159
  } catch (error) {
127
- this.log.error('Error during sync', {
128
- error
129
- });
160
+ if (error instanceof NoBlobBodiesFoundError) {
161
+ this.log.error(`Error syncing archiver: ${error.message}`);
162
+ } else if (error instanceof BlockTagTooOldError) {
163
+ this.log.warn(`Re-running archiver sync: ${error.message}`);
164
+ } else {
165
+ this.log.error('Error during archiver sync', error);
166
+ }
167
+ return false;
130
168
  }
131
169
  }
132
170
  /**
@@ -143,11 +181,21 @@ import { ArchiverInstrumentation } from './instrumentation.js';
143
181
  * The archiver will stay back, until there's data on L1 that will move the pointers forward.
144
182
  *
145
183
  * This code does not handle reorgs.
146
- */ const { l1StartBlock } = this.l1constants;
147
- const { blocksSynchedTo = l1StartBlock, messagesSynchedTo = l1StartBlock } = await this.store.getSynchPoint();
148
- const currentL1BlockNumber = await this.publicClient.getBlockNumber();
184
+ */ const { l1StartBlock, l1StartBlockHash } = this.l1constants;
185
+ const { blocksSynchedTo = l1StartBlock, messagesSynchedTo = {
186
+ l1BlockNumber: l1StartBlock,
187
+ l1BlockHash: l1StartBlockHash
188
+ } } = await this.store.getSynchPoint();
189
+ const currentL1Block = await this.publicClient.getBlock({
190
+ includeTransactions: false
191
+ });
192
+ const currentL1BlockNumber = currentL1Block.number;
193
+ const currentL1BlockHash = Buffer32.fromString(currentL1Block.hash);
149
194
  if (initialRun) {
150
- this.log.info(`Starting archiver sync to rollup contract ${this.l1Addresses.rollupAddress.toString()} from L1 block ${Math.min(Number(blocksSynchedTo), Number(messagesSynchedTo))} to current L1 block ${currentL1BlockNumber}`);
195
+ this.log.info(`Starting archiver sync to rollup contract ${this.l1Addresses.rollupAddress.toString()} from L1 block ${blocksSynchedTo}` + ` to current L1 block ${currentL1BlockNumber} with hash ${currentL1BlockHash.toString()}`, {
196
+ blocksSynchedTo,
197
+ messagesSynchedTo
198
+ });
151
199
  }
152
200
  // ********** Ensuring Consistency of data pulled from L1 **********
153
201
  /**
@@ -166,26 +214,38 @@ import { ArchiverInstrumentation } from './instrumentation.js';
166
214
  * data up to the currentBlockNumber captured at the top of this function. We might want to improve on this
167
215
  * in future but for the time being it should give us the guarantees that we need
168
216
  */ // ********** Events that are processed per L1 block **********
169
- await this.handleL1ToL2Messages(messagesSynchedTo, currentL1BlockNumber);
170
- // Store latest l1 block number and timestamp seen. Used for epoch and slots calculations.
171
- if (!this.l1BlockNumber || this.l1BlockNumber < currentL1BlockNumber) {
172
- this.l1Timestamp = (await this.publicClient.getBlock({
173
- blockNumber: currentL1BlockNumber
174
- })).timestamp;
175
- this.l1BlockNumber = currentL1BlockNumber;
176
- }
217
+ await this.handleL1ToL2Messages(messagesSynchedTo, currentL1BlockNumber, currentL1BlockHash);
218
+ // Get L1 timestamp for the current block
219
+ const currentL1Timestamp = !this.l1Timestamp || !this.l1BlockNumber || this.l1BlockNumber !== currentL1BlockNumber ? (await this.publicClient.getBlock({
220
+ blockNumber: currentL1BlockNumber
221
+ })).timestamp : this.l1Timestamp;
177
222
  // ********** Events that are processed per L2 block **********
178
223
  if (currentL1BlockNumber > blocksSynchedTo) {
179
- // First we retrieve new L2 blocks
180
- const { provenBlockNumber } = await this.handleL2blocks(blocksSynchedTo, currentL1BlockNumber);
181
- // And then we prune the current epoch if it'd reorg on next submission.
224
+ // First we retrieve new L2 blocks and store them in the DB. This will also update the
225
+ // pending chain validation status, proven block number, and synched L1 block number.
226
+ const rollupStatus = await this.handleL2blocks(blocksSynchedTo, currentL1BlockNumber);
227
+ // Then we prune the current epoch if it'd reorg on next submission.
182
228
  // Note that we don't do this before retrieving L2 blocks because we may need to retrieve
183
229
  // blocks from more than 2 epochs ago, so we want to make sure we have the latest view of
184
230
  // the chain locally before we start unwinding stuff. This can be optimized by figuring out
185
231
  // up to which point we're pruning, and then requesting L2 blocks up to that point only.
186
- await this.handleEpochPrune(provenBlockNumber, currentL1BlockNumber);
232
+ const { rollupCanPrune } = await this.handleEpochPrune(rollupStatus.provenBlockNumber, currentL1BlockNumber, currentL1Timestamp);
233
+ // And lastly we check if we are missing any L2 blocks behind us due to a possible L1 reorg.
234
+ // We only do this if rollup cant prune on the next submission. Otherwise we will end up
235
+ // re-syncing the blocks we have just unwound above. We also dont do this if the last block is invalid,
236
+ // since the archiver will rightfully refuse to sync up to it.
237
+ if (!rollupCanPrune && rollupStatus.validationResult?.valid) {
238
+ await this.checkForNewBlocksBeforeL1SyncPoint(rollupStatus, blocksSynchedTo, currentL1BlockNumber);
239
+ }
187
240
  this.instrumentation.updateL1BlockHeight(currentL1BlockNumber);
188
241
  }
242
+ // After syncing has completed, update the current l1 block number and timestamp,
243
+ // otherwise we risk announcing to the world that we've synced to a given point,
244
+ // but the corresponding blocks have not been processed (see #12631).
245
+ this.l1Timestamp = currentL1Timestamp;
246
+ this.l1BlockNumber = currentL1BlockNumber;
247
+ this.initialSyncComplete = true;
248
+ this.initialSyncPromise.resolve();
189
249
  if (initialRun) {
190
250
  this.log.info(`Initial archiver sync to L1 block ${currentL1BlockNumber} complete.`, {
191
251
  l1BlockNumber: currentL1BlockNumber,
@@ -194,36 +254,52 @@ import { ArchiverInstrumentation } from './instrumentation.js';
194
254
  });
195
255
  }
196
256
  }
197
- /** Queries the rollup contract on whether a prune can be executed on the immediatenext L1 block. */ async canPrune(currentL1BlockNumber) {
198
- const time = (this.l1Timestamp ?? 0n) + BigInt(this.l1constants.ethereumSlotDuration);
199
- return await this.rollup.read.canPruneAtTime([
200
- time
201
- ], {
257
+ /** Queries the rollup contract on whether a prune can be executed on the immediate next L1 block. */ async canPrune(currentL1BlockNumber, currentL1Timestamp) {
258
+ const time = (currentL1Timestamp ?? 0n) + BigInt(this.l1constants.ethereumSlotDuration);
259
+ const result = await this.rollup.canPruneAtTime(time, {
202
260
  blockNumber: currentL1BlockNumber
203
261
  });
262
+ if (result) {
263
+ this.log.debug(`Rollup contract allows pruning at L1 block ${currentL1BlockNumber} time ${time}`, {
264
+ currentL1Timestamp,
265
+ pruneTime: time,
266
+ currentL1BlockNumber
267
+ });
268
+ }
269
+ return result;
204
270
  }
205
- /** Checks if there'd be a reorg for the next block submission and start pruning now. */ async handleEpochPrune(provenBlockNumber, currentL1BlockNumber) {
206
- const localPendingBlockNumber = BigInt(await this.getBlockNumber());
207
- const canPrune = localPendingBlockNumber > provenBlockNumber && await this.canPrune(currentL1BlockNumber);
271
+ /** Checks if there'd be a reorg for the next block submission and start pruning now. */ async handleEpochPrune(provenBlockNumber, currentL1BlockNumber, currentL1Timestamp) {
272
+ const rollupCanPrune = await this.canPrune(currentL1BlockNumber, currentL1Timestamp);
273
+ const localPendingBlockNumber = await this.getBlockNumber();
274
+ const canPrune = localPendingBlockNumber > provenBlockNumber && rollupCanPrune;
208
275
  if (canPrune) {
209
- const localPendingSlotNumber = await this.getL2SlotNumber();
210
- const localPendingEpochNumber = getEpochAtSlot(localPendingSlotNumber, this.l1constants);
276
+ const timer = new Timer();
277
+ const pruneFrom = provenBlockNumber + 1;
278
+ const header = await this.getBlockHeader(Number(pruneFrom));
279
+ if (header === undefined) {
280
+ throw new Error(`Missing block header ${pruneFrom}`);
281
+ }
282
+ const pruneFromSlotNumber = header.globalVariables.slotNumber.toBigInt();
283
+ const pruneFromEpochNumber = getEpochAtSlot(pruneFromSlotNumber, this.l1constants);
284
+ const blocksToUnwind = localPendingBlockNumber - provenBlockNumber;
285
+ const blocks = await this.getBlocks(Number(provenBlockNumber) + 1, Number(blocksToUnwind));
211
286
  // Emit an event for listening services to react to the chain prune
212
287
  this.emit(L2BlockSourceEvents.L2PruneDetected, {
213
288
  type: L2BlockSourceEvents.L2PruneDetected,
214
- blockNumber: localPendingBlockNumber,
215
- slotNumber: localPendingSlotNumber,
216
- epochNumber: localPendingEpochNumber
289
+ epochNumber: pruneFromEpochNumber,
290
+ blocks
217
291
  });
218
- const blocksToUnwind = localPendingBlockNumber - provenBlockNumber;
219
- this.log.debug(`L2 prune from ${provenBlockNumber + 1n} to ${localPendingBlockNumber} will occur on next block submission.`);
292
+ this.log.debug(`L2 prune from ${provenBlockNumber + 1} to ${localPendingBlockNumber} will occur on next block submission.`);
220
293
  await this.store.unwindBlocks(Number(localPendingBlockNumber), Number(blocksToUnwind));
221
294
  this.log.warn(`Unwound ${count(blocksToUnwind, 'block')} from L2 block ${localPendingBlockNumber} ` + `to ${provenBlockNumber} due to predicted reorg at L1 block ${currentL1BlockNumber}. ` + `Updated L2 latest block is ${await this.getBlockNumber()}.`);
222
- this.instrumentation.processPrune();
295
+ this.instrumentation.processPrune(timer.ms());
223
296
  // TODO(palla/reorg): Do we need to set the block synched L1 block number here?
224
297
  // Seems like the next iteration should handle this.
225
298
  // await this.store.setBlockSynchedL1BlockNumber(currentL1BlockNumber);
226
299
  }
300
+ return {
301
+ rollupCanPrune
302
+ };
227
303
  }
228
304
  nextRange(end, limit) {
229
305
  const batchSize = this.config.batchSize * this.l1constants.slotDuration / this.l1constants.ethereumSlotDuration;
@@ -240,43 +316,183 @@ import { ArchiverInstrumentation } from './instrumentation.js';
240
316
  nextEnd
241
317
  ];
242
318
  }
243
- async handleL1ToL2Messages(messagesSynchedTo, currentL1BlockNumber) {
244
- this.log.trace(`Handling L1 to L2 messages from ${messagesSynchedTo} to ${currentL1BlockNumber}.`);
245
- if (currentL1BlockNumber <= messagesSynchedTo) {
319
+ async handleL1ToL2Messages(messagesSyncPoint, currentL1BlockNumber, _currentL1BlockHash) {
320
+ this.log.trace(`Handling L1 to L2 messages from ${messagesSyncPoint.l1BlockNumber} to ${currentL1BlockNumber}.`);
321
+ if (currentL1BlockNumber <= messagesSyncPoint.l1BlockNumber) {
246
322
  return;
247
323
  }
248
- const localTotalMessageCount = await this.store.getTotalL1ToL2MessageCount();
249
- const destinationTotalMessageCount = await this.inbox.read.totalMessagesInserted();
250
- if (localTotalMessageCount === destinationTotalMessageCount) {
251
- await this.store.setMessageSynchedL1BlockNumber(currentL1BlockNumber);
252
- this.log.trace(`Retrieved no new L1 to L2 messages between L1 blocks ${messagesSynchedTo + 1n} and ${currentL1BlockNumber}.`);
324
+ // Load remote and local inbox states.
325
+ const localMessagesInserted = await this.store.getTotalL1ToL2MessageCount();
326
+ const localLastMessage = await this.store.getLastL1ToL2Message();
327
+ const remoteMessagesState = await this.inbox.getState({
328
+ blockNumber: currentL1BlockNumber
329
+ });
330
+ this.log.trace(`Retrieved remote inbox state at L1 block ${currentL1BlockNumber}.`, {
331
+ localMessagesInserted,
332
+ localLastMessage,
333
+ remoteMessagesState
334
+ });
335
+ // Compare message count and rolling hash. If they match, no need to retrieve anything.
336
+ if (remoteMessagesState.totalMessagesInserted === localMessagesInserted && remoteMessagesState.messagesRollingHash.equals(localLastMessage?.rollingHash ?? Buffer16.ZERO)) {
337
+ this.log.debug(`No L1 to L2 messages to query between L1 blocks ${messagesSyncPoint.l1BlockNumber} and ${currentL1BlockNumber}.`);
253
338
  return;
254
339
  }
255
- // Retrieve messages in batches. Each batch is estimated to acommodate up to L2 'blockBatchSize' blocks,
256
- let searchStartBlock = messagesSynchedTo;
257
- let searchEndBlock = messagesSynchedTo;
340
+ // Check if our syncpoint is still valid. If not, there was an L1 reorg and we need to re-retrieve messages.
341
+ // Note that we need to fetch it from logs and not from inbox state at the syncpoint l1 block number, since it
342
+ // could be older than 128 blocks and non-archive nodes cannot resolve it.
343
+ if (localLastMessage) {
344
+ const remoteLastMessage = await this.retrieveL1ToL2Message(localLastMessage.leaf);
345
+ this.log.trace(`Retrieved remote message for local last`, {
346
+ remoteLastMessage,
347
+ localLastMessage
348
+ });
349
+ if (!remoteLastMessage || !remoteLastMessage.rollingHash.equals(localLastMessage.rollingHash)) {
350
+ this.log.warn(`Rolling back L1 to L2 messages due to hash mismatch or msg not found.`, {
351
+ remoteLastMessage,
352
+ messagesSyncPoint,
353
+ localLastMessage
354
+ });
355
+ messagesSyncPoint = await this.rollbackL1ToL2Messages(localLastMessage, messagesSyncPoint);
356
+ this.log.debug(`Rolled back L1 to L2 messages to L1 block ${messagesSyncPoint.l1BlockNumber}.`, {
357
+ messagesSyncPoint
358
+ });
359
+ }
360
+ }
361
+ // Retrieve and save messages in batches. Each batch is estimated to acommodate up to L2 'blockBatchSize' blocks,
362
+ let searchStartBlock = 0n;
363
+ let searchEndBlock = messagesSyncPoint.l1BlockNumber;
364
+ let lastMessage;
365
+ let messageCount = 0;
258
366
  do {
259
367
  [searchStartBlock, searchEndBlock] = this.nextRange(searchEndBlock, currentL1BlockNumber);
260
368
  this.log.trace(`Retrieving L1 to L2 messages between L1 blocks ${searchStartBlock} and ${searchEndBlock}.`);
261
- const retrievedL1ToL2Messages = await retrieveL1ToL2Messages(this.inbox, searchStartBlock, searchEndBlock);
262
- this.log.verbose(`Retrieved ${retrievedL1ToL2Messages.retrievedData.length} new L1 to L2 messages between L1 blocks ${searchStartBlock} and ${searchEndBlock}.`);
263
- await this.store.addL1ToL2Messages(retrievedL1ToL2Messages);
264
- for (const msg of retrievedL1ToL2Messages.retrievedData){
369
+ const messages = await retrieveL1ToL2Messages(this.inbox.getContract(), searchStartBlock, searchEndBlock);
370
+ this.log.verbose(`Retrieved ${messages.length} new L1 to L2 messages between L1 blocks ${searchStartBlock} and ${searchEndBlock}.`);
371
+ const timer = new Timer();
372
+ await this.store.addL1ToL2Messages(messages);
373
+ const perMsg = timer.ms() / messages.length;
374
+ this.instrumentation.processNewMessages(messages.length, perMsg);
375
+ for (const msg of messages){
265
376
  this.log.debug(`Downloaded L1 to L2 message`, {
266
- leaf: msg.leaf.toString(),
267
- index: msg.index
377
+ ...msg,
378
+ leaf: msg.leaf.toString()
268
379
  });
380
+ lastMessage = msg;
381
+ messageCount++;
382
+ }
383
+ }while (searchEndBlock < currentL1BlockNumber)
384
+ // Log stats for messages retrieved (if any).
385
+ if (messageCount > 0) {
386
+ this.log.info(`Retrieved ${messageCount} new L1 to L2 messages up to message with index ${lastMessage?.index} for L2 block ${lastMessage?.l2BlockNumber}`, {
387
+ lastMessage,
388
+ messageCount
389
+ });
390
+ }
391
+ // Warn if the resulting rolling hash does not match the remote state we had retrieved.
392
+ if (lastMessage && !lastMessage.rollingHash.equals(remoteMessagesState.messagesRollingHash)) {
393
+ this.log.warn(`Last message retrieved rolling hash does not match remote state.`, {
394
+ lastMessage,
395
+ remoteMessagesState
396
+ });
397
+ }
398
+ }
399
+ async retrieveL1ToL2Message(leaf) {
400
+ const currentL1BlockNumber = await this.publicClient.getBlockNumber();
401
+ let searchStartBlock = 0n;
402
+ let searchEndBlock = this.l1constants.l1StartBlock - 1n;
403
+ do {
404
+ [searchStartBlock, searchEndBlock] = this.nextRange(searchEndBlock, currentL1BlockNumber);
405
+ const message = await retrieveL1ToL2Message(this.inbox.getContract(), leaf, searchStartBlock, searchEndBlock);
406
+ if (message) {
407
+ return message;
269
408
  }
270
409
  }while (searchEndBlock < currentL1BlockNumber)
410
+ return undefined;
411
+ }
412
+ async rollbackL1ToL2Messages(localLastMessage, messagesSyncPoint) {
413
+ // Slowly go back through our messages until we find the last common message.
414
+ // We could query the logs in batch as an optimization, but the depth of the reorg should not be deep, and this
415
+ // is a very rare case, so it's fine to query one log at a time.
416
+ let commonMsg;
417
+ this.log.verbose(`Searching most recent common L1 to L2 message at or before index ${localLastMessage.index}`);
418
+ for await (const msg of this.store.iterateL1ToL2Messages({
419
+ reverse: true,
420
+ end: localLastMessage.index
421
+ })){
422
+ const remoteMsg = await this.retrieveL1ToL2Message(msg.leaf);
423
+ const logCtx = {
424
+ remoteMsg,
425
+ localMsg: msg
426
+ };
427
+ if (remoteMsg && remoteMsg.rollingHash.equals(msg.rollingHash)) {
428
+ this.log.verbose(`Found most recent common L1 to L2 message at index ${msg.index} on L1 block ${msg.l1BlockNumber}`, logCtx);
429
+ commonMsg = remoteMsg;
430
+ break;
431
+ } else if (remoteMsg) {
432
+ this.log.debug(`Local L1 to L2 message with index ${msg.index} has different rolling hash`, logCtx);
433
+ } else {
434
+ this.log.debug(`Local L1 to L2 message with index ${msg.index} not found on L1`, logCtx);
435
+ }
436
+ }
437
+ // Delete everything after the common message we found.
438
+ const lastGoodIndex = commonMsg?.index;
439
+ this.log.warn(`Deleting all local L1 to L2 messages after index ${lastGoodIndex ?? 'undefined'}`);
440
+ await this.store.removeL1ToL2Messages(lastGoodIndex !== undefined ? lastGoodIndex + 1n : 0n);
441
+ // Update the syncpoint so the loop below reprocesses the changed messages. We go to the block before
442
+ // the last common one, so we force reprocessing it, in case new messages were added on that same L1 block
443
+ // after the last common message.
444
+ const syncPointL1BlockNumber = commonMsg ? commonMsg.l1BlockNumber - 1n : this.l1constants.l1StartBlock;
445
+ const syncPointL1BlockHash = await this.getL1BlockHash(syncPointL1BlockNumber);
446
+ messagesSyncPoint = {
447
+ l1BlockNumber: syncPointL1BlockNumber,
448
+ l1BlockHash: syncPointL1BlockHash
449
+ };
450
+ await this.store.setMessageSynchedL1Block(messagesSyncPoint);
451
+ return messagesSyncPoint;
452
+ }
453
+ async getL1BlockHash(l1BlockNumber) {
454
+ const block = await this.publicClient.getBlock({
455
+ blockNumber: l1BlockNumber,
456
+ includeTransactions: false
457
+ });
458
+ if (!block) {
459
+ throw new Error(`Missing L1 block ${l1BlockNumber}`);
460
+ }
461
+ return Buffer32.fromString(block.hash);
271
462
  }
272
463
  async handleL2blocks(blocksSynchedTo, currentL1BlockNumber) {
273
- const localPendingBlockNumber = BigInt(await this.getBlockNumber());
274
- const [provenBlockNumber, provenArchive, pendingBlockNumber, pendingArchive, archiveForLocalPendingBlockNumber] = await this.rollup.read.status([
275
- localPendingBlockNumber
276
- ], {
464
+ const localPendingBlockNumber = await this.getBlockNumber();
465
+ const initialValidationResult = await this.store.getPendingChainValidationStatus();
466
+ const [provenBlockNumber, provenArchive, pendingBlockNumber, pendingArchive, archiveForLocalPendingBlockNumber] = await this.rollup.status(BigInt(localPendingBlockNumber), {
277
467
  blockNumber: currentL1BlockNumber
278
468
  });
469
+ const rollupStatus = {
470
+ provenBlockNumber: Number(provenBlockNumber),
471
+ provenArchive,
472
+ pendingBlockNumber: Number(pendingBlockNumber),
473
+ pendingArchive,
474
+ validationResult: initialValidationResult
475
+ };
476
+ this.log.trace(`Retrieved rollup status at current L1 block ${currentL1BlockNumber}.`, {
477
+ localPendingBlockNumber,
478
+ blocksSynchedTo,
479
+ currentL1BlockNumber,
480
+ archiveForLocalPendingBlockNumber,
481
+ ...rollupStatus
482
+ });
279
483
  const updateProvenBlock = async ()=>{
484
+ // Annoying edge case: if proven block is moved back to 0 due to a reorg at the beginning of the chain,
485
+ // we need to set it to zero. This is an edge case because we dont have a block zero (initial block is one),
486
+ // so localBlockForDestinationProvenBlockNumber would not be found below.
487
+ if (provenBlockNumber === 0n) {
488
+ const localProvenBlockNumber = await this.store.getProvenL2BlockNumber();
489
+ if (localProvenBlockNumber !== Number(provenBlockNumber)) {
490
+ await this.store.setProvenL2BlockNumber(Number(provenBlockNumber));
491
+ this.log.info(`Rolled back proven chain to block ${provenBlockNumber}`, {
492
+ provenBlockNumber
493
+ });
494
+ }
495
+ }
280
496
  const localBlockForDestinationProvenBlockNumber = await this.getBlock(Number(provenBlockNumber));
281
497
  // Sanity check. I've hit what seems to be a state where the proven block is set to a value greater than the latest
282
498
  // synched block when requesting L2Tips from the archiver. This is the only place where the proven block is set.
@@ -284,6 +500,7 @@ import { ArchiverInstrumentation } from './instrumentation.js';
284
500
  if (localBlockForDestinationProvenBlockNumber && synched < localBlockForDestinationProvenBlockNumber?.number) {
285
501
  this.log.error(`Hit local block greater than last synched block: ${localBlockForDestinationProvenBlockNumber.number} > ${synched}`);
286
502
  }
503
+ this.log.trace(`Local block for remote proven block ${provenBlockNumber} is ${localBlockForDestinationProvenBlockNumber?.archive.root.toString() ?? 'undefined'}`);
287
504
  if (localBlockForDestinationProvenBlockNumber && provenArchive === localBlockForDestinationProvenBlockNumber.archive.root.toString()) {
288
505
  const localProvenBlockNumber = await this.store.getProvenL2BlockNumber();
289
506
  if (localProvenBlockNumber !== Number(provenBlockNumber)) {
@@ -291,52 +508,68 @@ import { ArchiverInstrumentation } from './instrumentation.js';
291
508
  this.log.info(`Updated proven chain to block ${provenBlockNumber}`, {
292
509
  provenBlockNumber
293
510
  });
511
+ const provenSlotNumber = localBlockForDestinationProvenBlockNumber.header.globalVariables.slotNumber.toBigInt();
512
+ const provenEpochNumber = getEpochAtSlot(provenSlotNumber, this.l1constants);
513
+ this.emit(L2BlockSourceEvents.L2BlockProven, {
514
+ type: L2BlockSourceEvents.L2BlockProven,
515
+ blockNumber: provenBlockNumber,
516
+ slotNumber: provenSlotNumber,
517
+ epochNumber: provenEpochNumber
518
+ });
519
+ } else {
520
+ this.log.trace(`Proven block ${provenBlockNumber} already stored.`);
294
521
  }
295
522
  }
296
523
  this.instrumentation.updateLastProvenBlock(Number(provenBlockNumber));
297
524
  };
298
525
  // This is an edge case that we only hit if there are no proposed blocks.
299
526
  // If we have 0 blocks locally and there are no blocks onchain there is nothing to do.
300
- const noBlocks = localPendingBlockNumber === 0n && pendingBlockNumber === 0n;
527
+ const noBlocks = localPendingBlockNumber === 0 && pendingBlockNumber === 0n;
301
528
  if (noBlocks) {
302
529
  await this.store.setBlockSynchedL1BlockNumber(currentL1BlockNumber);
303
- this.log.debug(`No blocks to retrieve from ${blocksSynchedTo + 1n} to ${currentL1BlockNumber}`);
304
- return {
305
- provenBlockNumber
306
- };
530
+ this.log.debug(`No blocks to retrieve from ${blocksSynchedTo + 1n} to ${currentL1BlockNumber}, no blocks on chain`);
531
+ return rollupStatus;
307
532
  }
308
533
  await updateProvenBlock();
309
534
  // Related to the L2 reorgs of the pending chain. We are only interested in actually addressing a reorg if there
310
535
  // are any state that could be impacted by it. If we have no blocks, there is no impact.
311
536
  if (localPendingBlockNumber > 0) {
312
- const localPendingBlock = await this.getBlock(Number(localPendingBlockNumber));
537
+ const localPendingBlock = await this.getBlock(localPendingBlockNumber);
313
538
  if (localPendingBlock === undefined) {
314
539
  throw new Error(`Missing block ${localPendingBlockNumber}`);
315
540
  }
316
- const noBlockSinceLast = localPendingBlock && pendingArchive === localPendingBlock.archive.root.toString();
541
+ const localPendingArchiveRoot = localPendingBlock.archive.root.toString();
542
+ const noBlockSinceLast = localPendingBlock && pendingArchive === localPendingArchiveRoot;
317
543
  if (noBlockSinceLast) {
318
- await this.store.setBlockSynchedL1BlockNumber(currentL1BlockNumber);
544
+ // We believe the following line causes a problem when we encounter L1 re-orgs.
545
+ // Basically, by setting the synched L1 block number here, we are saying that we have
546
+ // processed all blocks up to the current L1 block number and we will not attempt to retrieve logs from
547
+ // this block again (or any blocks before).
548
+ // However, in the re-org scenario, our L1 node is temporarily lying to us and we end up potentially missing blocks
549
+ // We must only set this block number based on actually retrieved logs.
550
+ // TODO(#8621): Tackle this properly when we handle L1 Re-orgs.
551
+ // await this.store.setBlockSynchedL1BlockNumber(currentL1BlockNumber);
319
552
  this.log.debug(`No blocks to retrieve from ${blocksSynchedTo + 1n} to ${currentL1BlockNumber}`);
320
- return {
321
- provenBlockNumber
322
- };
553
+ return rollupStatus;
323
554
  }
324
- const localPendingBlockInChain = archiveForLocalPendingBlockNumber === localPendingBlock.archive.root.toString();
555
+ const localPendingBlockInChain = archiveForLocalPendingBlockNumber === localPendingArchiveRoot;
325
556
  if (!localPendingBlockInChain) {
326
557
  // If our local pending block tip is not in the chain on L1 a "prune" must have happened
327
558
  // or the L1 have reorged.
328
559
  // In any case, we have to figure out how far into the past the action will take us.
329
560
  // For simplicity here, we will simply rewind until we end in a block that is also on the chain on L1.
330
- this.log.debug(`L2 prune has been detected.`);
561
+ this.log.debug(`L2 prune has been detected due to local pending block ${localPendingBlockNumber} not in chain`, {
562
+ localPendingBlockNumber,
563
+ localPendingArchiveRoot,
564
+ archiveForLocalPendingBlockNumber
565
+ });
331
566
  let tipAfterUnwind = localPendingBlockNumber;
332
567
  while(true){
333
568
  const candidateBlock = await this.getBlock(Number(tipAfterUnwind));
334
569
  if (candidateBlock === undefined) {
335
570
  break;
336
571
  }
337
- const archiveAtContract = await this.rollup.read.archiveAt([
338
- BigInt(candidateBlock.number)
339
- ]);
572
+ const archiveAtContract = await this.rollup.archiveAt(BigInt(candidateBlock.number));
340
573
  if (archiveAtContract === candidateBlock.archive.root.toString()) {
341
574
  break;
342
575
  }
@@ -347,15 +580,16 @@ import { ArchiverInstrumentation } from './instrumentation.js';
347
580
  this.log.warn(`Unwound ${count(blocksToUnwind, 'block')} from L2 block ${localPendingBlockNumber} ` + `due to mismatched block hashes at L1 block ${currentL1BlockNumber}. ` + `Updated L2 latest block is ${await this.getBlockNumber()}.`);
348
581
  }
349
582
  }
350
- // Retrieve L2 blocks in batches. Each batch is estimated to acommodate up to L2 'blockBatchSize' blocks,
583
+ // Retrieve L2 blocks in batches. Each batch is estimated to accommodate up to L2 'blockBatchSize' blocks,
351
584
  // computed using the L2 block time vs the L1 block time.
352
585
  let searchStartBlock = blocksSynchedTo;
353
586
  let searchEndBlock = blocksSynchedTo;
587
+ let lastRetrievedBlock;
354
588
  do {
355
589
  [searchStartBlock, searchEndBlock] = this.nextRange(searchEndBlock, currentL1BlockNumber);
356
590
  this.log.trace(`Retrieving L2 blocks from L1 block ${searchStartBlock} to ${searchEndBlock}`);
357
- // TODO(md): Retreive from blob sink then from consensus client, then from peers
358
- const retrievedBlocks = await retrieveBlocksFromRollup(this.rollup, this.publicClient, this.blobSinkClient, searchStartBlock, searchEndBlock, this.log);
591
+ // TODO(md): Retrieve from blob sink then from consensus client, then from peers
592
+ const retrievedBlocks = await retrieveBlocksFromRollup(this.rollup.getContract(), this.publicClient, this.blobSinkClient, searchStartBlock, searchEndBlock, this.log);
359
593
  if (retrievedBlocks.length === 0) {
360
594
  // We are not calling `setBlockSynchedL1BlockNumber` because it may cause sync issues if based off infura.
361
595
  // See further details in earlier comments.
@@ -364,31 +598,118 @@ import { ArchiverInstrumentation } from './instrumentation.js';
364
598
  }
365
599
  const lastProcessedL1BlockNumber = retrievedBlocks[retrievedBlocks.length - 1].l1.blockNumber;
366
600
  this.log.debug(`Retrieved ${retrievedBlocks.length} new L2 blocks between L1 blocks ${searchStartBlock} and ${searchEndBlock} with last processed L1 block ${lastProcessedL1BlockNumber}.`);
367
- for (const block of retrievedBlocks){
368
- this.log.debug(`Ingesting new L2 block ${block.data.number} with ${block.data.body.txEffects.length} txs`, {
369
- blockHash: block.data.hash(),
601
+ const publishedBlocks = await Promise.all(retrievedBlocks.map((b)=>retrievedBlockToPublishedL2Block(b)));
602
+ const validBlocks = [];
603
+ for (const block of publishedBlocks){
604
+ const validationResult = this.config.skipValidateBlockAttestations ? {
605
+ valid: true
606
+ } : await validateBlockAttestations(block, this.epochCache, this.l1constants, this.log);
607
+ // Only update the validation result if it has changed, so we can keep track of the first invalid block
608
+ // in case there is a sequence of more than one invalid block, as we need to invalidate the first one.
609
+ // There is an exception though: if an invalid block is invalidated and replaced with another invalid block,
610
+ // we need to update the validation result, since we need to be able to invalidate the new one.
611
+ // See test 'chain progresses if an invalid block is invalidated with an invalid one' for more info.
612
+ if (rollupStatus.validationResult?.valid !== validationResult.valid || !rollupStatus.validationResult.valid && !validationResult.valid && rollupStatus.validationResult.block.blockNumber === validationResult.block.blockNumber) {
613
+ rollupStatus.validationResult = validationResult;
614
+ }
615
+ if (!validationResult.valid) {
616
+ this.log.warn(`Skipping block ${block.block.number} due to invalid attestations`, {
617
+ blockHash: block.block.hash(),
618
+ l1BlockNumber: block.l1.blockNumber,
619
+ ...pick(validationResult, 'reason')
620
+ });
621
+ // Emit event for invalid block detection
622
+ this.emit(L2BlockSourceEvents.InvalidAttestationsBlockDetected, {
623
+ type: L2BlockSourceEvents.InvalidAttestationsBlockDetected,
624
+ validationResult
625
+ });
626
+ continue;
627
+ }
628
+ validBlocks.push(block);
629
+ this.log.debug(`Ingesting new L2 block ${block.block.number} with ${block.block.body.txEffects.length} txs`, {
630
+ blockHash: block.block.hash(),
370
631
  l1BlockNumber: block.l1.blockNumber,
371
- ...block.data.header.globalVariables.toInspect(),
372
- ...block.data.getStats()
632
+ ...block.block.header.globalVariables.toInspect(),
633
+ ...block.block.getStats()
373
634
  });
374
635
  }
375
- const [processDuration] = await elapsed(()=>this.store.addBlocks(retrievedBlocks));
376
- this.instrumentation.processNewBlocks(processDuration / retrievedBlocks.length, retrievedBlocks.map((b)=>b.data));
377
- for (const block of retrievedBlocks){
378
- this.log.info(`Downloaded L2 block ${block.data.number}`, {
379
- blockHash: block.data.hash(),
380
- blockNumber: block.data.number,
381
- txCount: block.data.body.txEffects.length,
382
- globalVariables: block.data.header.globalVariables.toInspect()
636
+ try {
637
+ const updatedValidationResult = rollupStatus.validationResult === initialValidationResult ? undefined : rollupStatus.validationResult;
638
+ const [processDuration] = await elapsed(()=>this.store.addBlocks(validBlocks, updatedValidationResult));
639
+ this.instrumentation.processNewBlocks(processDuration / validBlocks.length, validBlocks.map((b)=>b.block));
640
+ } catch (err) {
641
+ if (err instanceof InitialBlockNumberNotSequentialError) {
642
+ const { previousBlockNumber, newBlockNumber } = err;
643
+ const previousBlock = previousBlockNumber ? await this.store.getPublishedBlock(previousBlockNumber) : undefined;
644
+ const updatedL1SyncPoint = previousBlock?.l1.blockNumber ?? this.l1constants.l1StartBlock;
645
+ await this.store.setBlockSynchedL1BlockNumber(updatedL1SyncPoint);
646
+ this.log.warn(`Attempting to insert block ${newBlockNumber} with previous block ${previousBlockNumber}. Rolling back L1 sync point to ${updatedL1SyncPoint} to try and fetch the missing blocks.`, {
647
+ previousBlockNumber,
648
+ previousBlockHash: await previousBlock?.block.hash(),
649
+ newBlockNumber,
650
+ updatedL1SyncPoint
651
+ });
652
+ }
653
+ throw err;
654
+ }
655
+ for (const block of validBlocks){
656
+ this.log.info(`Downloaded L2 block ${block.block.number}`, {
657
+ blockHash: await block.block.hash(),
658
+ blockNumber: block.block.number,
659
+ txCount: block.block.body.txEffects.length,
660
+ globalVariables: block.block.header.globalVariables.toInspect(),
661
+ archiveRoot: block.block.archive.root.toString(),
662
+ archiveNextLeafIndex: block.block.archive.nextAvailableLeafIndex
383
663
  });
384
664
  }
665
+ lastRetrievedBlock = validBlocks.at(-1) ?? lastRetrievedBlock;
385
666
  }while (searchEndBlock < currentL1BlockNumber)
386
667
  // Important that we update AFTER inserting the blocks.
387
668
  await updateProvenBlock();
388
669
  return {
389
- provenBlockNumber
670
+ ...rollupStatus,
671
+ lastRetrievedBlock
390
672
  };
391
673
  }
674
+ async checkForNewBlocksBeforeL1SyncPoint(status, blocksSynchedTo, currentL1BlockNumber) {
675
+ const { lastRetrievedBlock, pendingBlockNumber } = status;
676
+ // Compare the last L2 block we have (either retrieved in this round or loaded from store) with what the
677
+ // rollup contract told us was the latest one (pinned at the currentL1BlockNumber).
678
+ const latestLocalL2BlockNumber = lastRetrievedBlock?.block.number ?? await this.store.getSynchedL2BlockNumber();
679
+ if (latestLocalL2BlockNumber < pendingBlockNumber) {
680
+ // Here we have consumed all logs until the `currentL1Block` we pinned at the beginning of the archiver loop,
681
+ // but still havent reached the pending block according to the call to the rollup contract.
682
+ // We suspect an L1 reorg that added blocks *behind* us. If that is the case, it must have happened between the
683
+ // last L2 block we saw and the current one, so we reset the last synched L1 block number. In the edge case we
684
+ // don't have one, we go back 2 L1 epochs, which is the deepest possible reorg (assuming Casper is working).
685
+ const latestLocalL2Block = lastRetrievedBlock ?? (latestLocalL2BlockNumber > 0 ? await this.store.getPublishedBlocks(latestLocalL2BlockNumber, 1).then(([b])=>b) : undefined);
686
+ const targetL1BlockNumber = latestLocalL2Block?.l1.blockNumber ?? maxBigint(currentL1BlockNumber - 64n, 0n);
687
+ const latestLocalL2BlockArchive = latestLocalL2Block?.block.archive.root.toString();
688
+ this.log.warn(`Failed to reach L2 block ${pendingBlockNumber} at ${currentL1BlockNumber} (latest is ${latestLocalL2BlockNumber}). ` + `Rolling back last synched L1 block number to ${targetL1BlockNumber}.`, {
689
+ latestLocalL2BlockNumber,
690
+ latestLocalL2BlockArchive,
691
+ blocksSynchedTo,
692
+ currentL1BlockNumber,
693
+ ...status
694
+ });
695
+ await this.store.setBlockSynchedL1BlockNumber(targetL1BlockNumber);
696
+ } else {
697
+ this.log.trace(`No new blocks behind L1 sync point to retrieve.`, {
698
+ latestLocalL2BlockNumber,
699
+ pendingBlockNumber
700
+ });
701
+ }
702
+ }
703
+ /** Resumes the archiver after a stop. */ resume() {
704
+ if (!this.runningPromise) {
705
+ throw new Error(`Archiver was never started`);
706
+ }
707
+ if (this.runningPromise.isRunning()) {
708
+ this.log.warn(`Archiver already running`);
709
+ }
710
+ this.log.info(`Restarting archiver`);
711
+ this.runningPromise.start();
712
+ }
392
713
  /**
393
714
  * Stops the archiver.
394
715
  * @returns A promise signalling completion of the stop process.
@@ -398,9 +719,17 @@ import { ArchiverInstrumentation } from './instrumentation.js';
398
719
  this.log.info('Stopped.');
399
720
  return Promise.resolve();
400
721
  }
722
+ backupTo(destPath) {
723
+ return this.dataStore.backupTo(destPath);
724
+ }
401
725
  getL1Constants() {
402
726
  return Promise.resolve(this.l1constants);
403
727
  }
728
+ getGenesisValues() {
729
+ return Promise.resolve({
730
+ genesisArchiveRoot: this.l1constants.genesisArchiveRoot
731
+ });
732
+ }
404
733
  getRollupAddress() {
405
734
  return Promise.resolve(this.l1Addresses.rollupAddress);
406
735
  }
@@ -408,24 +737,16 @@ import { ArchiverInstrumentation } from './instrumentation.js';
408
737
  return Promise.resolve(this.l1Addresses.registryAddress);
409
738
  }
410
739
  getL1BlockNumber() {
411
- const l1BlockNumber = this.l1BlockNumber;
412
- if (!l1BlockNumber) {
413
- throw new Error('L1 block number not yet available. Complete an initial sync first.');
414
- }
415
- return l1BlockNumber;
740
+ return this.l1BlockNumber;
416
741
  }
417
742
  getL1Timestamp() {
418
- const l1Timestamp = this.l1Timestamp;
419
- if (!l1Timestamp) {
420
- throw new Error('L1 timestamp not yet available. Complete an initial sync first.');
421
- }
422
- return l1Timestamp;
743
+ return Promise.resolve(this.l1Timestamp);
423
744
  }
424
745
  getL2SlotNumber() {
425
- return Promise.resolve(getSlotAtTimestamp(this.getL1Timestamp(), this.l1constants));
746
+ return Promise.resolve(this.l1Timestamp === undefined ? undefined : getSlotAtTimestamp(this.l1Timestamp, this.l1constants));
426
747
  }
427
748
  getL2EpochNumber() {
428
- return Promise.resolve(getEpochNumberAtTimestamp(this.getL1Timestamp(), this.l1constants));
749
+ return Promise.resolve(this.l1Timestamp === undefined ? undefined : getEpochNumberAtTimestamp(this.l1Timestamp, this.l1constants));
429
750
  }
430
751
  async getBlocksForEpoch(epochNumber) {
431
752
  const [start, end] = getSlotRangeForEpoch(epochNumber, this.l1constants);
@@ -442,6 +763,22 @@ import { ArchiverInstrumentation } from './instrumentation.js';
442
763
  }
443
764
  return blocks.reverse();
444
765
  }
766
+ async getBlockHeadersForEpoch(epochNumber) {
767
+ const [start, end] = getSlotRangeForEpoch(epochNumber, this.l1constants);
768
+ const blocks = [];
769
+ // Walk the list of blocks backwards and filter by slots matching the requested epoch.
770
+ // We'll typically ask for blocks for a very recent epoch, so we shouldn't need an index here.
771
+ let number = await this.store.getSynchedL2BlockNumber();
772
+ let header = await this.getBlockHeader(number);
773
+ const slot = (b)=>b.globalVariables.slotNumber.toBigInt();
774
+ while(header && slot(header) >= start){
775
+ if (slot(header) <= end) {
776
+ blocks.push(header);
777
+ }
778
+ header = await this.getBlockHeader(--number);
779
+ }
780
+ return blocks.reverse();
781
+ }
445
782
  async isEpochComplete(epochNumber) {
446
783
  // The epoch is complete if the current L2 block is the last one in the epoch (or later)
447
784
  const header = await this.getBlockHeader('latest');
@@ -466,15 +803,33 @@ import { ArchiverInstrumentation } from './instrumentation.js';
466
803
  const leeway = 1n;
467
804
  return l1Timestamp + leeway >= endTimestamp;
468
805
  }
806
+ /** Returns whether the archiver has completed an initial sync run successfully. */ isInitialSyncComplete() {
807
+ return this.initialSyncComplete;
808
+ }
469
809
  /**
470
810
  * Gets up to `limit` amount of L2 blocks starting from `from`.
471
811
  * @param from - Number of the first block to return (inclusive).
472
812
  * @param limit - The number of blocks to return.
473
813
  * @param proven - If true, only return blocks that have been proven.
474
814
  * @returns The requested L2 blocks.
475
- */ async getBlocks(from, limit, proven) {
815
+ */ getBlocks(from, limit, proven) {
816
+ return this.getPublishedBlocks(from, limit, proven).then((blocks)=>blocks.map((b)=>b.block));
817
+ }
818
+ /** Equivalent to getBlocks but includes publish data. */ async getPublishedBlocks(from, limit, proven) {
476
819
  const limitWithProven = proven ? Math.min(limit, Math.max(await this.store.getProvenL2BlockNumber() - from + 1, 0)) : limit;
477
- return limitWithProven === 0 ? [] : (await this.store.getBlocks(from, limitWithProven)).map((b)=>b.data);
820
+ return limitWithProven === 0 ? [] : await this.store.getPublishedBlocks(from, limitWithProven);
821
+ }
822
+ getPublishedBlockByHash(blockHash) {
823
+ return this.store.getPublishedBlockByHash(blockHash);
824
+ }
825
+ getPublishedBlockByArchive(archive) {
826
+ return this.store.getPublishedBlockByArchive(archive);
827
+ }
828
+ getBlockHeaderByHash(blockHash) {
829
+ return this.store.getBlockHeaderByHash(blockHash);
830
+ }
831
+ getBlockHeaderByArchive(archive) {
832
+ return this.store.getBlockHeaderByArchive(archive);
478
833
  }
479
834
  /**
480
835
  * Gets an l2 block.
@@ -485,11 +840,11 @@ import { ArchiverInstrumentation } from './instrumentation.js';
485
840
  if (number < 0) {
486
841
  number = await this.store.getSynchedL2BlockNumber();
487
842
  }
488
- if (number == 0) {
843
+ if (number === 0) {
489
844
  return undefined;
490
845
  }
491
- const blocks = await this.store.getBlocks(number, 1);
492
- return blocks.length === 0 ? undefined : blocks[0].data;
846
+ const publishedBlock = await this.store.getPublishedBlock(number);
847
+ return publishedBlock?.block;
493
848
  }
494
849
  async getBlockHeader(number) {
495
850
  if (number === 'latest') {
@@ -508,22 +863,6 @@ import { ArchiverInstrumentation } from './instrumentation.js';
508
863
  return this.store.getSettledTxReceipt(txHash);
509
864
  }
510
865
  /**
511
- * Gets the public function data for a contract.
512
- * @param address - The contract address containing the function to fetch.
513
- * @param selector - The function selector of the function to fetch.
514
- * @returns The public function data (if found).
515
- */ async getPublicFunction(address, selector) {
516
- const instance = await this.getContract(address);
517
- if (!instance) {
518
- throw new Error(`Contract ${address.toString()} not found`);
519
- }
520
- const contractClass = await this.getContractClass(instance.currentContractClassId);
521
- if (!contractClass) {
522
- throw new Error(`Contract class ${instance.currentContractClassId.toString()} for ${address.toString()} not found`);
523
- }
524
- return contractClass.publicFunctions.find((f)=>f.selector.equals(selector));
525
- }
526
- /**
527
866
  * Retrieves all private logs from up to `limit` blocks, starting from the block number `from`.
528
867
  * @param from - The block number from which to begin retrieving logs.
529
868
  * @param limit - The maximum number of blocks to retrieve logs from.
@@ -540,15 +879,6 @@ import { ArchiverInstrumentation } from './instrumentation.js';
540
879
  return this.store.getLogsByTags(tags);
541
880
  }
542
881
  /**
543
- * Returns the provided nullifier indexes scoped to the block
544
- * they were first included in, or undefined if they're not present in the tree
545
- * @param blockNumber Max block number to search for the nullifiers
546
- * @param nullifiers Nullifiers to get
547
- * @returns The block scoped indexes of the provided nullifiers, or undefined if the nullifier doesn't exist in the tree
548
- */ findNullifiersIndexesWithBlock(blockNumber, nullifiers) {
549
- return this.store.findNullifiersIndexesWithBlock(blockNumber, nullifiers);
550
- }
551
- /**
552
882
  * Gets public logs based on the provided filter.
553
883
  * @param filter - The filter to apply to the logs.
554
884
  * @returns The requested logs.
@@ -580,8 +910,16 @@ import { ArchiverInstrumentation } from './instrumentation.js';
580
910
  getBytecodeCommitment(id) {
581
911
  return this.store.getBytecodeCommitment(id);
582
912
  }
583
- getContract(address) {
584
- return this.store.getContractInstance(address);
913
+ async getContract(address, maybeTimestamp) {
914
+ let timestamp;
915
+ if (maybeTimestamp === undefined) {
916
+ const latestBlockHeader = await this.getBlockHeader('latest');
917
+ // If we get undefined block header, it means that the archiver has not yet synced any block so we default to 0.
918
+ timestamp = latestBlockHeader ? latestBlockHeader.globalVariables.timestamp : 0n;
919
+ } else {
920
+ timestamp = maybeTimestamp;
921
+ }
922
+ return this.store.getContractInstance(address, timestamp);
585
923
  }
586
924
  /**
587
925
  * Gets L1 to L2 message (to be) included in a given block.
@@ -600,29 +938,33 @@ import { ArchiverInstrumentation } from './instrumentation.js';
600
938
  getContractClassIds() {
601
939
  return this.store.getContractClassIds();
602
940
  }
603
- // TODO(#10007): Remove this method
604
- async addContractClass(contractClass) {
605
- await this.store.addContractClasses([
606
- contractClass
607
- ], [
608
- await computePublicBytecodeCommitment(contractClass.packedBytecode)
609
- ], 0);
610
- return;
941
+ registerContractFunctionSignatures(signatures) {
942
+ return this.store.registerContractFunctionSignatures(signatures);
943
+ }
944
+ getDebugFunctionName(address, selector) {
945
+ return this.store.getDebugFunctionName(address, selector);
611
946
  }
612
- registerContractFunctionSignatures(address, signatures) {
613
- return this.store.registerContractFunctionSignatures(address, signatures);
947
+ async getPendingChainValidationStatus() {
948
+ return await this.store.getPendingChainValidationStatus() ?? {
949
+ valid: true
950
+ };
614
951
  }
615
- getContractFunctionName(address, selector) {
616
- return this.store.getContractFunctionName(address, selector);
952
+ isPendingChainInvalid() {
953
+ return this.getPendingChainValidationStatus().then((status)=>!status.valid);
617
954
  }
618
955
  async getL2Tips() {
619
956
  const [latestBlockNumber, provenBlockNumber] = await Promise.all([
620
957
  this.getBlockNumber(),
621
958
  this.getProvenBlockNumber()
622
959
  ]);
623
- const [latestBlockHeader, provenBlockHeader] = await Promise.all([
960
+ // TODO(#13569): Compute proper finalized block number based on L1 finalized block.
961
+ // We just force it 2 epochs worth of proven data for now.
962
+ // NOTE: update end-to-end/src/e2e_epochs/epochs_empty_blocks.test.ts as that uses finalized blocks in computations
963
+ const finalizedBlockNumber = Math.max(provenBlockNumber - this.l1constants.epochDuration * 2, 0);
964
+ const [latestBlockHeader, provenBlockHeader, finalizedBlockHeader] = await Promise.all([
624
965
  latestBlockNumber > 0 ? this.getBlockHeader(latestBlockNumber) : undefined,
625
- provenBlockNumber > 0 ? this.getBlockHeader(provenBlockNumber) : undefined
966
+ provenBlockNumber > 0 ? this.getBlockHeader(provenBlockNumber) : undefined,
967
+ finalizedBlockNumber > 0 ? this.getBlockHeader(finalizedBlockNumber) : undefined
626
968
  ]);
627
969
  if (latestBlockNumber > 0 && !latestBlockHeader) {
628
970
  throw new Error(`Failed to retrieve latest block header for block ${latestBlockNumber}`);
@@ -630,9 +972,12 @@ import { ArchiverInstrumentation } from './instrumentation.js';
630
972
  if (provenBlockNumber > 0 && !provenBlockHeader) {
631
973
  throw new Error(`Failed to retrieve proven block header for block ${provenBlockNumber} (latest block is ${latestBlockNumber})`);
632
974
  }
975
+ if (finalizedBlockNumber > 0 && !finalizedBlockHeader) {
976
+ throw new Error(`Failed to retrieve finalized block header for block ${finalizedBlockNumber} (latest block is ${latestBlockNumber})`);
977
+ }
633
978
  const latestBlockHeaderHash = await latestBlockHeader?.hash();
634
979
  const provenBlockHeaderHash = await provenBlockHeader?.hash();
635
- const finalizedBlockHeaderHash = await provenBlockHeader?.hash();
980
+ const finalizedBlockHeaderHash = await finalizedBlockHeader?.hash();
636
981
  return {
637
982
  latest: {
638
983
  number: latestBlockNumber,
@@ -643,11 +988,46 @@ import { ArchiverInstrumentation } from './instrumentation.js';
643
988
  hash: provenBlockHeaderHash?.toString()
644
989
  },
645
990
  finalized: {
646
- number: provenBlockNumber,
991
+ number: finalizedBlockNumber,
647
992
  hash: finalizedBlockHeaderHash?.toString()
648
993
  }
649
994
  };
650
995
  }
996
+ async rollbackTo(targetL2BlockNumber) {
997
+ const currentBlocks = await this.getL2Tips();
998
+ const currentL2Block = currentBlocks.latest.number;
999
+ const currentProvenBlock = currentBlocks.proven.number;
1000
+ // const currentFinalizedBlock = currentBlocks.finalized.number;
1001
+ if (targetL2BlockNumber >= currentL2Block) {
1002
+ throw new Error(`Target L2 block ${targetL2BlockNumber} must be less than current L2 block ${currentL2Block}`);
1003
+ }
1004
+ const blocksToUnwind = currentL2Block - targetL2BlockNumber;
1005
+ const targetL2Block = await this.store.getPublishedBlock(targetL2BlockNumber);
1006
+ if (!targetL2Block) {
1007
+ throw new Error(`Target L2 block ${targetL2BlockNumber} not found`);
1008
+ }
1009
+ const targetL1BlockNumber = targetL2Block.l1.blockNumber;
1010
+ const targetL1BlockHash = await this.getL1BlockHash(targetL1BlockNumber);
1011
+ this.log.info(`Unwinding ${blocksToUnwind} blocks from L2 block ${currentL2Block}`);
1012
+ await this.store.unwindBlocks(currentL2Block, blocksToUnwind);
1013
+ this.log.info(`Unwinding L1 to L2 messages to ${targetL2BlockNumber}`);
1014
+ await this.store.rollbackL1ToL2MessagesToL2Block(targetL2BlockNumber);
1015
+ this.log.info(`Setting L1 syncpoints to ${targetL1BlockNumber}`);
1016
+ await this.store.setBlockSynchedL1BlockNumber(targetL1BlockNumber);
1017
+ await this.store.setMessageSynchedL1Block({
1018
+ l1BlockNumber: targetL1BlockNumber,
1019
+ l1BlockHash: targetL1BlockHash
1020
+ });
1021
+ if (targetL2BlockNumber < currentProvenBlock) {
1022
+ this.log.info(`Clearing proven L2 block number`);
1023
+ await this.store.setProvenL2BlockNumber(0);
1024
+ }
1025
+ // TODO(palla/reorg): Set the finalized block when we add support for it.
1026
+ // if (targetL2BlockNumber < currentFinalizedBlock) {
1027
+ // this.log.info(`Clearing finalized L2 block number`);
1028
+ // await this.store.setFinalizedL2BlockNumber(0);
1029
+ // }
1030
+ }
651
1031
  }
652
1032
  _ts_decorate([
653
1033
  trackSpan('Archiver.sync', (initialRun)=>({
@@ -664,23 +1044,19 @@ var Operation = /*#__PURE__*/ function(Operation) {
664
1044
  *
665
1045
  * I would have preferred to not have this type. But it is useful for handling the logic that any
666
1046
  * store would need to include otherwise while exposing fewer functions and logic directly to the archiver.
667
- */ class ArchiverStoreHelper {
1047
+ */ export class ArchiverStoreHelper {
668
1048
  store;
669
1049
  #log;
670
1050
  constructor(store){
671
1051
  this.store = store;
672
1052
  this.#log = createLogger('archiver:block-helper');
673
1053
  }
674
- // TODO(#10007): Remove this method
675
- addContractClasses(contractClasses, bytecodeCommitments, blockNum) {
676
- return this.store.addContractClasses(contractClasses, bytecodeCommitments, blockNum);
677
- }
678
1054
  /**
679
- * Extracts and stores contract classes out of ContractClassRegistered events emitted by the class registerer contract.
1055
+ * Extracts and stores contract classes out of ContractClassPublished events emitted by the class registry contract.
680
1056
  * @param allLogs - All logs emitted in a bunch of blocks.
681
- */ async #updateRegisteredContractClasses(allLogs, blockNum, operation) {
682
- const contractClassRegisteredEvents = allLogs.filter((log)=>ContractClassRegisteredEvent.isContractClassRegisteredEvent(log)).map((log)=>ContractClassRegisteredEvent.fromLog(log));
683
- const contractClasses = await Promise.all(contractClassRegisteredEvents.map((e)=>e.toContractClassPublic()));
1057
+ */ async #updatePublishedContractClasses(allLogs, blockNum, operation) {
1058
+ const contractClassPublishedEvents = allLogs.filter((log)=>ContractClassPublishedEvent.isContractClassPublishedEvent(log)).map((log)=>ContractClassPublishedEvent.fromLog(log));
1059
+ const contractClasses = await Promise.all(contractClassPublishedEvents.map((e)=>e.toContractClassPublic()));
684
1060
  if (contractClasses.length > 0) {
685
1061
  contractClasses.forEach((c)=>this.#log.verbose(`${Operation[operation]} contract class ${c.id.toString()}`));
686
1062
  if (operation == 0) {
@@ -694,10 +1070,10 @@ var Operation = /*#__PURE__*/ function(Operation) {
694
1070
  return true;
695
1071
  }
696
1072
  /**
697
- * Extracts and stores contract instances out of ContractInstanceDeployed events emitted by the canonical deployer contract.
1073
+ * Extracts and stores contract instances out of ContractInstancePublished events emitted by the canonical deployer contract.
698
1074
  * @param allLogs - All logs emitted in a bunch of blocks.
699
1075
  */ async #updateDeployedContractInstances(allLogs, blockNum, operation) {
700
- const contractInstances = allLogs.filter((log)=>ContractInstanceDeployedEvent.isContractInstanceDeployedEvent(log)).map((log)=>ContractInstanceDeployedEvent.fromLog(log)).map((e)=>e.toContractInstance());
1076
+ const contractInstances = allLogs.filter((log)=>ContractInstancePublishedEvent.isContractInstancePublishedEvent(log)).map((log)=>ContractInstancePublishedEvent.fromLog(log)).map((e)=>e.toContractInstance());
701
1077
  if (contractInstances.length > 0) {
702
1078
  contractInstances.forEach((c)=>this.#log.verbose(`${Operation[operation]} contract instance at ${c.address.toString()}`));
703
1079
  if (operation == 0) {
@@ -709,22 +1085,24 @@ var Operation = /*#__PURE__*/ function(Operation) {
709
1085
  return true;
710
1086
  }
711
1087
  /**
712
- * Extracts and stores contract instances out of ContractInstanceDeployed events emitted by the canonical deployer contract.
1088
+ * Extracts and stores contract instances out of ContractInstancePublished events emitted by the canonical deployer contract.
713
1089
  * @param allLogs - All logs emitted in a bunch of blocks.
714
- */ async #updateUpdatedContractInstances(allLogs, blockNum, operation) {
1090
+ * @param timestamp - Timestamp at which the updates were scheduled.
1091
+ * @param operation - The operation to perform on the contract instance updates (Store or Delete).
1092
+ */ async #updateUpdatedContractInstances(allLogs, timestamp, operation) {
715
1093
  const contractUpdates = allLogs.filter((log)=>ContractInstanceUpdatedEvent.isContractInstanceUpdatedEvent(log)).map((log)=>ContractInstanceUpdatedEvent.fromLog(log)).map((e)=>e.toContractInstanceUpdate());
716
1094
  if (contractUpdates.length > 0) {
717
1095
  contractUpdates.forEach((c)=>this.#log.verbose(`${Operation[operation]} contract instance update at ${c.address.toString()}`));
718
1096
  if (operation == 0) {
719
- return await this.store.addContractInstanceUpdates(contractUpdates, blockNum);
1097
+ return await this.store.addContractInstanceUpdates(contractUpdates, timestamp);
720
1098
  } else if (operation == 1) {
721
- return await this.store.deleteContractInstanceUpdates(contractUpdates, blockNum);
1099
+ return await this.store.deleteContractInstanceUpdates(contractUpdates, timestamp);
722
1100
  }
723
1101
  }
724
1102
  return true;
725
1103
  }
726
1104
  /**
727
- * Stores the functions that was broadcasted individually
1105
+ * Stores the functions that were broadcasted individually
728
1106
  *
729
1107
  * @dev Beware that there is not a delete variant of this, since they are added to contract classes
730
1108
  * and will be deleted as part of the class if needed.
@@ -733,13 +1111,13 @@ var Operation = /*#__PURE__*/ function(Operation) {
733
1111
  * @param _blockNum - The block number
734
1112
  * @returns
735
1113
  */ async #storeBroadcastedIndividualFunctions(allLogs, _blockNum) {
736
- // Filter out private and unconstrained function broadcast events
1114
+ // Filter out private and utility function broadcast events
737
1115
  const privateFnEvents = allLogs.filter((log)=>PrivateFunctionBroadcastedEvent.isPrivateFunctionBroadcastedEvent(log)).map((log)=>PrivateFunctionBroadcastedEvent.fromLog(log));
738
- const unconstrainedFnEvents = allLogs.filter((log)=>UnconstrainedFunctionBroadcastedEvent.isUnconstrainedFunctionBroadcastedEvent(log)).map((log)=>UnconstrainedFunctionBroadcastedEvent.fromLog(log));
1116
+ const utilityFnEvents = allLogs.filter((log)=>UtilityFunctionBroadcastedEvent.isUtilityFunctionBroadcastedEvent(log)).map((log)=>UtilityFunctionBroadcastedEvent.fromLog(log));
739
1117
  // Group all events by contract class id
740
1118
  for (const [classIdString, classEvents] of Object.entries(groupBy([
741
1119
  ...privateFnEvents,
742
- ...unconstrainedFnEvents
1120
+ ...utilityFnEvents
743
1121
  ], (e)=>e.contractClassId.toString()))){
744
1122
  const contractClassId = Fr.fromHexString(classIdString);
745
1123
  const contractClass = await this.getContractClass(contractClassId);
@@ -747,21 +1125,21 @@ var Operation = /*#__PURE__*/ function(Operation) {
747
1125
  this.#log.warn(`Skipping broadcasted functions as contract class ${contractClassId.toString()} was not found`);
748
1126
  continue;
749
1127
  }
750
- // Split private and unconstrained functions, and filter out invalid ones
1128
+ // Split private and utility functions, and filter out invalid ones
751
1129
  const allFns = classEvents.map((e)=>e.toFunctionWithMembershipProof());
752
- const privateFns = allFns.filter((fn)=>'unconstrainedFunctionsArtifactTreeRoot' in fn);
753
- const unconstrainedFns = allFns.filter((fn)=>'privateFunctionsArtifactTreeRoot' in fn);
1130
+ const privateFns = allFns.filter((fn)=>'utilityFunctionsTreeRoot' in fn);
1131
+ const utilityFns = allFns.filter((fn)=>'privateFunctionsArtifactTreeRoot' in fn);
754
1132
  const privateFunctionsWithValidity = await Promise.all(privateFns.map(async (fn)=>({
755
1133
  fn,
756
1134
  valid: await isValidPrivateFunctionMembershipProof(fn, contractClass)
757
1135
  })));
758
1136
  const validPrivateFns = privateFunctionsWithValidity.filter(({ valid })=>valid).map(({ fn })=>fn);
759
- const unconstrainedFunctionsWithValidity = await Promise.all(unconstrainedFns.map(async (fn)=>({
1137
+ const utilityFunctionsWithValidity = await Promise.all(utilityFns.map(async (fn)=>({
760
1138
  fn,
761
- valid: await isValidUnconstrainedFunctionMembershipProof(fn, contractClass)
1139
+ valid: await isValidUtilityFunctionMembershipProof(fn, contractClass)
762
1140
  })));
763
- const validUnconstrainedFns = unconstrainedFunctionsWithValidity.filter(({ valid })=>valid).map(({ fn })=>fn);
764
- const validFnCount = validPrivateFns.length + validUnconstrainedFns.length;
1141
+ const validUtilityFns = utilityFunctionsWithValidity.filter(({ valid })=>valid).map(({ fn })=>fn);
1142
+ const validFnCount = validPrivateFns.length + validUtilityFns.length;
765
1143
  if (validFnCount !== allFns.length) {
766
1144
  this.#log.warn(`Skipping ${allFns.length - validFnCount} invalid functions`);
767
1145
  }
@@ -769,62 +1147,90 @@ var Operation = /*#__PURE__*/ function(Operation) {
769
1147
  if (validFnCount > 0) {
770
1148
  this.#log.verbose(`Storing ${validFnCount} functions for contract class ${contractClassId.toString()}`);
771
1149
  }
772
- return await this.store.addFunctions(contractClassId, validPrivateFns, validUnconstrainedFns);
1150
+ return await this.store.addFunctions(contractClassId, validPrivateFns, validUtilityFns);
773
1151
  }
774
1152
  return true;
775
1153
  }
776
- async addBlocks(blocks) {
777
- const opResults = await Promise.all([
778
- this.store.addLogs(blocks.map((block)=>block.data)),
779
- // Unroll all logs emitted during the retrieved blocks and extract any contract classes and instances from them
780
- ...blocks.map(async (block)=>{
781
- const contractClassLogs = block.data.body.txEffects.flatMap((txEffect)=>txEffect.contractClassLogs);
782
- // ContractInstanceDeployed event logs are broadcast in privateLogs.
783
- const privateLogs = block.data.body.txEffects.flatMap((txEffect)=>txEffect.privateLogs);
784
- const publicLogs = block.data.body.txEffects.flatMap((txEffect)=>txEffect.publicLogs);
785
- return (await Promise.all([
786
- this.#updateRegisteredContractClasses(contractClassLogs, block.data.number, 0),
787
- this.#updateDeployedContractInstances(privateLogs, block.data.number, 0),
788
- this.#updateUpdatedContractInstances(publicLogs, block.data.number, 0),
789
- this.#storeBroadcastedIndividualFunctions(contractClassLogs, block.data.number)
790
- ])).every(Boolean);
791
- }),
792
- this.store.addNullifiers(blocks.map((block)=>block.data)),
793
- this.store.addBlocks(blocks)
794
- ]);
795
- return opResults.every(Boolean);
1154
+ addBlocks(blocks, pendingChainValidationStatus) {
1155
+ // Add the blocks to the store. Store will throw if the blocks are not in order, there are gaps,
1156
+ // or if the previous block is not in the store.
1157
+ return this.store.transactionAsync(async ()=>{
1158
+ await this.store.addBlocks(blocks);
1159
+ const opResults = await Promise.all([
1160
+ // Update the pending chain validation status if provided
1161
+ pendingChainValidationStatus && this.store.setPendingChainValidationStatus(pendingChainValidationStatus),
1162
+ // Add any logs emitted during the retrieved blocks
1163
+ this.store.addLogs(blocks.map((block)=>block.block)),
1164
+ // Unroll all logs emitted during the retrieved blocks and extract any contract classes and instances from them
1165
+ ...blocks.map(async (block)=>{
1166
+ const contractClassLogs = block.block.body.txEffects.flatMap((txEffect)=>txEffect.contractClassLogs);
1167
+ // ContractInstancePublished event logs are broadcast in privateLogs.
1168
+ const privateLogs = block.block.body.txEffects.flatMap((txEffect)=>txEffect.privateLogs);
1169
+ const publicLogs = block.block.body.txEffects.flatMap((txEffect)=>txEffect.publicLogs);
1170
+ return (await Promise.all([
1171
+ this.#updatePublishedContractClasses(contractClassLogs, block.block.number, 0),
1172
+ this.#updateDeployedContractInstances(privateLogs, block.block.number, 0),
1173
+ this.#updateUpdatedContractInstances(publicLogs, block.block.header.globalVariables.timestamp, 0),
1174
+ this.#storeBroadcastedIndividualFunctions(contractClassLogs, block.block.number)
1175
+ ])).every(Boolean);
1176
+ })
1177
+ ]);
1178
+ return opResults.every(Boolean);
1179
+ });
796
1180
  }
797
1181
  async unwindBlocks(from, blocksToUnwind) {
798
1182
  const last = await this.getSynchedL2BlockNumber();
799
1183
  if (from != last) {
800
- throw new Error(`Can only remove from the tip`);
1184
+ throw new Error(`Cannot unwind blocks from block ${from} when the last block is ${last}`);
1185
+ }
1186
+ if (blocksToUnwind <= 0) {
1187
+ throw new Error(`Cannot unwind ${blocksToUnwind} blocks`);
801
1188
  }
802
1189
  // from - blocksToUnwind = the new head, so + 1 for what we need to remove
803
- const blocks = await this.getBlocks(from - blocksToUnwind + 1, blocksToUnwind);
1190
+ const blocks = await this.getPublishedBlocks(from - blocksToUnwind + 1, blocksToUnwind);
804
1191
  const opResults = await Promise.all([
1192
+ // Prune rolls back to the last proven block, which is by definition valid
1193
+ this.store.setPendingChainValidationStatus({
1194
+ valid: true
1195
+ }),
805
1196
  // Unroll all logs emitted during the retrieved blocks and extract any contract classes and instances from them
806
1197
  ...blocks.map(async (block)=>{
807
- const contractClassLogs = block.data.body.txEffects.flatMap((txEffect)=>txEffect.contractClassLogs);
808
- // ContractInstanceDeployed event logs are broadcast in privateLogs.
809
- const privateLogs = block.data.body.txEffects.flatMap((txEffect)=>txEffect.privateLogs);
810
- const publicLogs = block.data.body.txEffects.flatMap((txEffect)=>txEffect.publicLogs);
1198
+ const contractClassLogs = block.block.body.txEffects.flatMap((txEffect)=>txEffect.contractClassLogs);
1199
+ // ContractInstancePublished event logs are broadcast in privateLogs.
1200
+ const privateLogs = block.block.body.txEffects.flatMap((txEffect)=>txEffect.privateLogs);
1201
+ const publicLogs = block.block.body.txEffects.flatMap((txEffect)=>txEffect.publicLogs);
811
1202
  return (await Promise.all([
812
- this.#updateRegisteredContractClasses(contractClassLogs, block.data.number, 1),
813
- this.#updateDeployedContractInstances(privateLogs, block.data.number, 1),
814
- this.#updateUpdatedContractInstances(publicLogs, block.data.number, 1)
1203
+ this.#updatePublishedContractClasses(contractClassLogs, block.block.number, 1),
1204
+ this.#updateDeployedContractInstances(privateLogs, block.block.number, 1),
1205
+ this.#updateUpdatedContractInstances(publicLogs, block.block.header.globalVariables.timestamp, 1)
815
1206
  ])).every(Boolean);
816
1207
  }),
817
- this.store.deleteLogs(blocks.map((b)=>b.data)),
1208
+ this.store.deleteLogs(blocks.map((b)=>b.block)),
818
1209
  this.store.unwindBlocks(from, blocksToUnwind)
819
1210
  ]);
820
1211
  return opResults.every(Boolean);
821
1212
  }
822
- getBlocks(from, limit) {
823
- return this.store.getBlocks(from, limit);
1213
+ getPublishedBlocks(from, limit) {
1214
+ return this.store.getPublishedBlocks(from, limit);
1215
+ }
1216
+ getPublishedBlock(number) {
1217
+ return this.store.getPublishedBlock(number);
1218
+ }
1219
+ getPublishedBlockByHash(blockHash) {
1220
+ return this.store.getPublishedBlockByHash(blockHash);
1221
+ }
1222
+ getPublishedBlockByArchive(archive) {
1223
+ return this.store.getPublishedBlockByArchive(archive);
824
1224
  }
825
1225
  getBlockHeaders(from, limit) {
826
1226
  return this.store.getBlockHeaders(from, limit);
827
1227
  }
1228
+ getBlockHeaderByHash(blockHash) {
1229
+ return this.store.getBlockHeaderByHash(blockHash);
1230
+ }
1231
+ getBlockHeaderByArchive(archive) {
1232
+ return this.store.getBlockHeaderByArchive(archive);
1233
+ }
828
1234
  getTxEffect(txHash) {
829
1235
  return this.store.getTxEffect(txHash);
830
1236
  }
@@ -843,11 +1249,8 @@ var Operation = /*#__PURE__*/ function(Operation) {
843
1249
  getPrivateLogs(from, limit) {
844
1250
  return this.store.getPrivateLogs(from, limit);
845
1251
  }
846
- getLogsByTags(tags) {
847
- return this.store.getLogsByTags(tags);
848
- }
849
- findNullifiersIndexesWithBlock(blockNumber, nullifiers) {
850
- return this.store.findNullifiersIndexesWithBlock(blockNumber, nullifiers);
1252
+ getLogsByTags(tags, logsPerTag) {
1253
+ return this.store.getLogsByTags(tags, logsPerTag);
851
1254
  }
852
1255
  getPublicLogs(filter) {
853
1256
  return this.store.getPublicLogs(filter);
@@ -867,8 +1270,8 @@ var Operation = /*#__PURE__*/ function(Operation) {
867
1270
  setBlockSynchedL1BlockNumber(l1BlockNumber) {
868
1271
  return this.store.setBlockSynchedL1BlockNumber(l1BlockNumber);
869
1272
  }
870
- setMessageSynchedL1BlockNumber(l1BlockNumber) {
871
- return this.store.setMessageSynchedL1BlockNumber(l1BlockNumber);
1273
+ setMessageSynchedL1Block(l1Block) {
1274
+ return this.store.setMessageSynchedL1Block(l1Block);
872
1275
  }
873
1276
  getSynchPoint() {
874
1277
  return this.store.getSynchPoint();
@@ -879,17 +1282,17 @@ var Operation = /*#__PURE__*/ function(Operation) {
879
1282
  getBytecodeCommitment(contractClassId) {
880
1283
  return this.store.getBytecodeCommitment(contractClassId);
881
1284
  }
882
- getContractInstance(address) {
883
- return this.store.getContractInstance(address);
1285
+ getContractInstance(address, timestamp) {
1286
+ return this.store.getContractInstance(address, timestamp);
884
1287
  }
885
1288
  getContractClassIds() {
886
1289
  return this.store.getContractClassIds();
887
1290
  }
888
- registerContractFunctionSignatures(address, signatures) {
889
- return this.store.registerContractFunctionSignatures(address, signatures);
1291
+ registerContractFunctionSignatures(signatures) {
1292
+ return this.store.registerContractFunctionSignatures(signatures);
890
1293
  }
891
- getContractFunctionName(address, selector) {
892
- return this.store.getContractFunctionName(address, selector);
1294
+ getDebugFunctionName(address, selector) {
1295
+ return this.store.getDebugFunctionName(address, selector);
893
1296
  }
894
1297
  getTotalL1ToL2MessageCount() {
895
1298
  return this.store.getTotalL1ToL2MessageCount();
@@ -897,4 +1300,23 @@ var Operation = /*#__PURE__*/ function(Operation) {
897
1300
  estimateSize() {
898
1301
  return this.store.estimateSize();
899
1302
  }
1303
+ rollbackL1ToL2MessagesToL2Block(targetBlockNumber) {
1304
+ return this.store.rollbackL1ToL2MessagesToL2Block(targetBlockNumber);
1305
+ }
1306
+ iterateL1ToL2Messages(range = {}) {
1307
+ return this.store.iterateL1ToL2Messages(range);
1308
+ }
1309
+ removeL1ToL2Messages(startIndex) {
1310
+ return this.store.removeL1ToL2Messages(startIndex);
1311
+ }
1312
+ getLastL1ToL2Message() {
1313
+ return this.store.getLastL1ToL2Message();
1314
+ }
1315
+ getPendingChainValidationStatus() {
1316
+ return this.store.getPendingChainValidationStatus();
1317
+ }
1318
+ setPendingChainValidationStatus(status) {
1319
+ this.#log.debug(`Setting pending chain validation status to valid ${status?.valid}`, status);
1320
+ return this.store.setPendingChainValidationStatus(status);
1321
+ }
900
1322
  }