envio 3.8.0 → 3.9.0-resolvers.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/evm.schema.json +7 -0
- package/fuel.schema.json +7 -0
- package/index.d.ts +433 -109
- package/index.js +10 -0
- package/package.json +6 -7
- package/src/AddressRows.res +121 -0
- package/src/AddressRows.res.mjs +143 -0
- package/src/Batch.res +11 -0
- package/src/Batch.res.mjs +2 -1
- package/src/Bin.res +47 -0
- package/src/Bin.res.mjs +34 -1
- package/src/ChainState.res +102 -100
- package/src/ChainState.res.mjs +81 -67
- package/src/ChainState.resi +28 -15
- package/src/Config.res +43 -122
- package/src/Config.res.mjs +33 -75
- package/src/ContractMapping.res +66 -0
- package/src/ContractMapping.res.mjs +73 -0
- package/src/Core.res +24 -0
- package/src/Core.res.mjs +4 -0
- package/src/Ecosystem.res +7 -2
- package/src/Env.res +128 -1
- package/src/Env.res.mjs +123 -1
- package/src/EventConfigBuilder.res +169 -28
- package/src/EventConfigBuilder.res.mjs +124 -17
- package/src/FetchState.res +59 -67
- package/src/FetchState.res.mjs +34 -44
- package/src/HandlerRegister.res +6 -4
- package/src/HandlerRegister.res.mjs +8 -5
- package/src/InMemoryStore.res +22 -32
- package/src/InMemoryStore.res.mjs +12 -24
- package/src/IndexerState.res +40 -3
- package/src/IndexerState.res.mjs +30 -4
- package/src/IndexerState.resi +4 -1
- package/src/Internal.res +16 -17
- package/src/Main.res +25 -15
- package/src/Main.res.mjs +19 -14
- package/src/Metrics.res +15 -1
- package/src/Metrics.res.mjs +5 -1
- package/src/Persistence.res +42 -19
- package/src/Persistence.res.mjs +8 -5
- package/src/PgStorage.res +232 -78
- package/src/PgStorage.res.mjs +114 -72
- package/src/PruneStaleHistory.res +1 -1
- package/src/PruneStaleHistory.res.mjs +1 -1
- package/src/Rollback.res +62 -41
- package/src/Rollback.res.mjs +61 -19
- package/src/RollbackCommit.res +7 -9
- package/src/RollbackCommit.res.mjs +5 -6
- package/src/SimulateItems.res +5 -12
- package/src/SimulateItems.res.mjs +6 -10
- package/src/Sink.res +35 -26
- package/src/Sink.res.mjs +20 -16
- package/src/TestIndexer.res +95 -113
- package/src/TestIndexer.res.mjs +63 -90
- package/src/Utils.res +38 -0
- package/src/Utils.res.mjs +39 -2
- package/src/Writing.res +4 -2
- package/src/Writing.res.mjs +3 -2
- package/src/bindings/ClickHouse.res +311 -683
- package/src/bindings/ClickHouse.res.mjs +294 -416
- package/src/bindings/ClickHouseSink.res +303 -0
- package/src/bindings/ClickHouseSink.res.mjs +216 -0
- package/src/bindings/NodeJs.res +9 -0
- package/src/bindings/NodeJs.res.mjs +8 -1
- package/src/bindings/Postgres.res +9 -0
- package/src/bindings/Postgres.res.mjs +3 -0
- package/src/db/EntityHistory.res +29 -24
- package/src/db/EntityHistory.res.mjs +11 -19
- package/src/db/InternalTable.res +247 -83
- package/src/db/InternalTable.res.mjs +216 -85
- package/src/db/RollbackScope.res +29 -0
- package/src/db/RollbackScope.res.mjs +31 -0
- package/src/db/Table.res +28 -1
- package/src/db/Table.res.mjs +26 -2
- package/src/resolvers/ResolverProcess.res +809 -0
- package/src/resolvers/ResolverProcess.res.mjs +667 -0
- package/src/resolvers/ResolverQuery.res +317 -0
- package/src/resolvers/ResolverQuery.res.mjs +220 -0
- package/src/resolvers/collisions.js +108 -0
- package/src/resolvers/db.js +363 -0
- package/src/resolvers/dispatch.js +165 -0
- package/src/resolvers/errors.js +30 -0
- package/src/resolvers/graphqlSelection.js +292 -0
- package/src/resolvers/hasuraAction.js +84 -0
- package/src/resolvers/hasuraApply.js +323 -0
- package/src/resolvers/hasuraMetadata.js +119 -0
- package/src/resolvers/index.js +95 -0
- package/src/resolvers/manifest.js +370 -0
- package/src/resolvers/server.js +270 -0
- package/src/sources/AddressSet.res +0 -22
- package/src/sources/AddressStore.res +48 -88
- package/src/sources/AddressStore.res.mjs +11 -22
- package/src/sources/Evm.res +2 -0
- package/src/sources/Evm.res.mjs +2 -0
- package/src/sources/Fuel.res +2 -0
- package/src/sources/Fuel.res.mjs +2 -0
- package/src/sources/Svm.res +2 -0
- package/src/sources/Svm.res.mjs +2 -0
- package/src/sources/SvmHyperSyncClient.res +13 -107
- package/src/sources/SvmHyperSyncClient.res.mjs +2 -5
- package/src/sources/SvmHyperSyncSource.res +5 -10
- package/src/sources/SvmHyperSyncSource.res.mjs +5 -13
- package/src/tui/Tui.res +1 -1
- package/src/tui/Tui.res.mjs +1 -1
- package/svm.schema.json +17 -78
- package/src/MemoryStorage.res +0 -756
- package/src/MemoryStorage.res.mjs +0 -623
package/src/ChainState.res
CHANGED
|
@@ -62,20 +62,48 @@ type t = {
|
|
|
62
62
|
mutable progressLatencyMs: option<int>,
|
|
63
63
|
}
|
|
64
64
|
|
|
65
|
-
|
|
65
|
+
// The chain's config-declared addresses as storage rows: what a clean run
|
|
66
|
+
// writes into `envio_addresses`. The keys are encoded by the Rust codec, so the
|
|
67
|
+
// bytes a clean run writes are exactly the bytes a resume seeds from.
|
|
68
|
+
let configStorageRows = (
|
|
69
|
+
chainConfig: Config.chain,
|
|
70
|
+
~ecosystem: Ecosystem.name,
|
|
71
|
+
~contractMapping: ContractMapping.t,
|
|
72
|
+
): array<AddressRows.row> => {
|
|
66
73
|
let addresses = []
|
|
74
|
+
let contractIds = []
|
|
67
75
|
chainConfig.contracts->Array.forEach(contract => {
|
|
76
|
+
let contractId = contractMapping->ContractMapping.idOfOrThrow(contract.name)
|
|
68
77
|
contract.addresses->Array.forEach(address => {
|
|
69
|
-
addresses->Array.push(
|
|
70
|
-
|
|
71
|
-
contractName: contract.name,
|
|
72
|
-
registrationBlock: -1,
|
|
73
|
-
})
|
|
78
|
+
addresses->Array.push(address)->ignore
|
|
79
|
+
contractIds->Array.push(contractId)->ignore
|
|
74
80
|
})
|
|
75
81
|
})
|
|
76
|
-
|
|
82
|
+
Core.getAddon().encodeAddresses(
|
|
83
|
+
~ecosystem=(ecosystem :> string),
|
|
84
|
+
~addresses,
|
|
85
|
+
)->Array.mapWithIndex((address, idx) => {
|
|
86
|
+
AddressRows.chainId: chainConfig.id,
|
|
87
|
+
address,
|
|
88
|
+
contractId: contractIds->Array.getUnsafe(idx),
|
|
89
|
+
registrationBlock: AddressRows.configRegistrationBlock,
|
|
90
|
+
})
|
|
77
91
|
}
|
|
78
92
|
|
|
93
|
+
// Seeds an in-memory address table with every chain's config-declared
|
|
94
|
+
// addresses, the twin of what `PgStorage.initialize` writes in one insert.
|
|
95
|
+
let seedConfigAddresses = (
|
|
96
|
+
table: AddressRows.Table.t,
|
|
97
|
+
~chainConfigs: array<Config.chain>,
|
|
98
|
+
~ecosystem: Ecosystem.name,
|
|
99
|
+
~contractMapping: ContractMapping.t,
|
|
100
|
+
) =>
|
|
101
|
+
chainConfigs->Array.forEach(chainConfig =>
|
|
102
|
+
table->AddressRows.Table.insertMany(
|
|
103
|
+
chainConfig->configStorageRows(~ecosystem, ~contractMapping),
|
|
104
|
+
)
|
|
105
|
+
)
|
|
106
|
+
|
|
79
107
|
let validateOnEventRegistrations = (
|
|
80
108
|
~chainId: ChainId.t,
|
|
81
109
|
registrations: array<Internal.onEventRegistration>,
|
|
@@ -142,22 +170,12 @@ let make = (
|
|
|
142
170
|
}
|
|
143
171
|
}
|
|
144
172
|
|
|
145
|
-
// Every chain's contracts, not just one chain's: `context.chain.X.add` accepts
|
|
146
|
-
// any config contract, whichever chain declared it, so every chain's address
|
|
147
|
-
// store has to hold the whole set.
|
|
148
|
-
let configContractNames = (config: Config.t) => {
|
|
149
|
-
let names = Utils.Set.make()
|
|
150
|
-
config.chainMap
|
|
151
|
-
->ChainMap.values
|
|
152
|
-
->Array.forEach(chain =>
|
|
153
|
-
chain.contracts->Array.forEach(contract => names->Utils.Set.add(contract.name)->ignore)
|
|
154
|
-
)
|
|
155
|
-
names->Utils.Set.toArray
|
|
156
|
-
}
|
|
157
|
-
|
|
158
173
|
let makeInternal = (
|
|
159
174
|
~chainConfig: Config.chain,
|
|
160
|
-
~
|
|
175
|
+
~addressRows: AddressRows.seedRows,
|
|
176
|
+
// Passed rather than read off `config`, because a resume must use the mapping
|
|
177
|
+
// storage holds: the ids stored rows carry, not the ids this config derives.
|
|
178
|
+
~contractMapping: ContractMapping.t,
|
|
161
179
|
~startBlock,
|
|
162
180
|
~endBlock,
|
|
163
181
|
~firstEventBlock=None,
|
|
@@ -199,16 +217,13 @@ let makeInternal = (
|
|
|
199
217
|
let addressStore = AddressStore.make(
|
|
200
218
|
~ecosystem=config.ecosystem.name,
|
|
201
219
|
~shouldChecksum=!lowercaseAddresses,
|
|
202
|
-
~contracts=AddressStore.contractsOf(
|
|
203
|
-
~onEventRegistrations,
|
|
204
|
-
~configContractNames=config->configContractNames,
|
|
205
|
-
),
|
|
220
|
+
~contracts=AddressStore.contractsOf(~onEventRegistrations, ~contractMapping),
|
|
206
221
|
)
|
|
207
222
|
|
|
208
223
|
let fetchState = FetchState.make(
|
|
209
224
|
~maxAddrInPartition=config.maxAddrInPartition,
|
|
210
225
|
~addressStore,
|
|
211
|
-
~
|
|
226
|
+
~addressRows,
|
|
212
227
|
~progressBlockNumber,
|
|
213
228
|
~startBlock,
|
|
214
229
|
~endBlock,
|
|
@@ -295,7 +310,11 @@ let makeInternal = (
|
|
|
295
310
|
chainId,
|
|
296
311
|
endpointUrl: hypersyncUrl,
|
|
297
312
|
apiToken,
|
|
298
|
-
onEventRegistrations
|
|
313
|
+
onEventRegistrations: onEventRegistrations->(
|
|
314
|
+
Utils.magic: array<Internal.onEventRegistration> => array<
|
|
315
|
+
Internal.svmOnEventRegistration,
|
|
316
|
+
>
|
|
317
|
+
),
|
|
299
318
|
clientTimeoutMillis: Env.hyperSyncClientTimeoutMillis,
|
|
300
319
|
addressStore,
|
|
301
320
|
}),
|
|
@@ -359,7 +378,7 @@ let makeInternal = (
|
|
|
359
378
|
~chainReorgCheckpoints,
|
|
360
379
|
),
|
|
361
380
|
~committedProgressBlockNumber=progressBlockNumber,
|
|
362
|
-
~perChainEntities=config.
|
|
381
|
+
~perChainEntities=config.userEntities->EntityTables.perChain,
|
|
363
382
|
~timestampCaughtUpToHeadOrEndblock,
|
|
364
383
|
~numEventsProcessed,
|
|
365
384
|
~transactionStore=TransactionStore.make(
|
|
@@ -373,33 +392,6 @@ let makeInternal = (
|
|
|
373
392
|
)
|
|
374
393
|
}
|
|
375
394
|
|
|
376
|
-
let makeFromConfig = (
|
|
377
|
-
chainConfig: Config.chain,
|
|
378
|
-
~config,
|
|
379
|
-
~registrationsByChainId,
|
|
380
|
-
~knownHeight,
|
|
381
|
-
) => {
|
|
382
|
-
let logger = Logging.createChild(~params={"chainId": chainConfig.id})
|
|
383
|
-
|
|
384
|
-
makeInternal(
|
|
385
|
-
~chainConfig,
|
|
386
|
-
~config,
|
|
387
|
-
~registrationsByChainId,
|
|
388
|
-
~startBlock=chainConfig.startBlock,
|
|
389
|
-
~endBlock=chainConfig.endBlock,
|
|
390
|
-
~reorgCheckpoints=[],
|
|
391
|
-
~maxReorgDepth=chainConfig.maxReorgDepth,
|
|
392
|
-
~progressBlockNumber=-1,
|
|
393
|
-
~timestampCaughtUpToHeadOrEndblock=None,
|
|
394
|
-
~numEventsProcessed=0.,
|
|
395
|
-
~logger,
|
|
396
|
-
~indexingAddresses=configAddresses(chainConfig),
|
|
397
|
-
~isInReorgThreshold=false,
|
|
398
|
-
~isRealtime=false,
|
|
399
|
-
~knownHeight,
|
|
400
|
-
)
|
|
401
|
-
}
|
|
402
|
-
|
|
403
395
|
/**
|
|
404
396
|
* This function allows a chain state to be created from metadata, in particular this is useful for restarting an indexer and making sure it fetches blocks from the same place.
|
|
405
397
|
*/
|
|
@@ -410,6 +402,7 @@ let makeFromDbState = (
|
|
|
410
402
|
~isInReorgThreshold,
|
|
411
403
|
~isRealtime,
|
|
412
404
|
~config,
|
|
405
|
+
~contractMapping,
|
|
413
406
|
~registrationsByChainId,
|
|
414
407
|
~reducedPollingInterval=?,
|
|
415
408
|
) => {
|
|
@@ -423,7 +416,8 @@ let makeFromDbState = (
|
|
|
423
416
|
: resumedChainState.startBlock - 1
|
|
424
417
|
|
|
425
418
|
makeInternal(
|
|
426
|
-
~
|
|
419
|
+
~addressRows=resumedChainState.addressRows,
|
|
420
|
+
~contractMapping,
|
|
427
421
|
~chainConfig,
|
|
428
422
|
~startBlock=resumedChainState.startBlock,
|
|
429
423
|
~endBlock=resumedChainState.endBlock,
|
|
@@ -507,7 +501,7 @@ let contractAddresses = (cs: t, ~contractName) =>
|
|
|
507
501
|
|
|
508
502
|
// Hands over the dynamically registered addresses the database hasn't seen yet,
|
|
509
503
|
// up to the block the caller is about to commit, each paired with the checkpoint
|
|
510
|
-
// that
|
|
504
|
+
// that registered it. Destructive: the caller must persist them or take the
|
|
511
505
|
// indexer down. Throws with nothing consumed when a registration's block has no
|
|
512
506
|
// checkpoint in the batch.
|
|
513
507
|
let drainAddressesForWrite = (cs: t, ~toBlockInclusive, ~checkpointBlockNumbers) =>
|
|
@@ -854,9 +848,9 @@ let applyTransactionGroups = async (store: TransactionStore.t, g: transactionGro
|
|
|
854
848
|
g.payloadGroups->Array.forEachWithIndex((payloads, i) => {
|
|
855
849
|
let tx = txs->Array.getUnsafe(i)
|
|
856
850
|
payloads->Array.forEach(payload => payload->Internal.setPayloadTransaction(tx))
|
|
857
|
-
switch
|
|
858
|
-
|
|
859
|
-
|
|
851
|
+
switch tx
|
|
852
|
+
->(Utils.magic: Internal.eventTransaction => dict<unknown>)
|
|
853
|
+
->Dict.get("accountActivities") {
|
|
860
854
|
| Some(_) => payloads->Array.forEach(payload => Svm.attachAccountActivities(payload, tx))
|
|
861
855
|
| None => ()
|
|
862
856
|
}
|
|
@@ -1027,6 +1021,9 @@ let toMetrics = (cs: t): Metrics.chainMetrics => {
|
|
|
1027
1021
|
startBlock: cs.fetchState.startBlock,
|
|
1028
1022
|
endBlock: cs.fetchState.endBlock,
|
|
1029
1023
|
numAddresses: cs.addressStore->AddressStore.size,
|
|
1024
|
+
addressesByContract: cs.addressStore
|
|
1025
|
+
->AddressStore.contractCounts
|
|
1026
|
+
->Array.map(({contractName, count}) => (contractName, count)),
|
|
1030
1027
|
isReady: cs->isReady,
|
|
1031
1028
|
sourceBlockNumber: cs.fetchState.knownHeight,
|
|
1032
1029
|
progressBlockNumber: cs.committedProgressBlockNumber,
|
|
@@ -1212,27 +1209,40 @@ let markReady = (cs: t, ~readyAt) =>
|
|
|
1212
1209
|
cs.timestampCaughtUpToHeadOrEndblock = Some(readyAt)
|
|
1213
1210
|
}
|
|
1214
1211
|
|
|
1215
|
-
|
|
1216
|
-
|
|
1217
|
-
|
|
1218
|
-
|
|
1219
|
-
|
|
1220
|
-
|
|
1221
|
-
|
|
1222
|
-
|
|
1223
|
-
|
|
1224
|
-
|
|
1225
|
-
|
|
1226
|
-
|
|
1227
|
-
|
|
1228
|
-
|
|
1229
|
-
|
|
1230
|
-
|
|
1231
|
-
// Both dicts are populated together per progress-diff row, so a chain with
|
|
1232
|
-
// a progress diff always has an events-processed diff too.
|
|
1233
|
-
eventsProcessedDiff->Option.getOrThrow(
|
|
1234
|
-
~message="Missing events-processed diff for rolled-back chain",
|
|
1212
|
+
type progressDiff = {blockNumber: int, eventsProcessed: float}
|
|
1213
|
+
|
|
1214
|
+
type rolledBackTo =
|
|
1215
|
+
| RecomputedProgress(progressDiff)
|
|
1216
|
+
| ForkBlock(int)
|
|
1217
|
+
| Untouched
|
|
1218
|
+
|
|
1219
|
+
// Returns the address registrations the storage has to delete along with the
|
|
1220
|
+
// chain — the store is what decides which ones died, so the two halves of a
|
|
1221
|
+
// rollback can't disagree.
|
|
1222
|
+
let rollback = (cs: t, ~rolledBackTo: rolledBackTo): array<AddressRows.key> => {
|
|
1223
|
+
let rollbackTo = targetBlockNumber => {
|
|
1224
|
+
let {fetchState, rolledBackAddresses} =
|
|
1225
|
+
cs.fetchState->FetchState.rollback(
|
|
1226
|
+
~rolledBackAddressStore=cs.addressStore,
|
|
1227
|
+
~targetBlockNumber,
|
|
1235
1228
|
)
|
|
1229
|
+
cs.fetchState = fetchState
|
|
1230
|
+
rolledBackAddresses->Array.map(({address, contractId}): AddressRows.key => {
|
|
1231
|
+
chainId: cs.chainConfig.id,
|
|
1232
|
+
address,
|
|
1233
|
+
contractId,
|
|
1234
|
+
})
|
|
1235
|
+
}
|
|
1236
|
+
|
|
1237
|
+
switch rolledBackTo {
|
|
1238
|
+
| RecomputedProgress({blockNumber, eventsProcessed}) =>
|
|
1239
|
+
// A rollback only ever takes a chain back. The diff recomputes progress from
|
|
1240
|
+
// the checkpoints still in the database, so a chain that an unwritten
|
|
1241
|
+
// rollback already took below them — to a fork block only that rollback
|
|
1242
|
+
// knew — would be handed their MIN back and moved forward onto blocks it
|
|
1243
|
+
// never re-indexed.
|
|
1244
|
+
let newProgressBlockNumber = Pervasives.min(blockNumber, cs.committedProgressBlockNumber)
|
|
1245
|
+
let newTotalEventsProcessed = cs.numEventsProcessed -. eventsProcessed
|
|
1236
1246
|
|
|
1237
1247
|
switch cs.safeCheckpointTracking {
|
|
1238
1248
|
| Some(safeCheckpointTracking) =>
|
|
@@ -1243,30 +1253,22 @@ let rollback = (
|
|
|
1243
1253
|
)
|
|
1244
1254
|
| None => ()
|
|
1245
1255
|
}
|
|
1246
|
-
|
|
1247
|
-
cs.fetchState->FetchState.rollback(
|
|
1248
|
-
~addressStore=cs.addressStore,
|
|
1249
|
-
~targetBlockNumber=newProgressBlockNumber,
|
|
1250
|
-
)
|
|
1256
|
+
let rolledBackAddresses = rollbackTo(newProgressBlockNumber)
|
|
1251
1257
|
cs.transactionStore->TransactionStore.rollback(newProgressBlockNumber)
|
|
1252
1258
|
cs.blockStore->BlockStore.rollback(newProgressBlockNumber)
|
|
1253
1259
|
cs.committedProgressBlockNumber = newProgressBlockNumber
|
|
1254
1260
|
cs.processingBlockNumber = newProgressBlockNumber
|
|
1255
1261
|
cs.numEventsProcessed = newTotalEventsProcessed
|
|
1256
|
-
|
|
1257
|
-
|
|
1258
|
-
|
|
1259
|
-
|
|
1260
|
-
|
|
1261
|
-
|
|
1262
|
-
|
|
1263
|
-
|
|
1264
|
-
|
|
1265
|
-
|
|
1266
|
-
|
|
1267
|
-
rollbackTargetBlockNumber,
|
|
1268
|
-
)
|
|
1269
|
-
cs.processingBlockNumber = Pervasives.min(cs.processingBlockNumber, rollbackTargetBlockNumber)
|
|
1270
|
-
}
|
|
1262
|
+
rolledBackAddresses
|
|
1263
|
+
| ForkBlock(forkBlock) =>
|
|
1264
|
+
// Rewinds the stores too, not just progress: otherwise the stale block hash
|
|
1265
|
+
// survives and re-triggers the same reorg.
|
|
1266
|
+
let rolledBackAddresses = rollbackTo(forkBlock)
|
|
1267
|
+
cs.transactionStore->TransactionStore.rollback(forkBlock)
|
|
1268
|
+
cs.blockStore->BlockStore.rollback(forkBlock)
|
|
1269
|
+
cs.committedProgressBlockNumber = Pervasives.min(cs.committedProgressBlockNumber, forkBlock)
|
|
1270
|
+
cs.processingBlockNumber = Pervasives.min(cs.processingBlockNumber, forkBlock)
|
|
1271
|
+
rolledBackAddresses
|
|
1272
|
+
| Untouched => []
|
|
1271
1273
|
}
|
|
1272
1274
|
}
|
package/src/ChainState.res.mjs
CHANGED
|
@@ -2,15 +2,16 @@
|
|
|
2
2
|
|
|
3
3
|
import * as Env from "./Env.res.mjs";
|
|
4
4
|
import * as Svm from "./sources/Svm.res.mjs";
|
|
5
|
+
import * as Core from "./Core.res.mjs";
|
|
5
6
|
import * as Batch from "./Batch.res.mjs";
|
|
6
7
|
import * as Utils from "./Utils.res.mjs";
|
|
7
8
|
import * as ChainId from "./ChainId.res.mjs";
|
|
8
9
|
import * as Logging from "./Logging.res.mjs";
|
|
9
|
-
import * as ChainMap from "./ChainMap.res.mjs";
|
|
10
10
|
import * as EvmChain from "./sources/EvmChain.res.mjs";
|
|
11
11
|
import * as FieldMask from "./sources/FieldMask.res.mjs";
|
|
12
12
|
import * as BlockStore from "./sources/BlockStore.res.mjs";
|
|
13
13
|
import * as FetchState from "./FetchState.res.mjs";
|
|
14
|
+
import * as AddressRows from "./AddressRows.res.mjs";
|
|
14
15
|
import * as Stdlib_Null from "@rescript/runtime/lib/es6/Stdlib_Null.js";
|
|
15
16
|
import * as AddressStore from "./sources/AddressStore.res.mjs";
|
|
16
17
|
import * as EntityTables from "./EntityTables.res.mjs";
|
|
@@ -20,6 +21,7 @@ import * as SourceManager from "./sources/SourceManager.res.mjs";
|
|
|
20
21
|
import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js";
|
|
21
22
|
import * as SimulateSource from "./sources/SimulateSource.res.mjs";
|
|
22
23
|
import * as Stdlib_JsError from "@rescript/runtime/lib/es6/Stdlib_JsError.js";
|
|
24
|
+
import * as ContractMapping from "./ContractMapping.res.mjs";
|
|
23
25
|
import * as Primitive_float from "@rescript/runtime/lib/es6/Primitive_float.js";
|
|
24
26
|
import * as Primitive_option from "@rescript/runtime/lib/es6/Primitive_option.js";
|
|
25
27
|
import * as TransactionStore from "./sources/TransactionStore.res.mjs";
|
|
@@ -27,18 +29,26 @@ import * as SvmHyperSyncSource from "./sources/SvmHyperSyncSource.res.mjs";
|
|
|
27
29
|
import * as FuelHyperSyncSource from "./sources/FuelHyperSyncSource.res.mjs";
|
|
28
30
|
import * as SafeCheckpointTracking from "./SafeCheckpointTracking.res.mjs";
|
|
29
31
|
|
|
30
|
-
function
|
|
32
|
+
function configStorageRows(chainConfig, ecosystem, contractMapping) {
|
|
31
33
|
let addresses = [];
|
|
34
|
+
let contractIds = [];
|
|
32
35
|
chainConfig.contracts.forEach(contract => {
|
|
36
|
+
let contractId = ContractMapping.idOfOrThrow(contractMapping, contract.name, undefined);
|
|
33
37
|
contract.addresses.forEach(address => {
|
|
34
|
-
addresses.push(
|
|
35
|
-
|
|
36
|
-
contractName: contract.name,
|
|
37
|
-
registrationBlock: -1
|
|
38
|
-
});
|
|
38
|
+
addresses.push(address);
|
|
39
|
+
contractIds.push(contractId);
|
|
39
40
|
});
|
|
40
41
|
});
|
|
41
|
-
return addresses
|
|
42
|
+
return Core.getAddon().encodeAddresses(ecosystem, addresses).map((address, idx) => ({
|
|
43
|
+
chainId: chainConfig.id,
|
|
44
|
+
address: address,
|
|
45
|
+
contractId: contractIds[idx],
|
|
46
|
+
registrationBlock: AddressRows.configRegistrationBlock
|
|
47
|
+
}));
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
function seedConfigAddresses(table, chainConfigs, ecosystem, contractMapping) {
|
|
51
|
+
chainConfigs.forEach(chainConfig => AddressRows.Table.insertMany(table, configStorageRows(chainConfig, ecosystem, contractMapping)));
|
|
42
52
|
}
|
|
43
53
|
|
|
44
54
|
function validateOnEventRegistrations(chainId, registrations) {
|
|
@@ -94,17 +104,21 @@ function make(chainConfig, fetchState, onEventRegistrationsOpt, addressStore, so
|
|
|
94
104
|
};
|
|
95
105
|
}
|
|
96
106
|
|
|
97
|
-
function
|
|
98
|
-
let
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
names.add(contract.name);
|
|
102
|
-
});
|
|
107
|
+
function makeFromDbState(chainConfig, resumedChainState, reorgCheckpoints, isInReorgThreshold, isRealtime, config, contractMapping, registrationsByChainId, reducedPollingInterval) {
|
|
108
|
+
let chainId = chainConfig.id;
|
|
109
|
+
let logger = Logging.createChild({
|
|
110
|
+
chainId: chainId
|
|
103
111
|
});
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
112
|
+
let progressBlockNumber = resumedChainState.progressBlockNumber >= 0 ? resumedChainState.progressBlockNumber : resumedChainState.startBlock - 1 | 0;
|
|
113
|
+
let addressRows = resumedChainState.addressRows;
|
|
114
|
+
let startBlock = resumedChainState.startBlock;
|
|
115
|
+
let endBlock = resumedChainState.endBlock;
|
|
116
|
+
let firstEventBlockOpt = Primitive_option.some(resumedChainState.firstEventBlockNumber);
|
|
117
|
+
let timestampCaughtUpToHeadOrEndblock = resumedChainState.timestampCaughtUpToHeadOrEndblock;
|
|
118
|
+
let numEventsProcessed = resumedChainState.numEventsProcessed;
|
|
119
|
+
let maxReorgDepth = resumedChainState.maxReorgDepth;
|
|
120
|
+
let knownHeightOpt = resumedChainState.sourceBlockNumber;
|
|
121
|
+
let isResumedOpt = true;
|
|
108
122
|
let firstEventBlock = firstEventBlockOpt !== undefined ? Primitive_option.valFromOption(firstEventBlockOpt) : undefined;
|
|
109
123
|
let knownHeight = knownHeightOpt !== undefined ? knownHeightOpt : 0;
|
|
110
124
|
let isResumed = isResumedOpt !== undefined ? isResumedOpt : false;
|
|
@@ -120,7 +134,7 @@ function makeInternal(chainConfig, indexingAddresses, startBlock, endBlock, firs
|
|
|
120
134
|
}
|
|
121
135
|
});
|
|
122
136
|
let lowercaseAddresses = config.lowercaseAddresses;
|
|
123
|
-
let addressStore = AddressStore.make(config.ecosystem.name, !lowercaseAddresses, AddressStore.contractsOf(onEventRegistrations,
|
|
137
|
+
let addressStore = AddressStore.make(config.ecosystem.name, !lowercaseAddresses, AddressStore.contractsOf(onEventRegistrations, contractMapping));
|
|
124
138
|
let match$1 = config.ecosystem.name;
|
|
125
139
|
let tmp;
|
|
126
140
|
switch (match$1) {
|
|
@@ -132,13 +146,13 @@ function makeInternal(chainConfig, indexingAddresses, startBlock, endBlock, firs
|
|
|
132
146
|
tmp = undefined;
|
|
133
147
|
break;
|
|
134
148
|
}
|
|
135
|
-
let fetchState = FetchState.make(startBlock, endBlock, onEventRegistrations, addressStore,
|
|
149
|
+
let fetchState = FetchState.make(startBlock, endBlock, onEventRegistrations, addressStore, addressRows, config.maxAddrInPartition, chainConfig.id, (config.batchSize << 1), knownHeight, progressBlockNumber, match.onBlockRegistrations, Primitive_int.max(!config.shouldRollbackOnReorg || isInReorgThreshold ? 0 : maxReorgDepth, chainConfig.blockLag), Primitive_option.some(firstEventBlock), Primitive_option.some(tmp), isResumed);
|
|
136
150
|
let chainReorgCheckpoints = Stdlib_Array.filterMap(reorgCheckpoints, reorgCheckpoint => {
|
|
137
151
|
if (reorgCheckpoint.chain_id === chainConfig.id) {
|
|
138
152
|
return reorgCheckpoint;
|
|
139
153
|
}
|
|
140
154
|
});
|
|
141
|
-
let chainId = chainConfig.id;
|
|
155
|
+
let chainId$1 = chainConfig.id;
|
|
142
156
|
let sources = chainConfig.sourceConfig;
|
|
143
157
|
let sources$1;
|
|
144
158
|
switch (sources.TAG) {
|
|
@@ -155,11 +169,11 @@ function makeInternal(chainConfig, indexingAddresses, startBlock, endBlock, firs
|
|
|
155
169
|
headers: headers
|
|
156
170
|
};
|
|
157
171
|
});
|
|
158
|
-
sources$1 = EvmChain.makeSources(chainId, onEventRegistrations, sources.hypersync, evmRpcs, lowercaseAddresses, addressStore);
|
|
172
|
+
sources$1 = EvmChain.makeSources(chainId$1, onEventRegistrations, sources.hypersync, evmRpcs, lowercaseAddresses, addressStore);
|
|
159
173
|
break;
|
|
160
174
|
case "FuelSourceConfig" :
|
|
161
175
|
sources$1 = [FuelHyperSyncSource.make({
|
|
162
|
-
chainId: chainId,
|
|
176
|
+
chainId: chainId$1,
|
|
163
177
|
endpointUrl: sources.hypersync,
|
|
164
178
|
apiToken: Env.envioApiToken,
|
|
165
179
|
onEventRegistrations: onEventRegistrations,
|
|
@@ -170,18 +184,18 @@ function makeInternal(chainConfig, indexingAddresses, startBlock, endBlock, firs
|
|
|
170
184
|
let rpc = sources.rpc;
|
|
171
185
|
let hypersync = sources.hypersync;
|
|
172
186
|
sources$1 = hypersync !== undefined ? [SvmHyperSyncSource.make({
|
|
173
|
-
chainId: chainId,
|
|
187
|
+
chainId: chainId$1,
|
|
174
188
|
endpointUrl: hypersync,
|
|
175
189
|
apiToken: Env.envioApiToken,
|
|
176
190
|
onEventRegistrations: onEventRegistrations,
|
|
177
191
|
clientTimeoutMillis: Env.hyperSyncClientTimeoutMillis,
|
|
178
192
|
addressStore: addressStore
|
|
179
193
|
})] : (
|
|
180
|
-
rpc !== undefined ? [Svm.makeRPCSource(chainId, rpc, undefined)] : Stdlib_JsError.throwWithMessage(`Chain ` + ChainId.toString(chainId) + ` has no SVM data source`)
|
|
194
|
+
rpc !== undefined ? [Svm.makeRPCSource(chainId$1, rpc, undefined)] : Stdlib_JsError.throwWithMessage(`Chain ` + ChainId.toString(chainId$1) + ` has no SVM data source`)
|
|
181
195
|
);
|
|
182
196
|
break;
|
|
183
197
|
case "SimulateSourceConfig" :
|
|
184
|
-
sources$1 = [SimulateSource.make(sources.items, sources.endBlock, chainId, addressStore, config.ecosystem.name, Primitive_option.some(sources.transactionStore), Primitive_option.some(sources.blockStore))];
|
|
198
|
+
sources$1 = [SimulateSource.make(sources.items, sources.endBlock, chainId$1, addressStore, config.ecosystem.name, Primitive_option.some(sources.transactionStore), Primitive_option.some(sources.blockStore))];
|
|
185
199
|
break;
|
|
186
200
|
case "CustomSources" :
|
|
187
201
|
sources$1 = sources._0;
|
|
@@ -197,23 +211,7 @@ function makeInternal(chainConfig, indexingAddresses, startBlock, endBlock, firs
|
|
|
197
211
|
}
|
|
198
212
|
let firstEventBlock$1 = fetchState.firstEventBlock;
|
|
199
213
|
let chainDensity = firstEventBlock$1 !== undefined && progressBlockNumber > firstEventBlock$1 && numEventsProcessed > 0 ? numEventsProcessed / (progressBlockNumber - firstEventBlock$1 | 0) : undefined;
|
|
200
|
-
return make(chainConfig, fetchState, onEventRegistrations, addressStore, SourceManager.make(sources$1, isRealtime, undefined, undefined, undefined, reducedPollingInterval, undefined, undefined), progressBlockNumber, Primitive_option.some(SafeCheckpointTracking.make(maxReorgDepth, config.shouldRollbackOnReorg, chainReorgCheckpoints)), config.shouldRollbackOnReorg, maxReorgDepth, numEventsProcessed, Primitive_option.some(timestampCaughtUpToHeadOrEndblock), undefined, Primitive_option.some(TransactionStore.make(config.ecosystem.name, !lowercaseAddresses)), Primitive_option.some(chainDensity), Primitive_option.some(blockStore), config.reorgThresholdReadyTolerance, EntityTables.perChain(config.
|
|
201
|
-
}
|
|
202
|
-
|
|
203
|
-
function makeFromConfig(chainConfig, config, registrationsByChainId, knownHeight) {
|
|
204
|
-
let logger = Logging.createChild({
|
|
205
|
-
chainId: chainConfig.id
|
|
206
|
-
});
|
|
207
|
-
return makeInternal(chainConfig, configAddresses(chainConfig), chainConfig.startBlock, chainConfig.endBlock, undefined, -1, config, registrationsByChainId, logger, undefined, 0, false, false, [], chainConfig.maxReorgDepth, knownHeight, undefined, undefined);
|
|
208
|
-
}
|
|
209
|
-
|
|
210
|
-
function makeFromDbState(chainConfig, resumedChainState, reorgCheckpoints, isInReorgThreshold, isRealtime, config, registrationsByChainId, reducedPollingInterval) {
|
|
211
|
-
let chainId = chainConfig.id;
|
|
212
|
-
let logger = Logging.createChild({
|
|
213
|
-
chainId: chainId
|
|
214
|
-
});
|
|
215
|
-
let progressBlockNumber = resumedChainState.progressBlockNumber >= 0 ? resumedChainState.progressBlockNumber : resumedChainState.startBlock - 1 | 0;
|
|
216
|
-
return makeInternal(chainConfig, resumedChainState.indexingAddresses, resumedChainState.startBlock, resumedChainState.endBlock, Primitive_option.some(resumedChainState.firstEventBlockNumber), progressBlockNumber, config, registrationsByChainId, logger, resumedChainState.timestampCaughtUpToHeadOrEndblock, resumedChainState.numEventsProcessed, isInReorgThreshold, isRealtime, reorgCheckpoints, resumedChainState.maxReorgDepth, resumedChainState.sourceBlockNumber, true, reducedPollingInterval);
|
|
214
|
+
return make(chainConfig, fetchState, onEventRegistrations, addressStore, SourceManager.make(sources$1, isRealtime, undefined, undefined, undefined, reducedPollingInterval, undefined, undefined), progressBlockNumber, Primitive_option.some(SafeCheckpointTracking.make(maxReorgDepth, config.shouldRollbackOnReorg, chainReorgCheckpoints)), config.shouldRollbackOnReorg, maxReorgDepth, numEventsProcessed, Primitive_option.some(timestampCaughtUpToHeadOrEndblock), undefined, Primitive_option.some(TransactionStore.make(config.ecosystem.name, !lowercaseAddresses)), Primitive_option.some(chainDensity), Primitive_option.some(blockStore), config.reorgThresholdReadyTolerance, EntityTables.perChain(config.userEntities), logger);
|
|
217
215
|
}
|
|
218
216
|
|
|
219
217
|
function logger(cs) {
|
|
@@ -721,6 +719,10 @@ function toMetrics(cs) {
|
|
|
721
719
|
startBlock: cs.fetchState.startBlock,
|
|
722
720
|
endBlock: cs.fetchState.endBlock,
|
|
723
721
|
numAddresses: cs.addressStore.size(),
|
|
722
|
+
addressesByContract: cs.addressStore.contractCounts().map(param => [
|
|
723
|
+
param.contractName,
|
|
724
|
+
param.count
|
|
725
|
+
]),
|
|
724
726
|
isReady: cs.timestampCaughtUpToHeadOrEndblock !== undefined,
|
|
725
727
|
sourceBlockNumber: cs.fetchState.knownHeight,
|
|
726
728
|
progressBlockNumber: cs.committedProgressBlockNumber,
|
|
@@ -855,36 +857,48 @@ function markReady(cs, readyAt) {
|
|
|
855
857
|
}
|
|
856
858
|
}
|
|
857
859
|
|
|
858
|
-
function rollback(cs,
|
|
859
|
-
|
|
860
|
-
|
|
861
|
-
|
|
862
|
-
|
|
863
|
-
cs.
|
|
864
|
-
|
|
865
|
-
|
|
866
|
-
|
|
867
|
-
|
|
868
|
-
|
|
869
|
-
|
|
860
|
+
function rollback(cs, rolledBackTo) {
|
|
861
|
+
let rollbackTo = targetBlockNumber => {
|
|
862
|
+
let match = FetchState.rollback(cs.fetchState, cs.addressStore, targetBlockNumber);
|
|
863
|
+
cs.fetchState = match.fetchState;
|
|
864
|
+
return match.rolledBackAddresses.map(param => ({
|
|
865
|
+
chainId: cs.chainConfig.id,
|
|
866
|
+
address: param.address,
|
|
867
|
+
contractId: param.contractId
|
|
868
|
+
}));
|
|
869
|
+
};
|
|
870
|
+
if (typeof rolledBackTo !== "object") {
|
|
871
|
+
return [];
|
|
870
872
|
}
|
|
871
|
-
|
|
872
|
-
|
|
873
|
-
|
|
874
|
-
cs.
|
|
873
|
+
if (rolledBackTo.TAG === "RecomputedProgress") {
|
|
874
|
+
let match = rolledBackTo._0;
|
|
875
|
+
let newProgressBlockNumber = Primitive_int.min(match.blockNumber, cs.committedProgressBlockNumber);
|
|
876
|
+
let newTotalEventsProcessed = cs.numEventsProcessed - match.eventsProcessed;
|
|
877
|
+
let safeCheckpointTracking = cs.safeCheckpointTracking;
|
|
878
|
+
if (safeCheckpointTracking !== undefined) {
|
|
879
|
+
cs.safeCheckpointTracking = SafeCheckpointTracking.rollback(safeCheckpointTracking, newProgressBlockNumber);
|
|
880
|
+
}
|
|
881
|
+
let rolledBackAddresses = rollbackTo(newProgressBlockNumber);
|
|
882
|
+
cs.transactionStore.rollback(newProgressBlockNumber);
|
|
883
|
+
cs.blockStore.rollback(newProgressBlockNumber);
|
|
884
|
+
cs.committedProgressBlockNumber = newProgressBlockNumber;
|
|
885
|
+
cs.processingBlockNumber = newProgressBlockNumber;
|
|
886
|
+
cs.numEventsProcessed = newTotalEventsProcessed;
|
|
887
|
+
return rolledBackAddresses;
|
|
875
888
|
}
|
|
876
|
-
|
|
877
|
-
|
|
878
|
-
cs.
|
|
879
|
-
cs.
|
|
880
|
-
cs.
|
|
881
|
-
cs.
|
|
889
|
+
let forkBlock = rolledBackTo._0;
|
|
890
|
+
let rolledBackAddresses$1 = rollbackTo(forkBlock);
|
|
891
|
+
cs.transactionStore.rollback(forkBlock);
|
|
892
|
+
cs.blockStore.rollback(forkBlock);
|
|
893
|
+
cs.committedProgressBlockNumber = Primitive_int.min(cs.committedProgressBlockNumber, forkBlock);
|
|
894
|
+
cs.processingBlockNumber = Primitive_int.min(cs.processingBlockNumber, forkBlock);
|
|
895
|
+
return rolledBackAddresses$1;
|
|
882
896
|
}
|
|
883
897
|
|
|
884
898
|
export {
|
|
885
|
-
|
|
899
|
+
configStorageRows,
|
|
900
|
+
seedConfigAddresses,
|
|
886
901
|
make,
|
|
887
|
-
makeFromConfig,
|
|
888
902
|
makeFromDbState,
|
|
889
903
|
logger,
|
|
890
904
|
blockStore,
|
package/src/ChainState.resi
CHANGED
|
@@ -3,7 +3,18 @@
|
|
|
3
3
|
|
|
4
4
|
type t
|
|
5
5
|
|
|
6
|
-
let
|
|
6
|
+
let configStorageRows: (
|
|
7
|
+
Config.chain,
|
|
8
|
+
~ecosystem: Ecosystem.name,
|
|
9
|
+
~contractMapping: ContractMapping.t,
|
|
10
|
+
) => array<AddressRows.row>
|
|
11
|
+
|
|
12
|
+
let seedConfigAddresses: (
|
|
13
|
+
AddressRows.Table.t,
|
|
14
|
+
~chainConfigs: array<Config.chain>,
|
|
15
|
+
~ecosystem: Ecosystem.name,
|
|
16
|
+
~contractMapping: ContractMapping.t,
|
|
17
|
+
) => unit
|
|
7
18
|
|
|
8
19
|
let make: (
|
|
9
20
|
~chainConfig: Config.chain,
|
|
@@ -26,13 +37,6 @@ let make: (
|
|
|
26
37
|
~logger: Pino.t,
|
|
27
38
|
) => t
|
|
28
39
|
|
|
29
|
-
let makeFromConfig: (
|
|
30
|
-
Config.chain,
|
|
31
|
-
~config: Config.t,
|
|
32
|
-
~registrationsByChainId: HandlerRegister.registrationsByChainId,
|
|
33
|
-
~knownHeight: int,
|
|
34
|
-
) => t
|
|
35
|
-
|
|
36
40
|
let makeFromDbState: (
|
|
37
41
|
Config.chain,
|
|
38
42
|
~resumedChainState: Persistence.initialChainState,
|
|
@@ -40,6 +44,7 @@ let makeFromDbState: (
|
|
|
40
44
|
~isInReorgThreshold: bool,
|
|
41
45
|
~isRealtime: bool,
|
|
42
46
|
~config: Config.t,
|
|
47
|
+
~contractMapping: ContractMapping.t,
|
|
43
48
|
~registrationsByChainId: HandlerRegister.registrationsByChainId,
|
|
44
49
|
~reducedPollingInterval: int=?,
|
|
45
50
|
) => t
|
|
@@ -152,10 +157,18 @@ let enterReorgThreshold: t => unit
|
|
|
152
157
|
let advanceAfterBatch: (t, ~batch: Batch.t, ~enteringReorgThreshold: bool) => unit
|
|
153
158
|
let applyBatchProgress: (t, ~batch: Batch.t, ~blockTimestampName: string) => unit
|
|
154
159
|
let markReady: (t, ~readyAt: Date.t) => unit
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
160
|
+
// What a rollback's progress diff says about one chain: the block its surviving
|
|
161
|
+
// checkpoints put it back to, and the events the deleted ones carried away. One
|
|
162
|
+
// record because one diff row yields both — they are never known separately.
|
|
163
|
+
type progressDiff = {blockNumber: int, eventsProcessed: float}
|
|
164
|
+
|
|
165
|
+
// Where a rollback leaves one chain.
|
|
166
|
+
type rolledBackTo =
|
|
167
|
+
| RecomputedProgress(progressDiff)
|
|
168
|
+
// The reorg chain with no checkpoint left at or below the target, so only the
|
|
169
|
+
// fork block says where it goes.
|
|
170
|
+
| ForkBlock(int)
|
|
171
|
+
// A chain this rollback doesn't reach.
|
|
172
|
+
| Untouched
|
|
173
|
+
|
|
174
|
+
let rollback: (t, ~rolledBackTo: rolledBackTo) => array<AddressRows.key>
|