envio 3.8.0 → 3.9.0-resolvers.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (109) hide show
  1. package/README.md +2 -2
  2. package/evm.schema.json +7 -0
  3. package/fuel.schema.json +7 -0
  4. package/index.d.ts +433 -109
  5. package/index.js +10 -0
  6. package/package.json +6 -7
  7. package/src/AddressRows.res +121 -0
  8. package/src/AddressRows.res.mjs +143 -0
  9. package/src/Batch.res +11 -0
  10. package/src/Batch.res.mjs +2 -1
  11. package/src/Bin.res +47 -0
  12. package/src/Bin.res.mjs +34 -1
  13. package/src/ChainState.res +102 -100
  14. package/src/ChainState.res.mjs +81 -67
  15. package/src/ChainState.resi +28 -15
  16. package/src/Config.res +43 -122
  17. package/src/Config.res.mjs +33 -75
  18. package/src/ContractMapping.res +66 -0
  19. package/src/ContractMapping.res.mjs +73 -0
  20. package/src/Core.res +24 -0
  21. package/src/Core.res.mjs +4 -0
  22. package/src/Ecosystem.res +7 -2
  23. package/src/Env.res +128 -1
  24. package/src/Env.res.mjs +123 -1
  25. package/src/EventConfigBuilder.res +169 -28
  26. package/src/EventConfigBuilder.res.mjs +124 -17
  27. package/src/FetchState.res +59 -67
  28. package/src/FetchState.res.mjs +34 -44
  29. package/src/HandlerRegister.res +6 -4
  30. package/src/HandlerRegister.res.mjs +8 -5
  31. package/src/InMemoryStore.res +22 -32
  32. package/src/InMemoryStore.res.mjs +12 -24
  33. package/src/IndexerState.res +40 -3
  34. package/src/IndexerState.res.mjs +30 -4
  35. package/src/IndexerState.resi +4 -1
  36. package/src/Internal.res +16 -17
  37. package/src/Main.res +25 -15
  38. package/src/Main.res.mjs +19 -14
  39. package/src/Metrics.res +15 -1
  40. package/src/Metrics.res.mjs +5 -1
  41. package/src/Persistence.res +42 -19
  42. package/src/Persistence.res.mjs +8 -5
  43. package/src/PgStorage.res +232 -78
  44. package/src/PgStorage.res.mjs +114 -72
  45. package/src/PruneStaleHistory.res +1 -1
  46. package/src/PruneStaleHistory.res.mjs +1 -1
  47. package/src/Rollback.res +62 -41
  48. package/src/Rollback.res.mjs +61 -19
  49. package/src/RollbackCommit.res +7 -9
  50. package/src/RollbackCommit.res.mjs +5 -6
  51. package/src/SimulateItems.res +5 -12
  52. package/src/SimulateItems.res.mjs +6 -10
  53. package/src/Sink.res +35 -26
  54. package/src/Sink.res.mjs +20 -16
  55. package/src/TestIndexer.res +95 -113
  56. package/src/TestIndexer.res.mjs +63 -90
  57. package/src/Utils.res +38 -0
  58. package/src/Utils.res.mjs +39 -2
  59. package/src/Writing.res +4 -2
  60. package/src/Writing.res.mjs +3 -2
  61. package/src/bindings/ClickHouse.res +311 -683
  62. package/src/bindings/ClickHouse.res.mjs +294 -416
  63. package/src/bindings/ClickHouseSink.res +303 -0
  64. package/src/bindings/ClickHouseSink.res.mjs +216 -0
  65. package/src/bindings/NodeJs.res +9 -0
  66. package/src/bindings/NodeJs.res.mjs +8 -1
  67. package/src/bindings/Postgres.res +9 -0
  68. package/src/bindings/Postgres.res.mjs +3 -0
  69. package/src/db/EntityHistory.res +29 -24
  70. package/src/db/EntityHistory.res.mjs +11 -19
  71. package/src/db/InternalTable.res +247 -83
  72. package/src/db/InternalTable.res.mjs +216 -85
  73. package/src/db/RollbackScope.res +29 -0
  74. package/src/db/RollbackScope.res.mjs +31 -0
  75. package/src/db/Table.res +28 -1
  76. package/src/db/Table.res.mjs +26 -2
  77. package/src/resolvers/ResolverProcess.res +809 -0
  78. package/src/resolvers/ResolverProcess.res.mjs +667 -0
  79. package/src/resolvers/ResolverQuery.res +317 -0
  80. package/src/resolvers/ResolverQuery.res.mjs +220 -0
  81. package/src/resolvers/collisions.js +108 -0
  82. package/src/resolvers/db.js +363 -0
  83. package/src/resolvers/dispatch.js +165 -0
  84. package/src/resolvers/errors.js +30 -0
  85. package/src/resolvers/graphqlSelection.js +292 -0
  86. package/src/resolvers/hasuraAction.js +84 -0
  87. package/src/resolvers/hasuraApply.js +323 -0
  88. package/src/resolvers/hasuraMetadata.js +119 -0
  89. package/src/resolvers/index.js +95 -0
  90. package/src/resolvers/manifest.js +370 -0
  91. package/src/resolvers/server.js +270 -0
  92. package/src/sources/AddressSet.res +0 -22
  93. package/src/sources/AddressStore.res +48 -88
  94. package/src/sources/AddressStore.res.mjs +11 -22
  95. package/src/sources/Evm.res +2 -0
  96. package/src/sources/Evm.res.mjs +2 -0
  97. package/src/sources/Fuel.res +2 -0
  98. package/src/sources/Fuel.res.mjs +2 -0
  99. package/src/sources/Svm.res +2 -0
  100. package/src/sources/Svm.res.mjs +2 -0
  101. package/src/sources/SvmHyperSyncClient.res +13 -107
  102. package/src/sources/SvmHyperSyncClient.res.mjs +2 -5
  103. package/src/sources/SvmHyperSyncSource.res +5 -10
  104. package/src/sources/SvmHyperSyncSource.res.mjs +5 -13
  105. package/src/tui/Tui.res +1 -1
  106. package/src/tui/Tui.res.mjs +1 -1
  107. package/svm.schema.json +17 -78
  108. package/src/MemoryStorage.res +0 -756
  109. package/src/MemoryStorage.res.mjs +0 -623
@@ -62,20 +62,48 @@ type t = {
62
62
  mutable progressLatencyMs: option<int>,
63
63
  }
64
64
 
65
- let configAddresses = (chainConfig: Config.chain): array<Internal.indexingAddress> => {
65
+ // The chain's config-declared addresses as storage rows: what a clean run
66
+ // writes into `envio_addresses`. The keys are encoded by the Rust codec, so the
67
+ // bytes a clean run writes are exactly the bytes a resume seeds from.
68
+ let configStorageRows = (
69
+ chainConfig: Config.chain,
70
+ ~ecosystem: Ecosystem.name,
71
+ ~contractMapping: ContractMapping.t,
72
+ ): array<AddressRows.row> => {
66
73
  let addresses = []
74
+ let contractIds = []
67
75
  chainConfig.contracts->Array.forEach(contract => {
76
+ let contractId = contractMapping->ContractMapping.idOfOrThrow(contract.name)
68
77
  contract.addresses->Array.forEach(address => {
69
- addresses->Array.push({
70
- Internal.address,
71
- contractName: contract.name,
72
- registrationBlock: -1,
73
- })
78
+ addresses->Array.push(address)->ignore
79
+ contractIds->Array.push(contractId)->ignore
74
80
  })
75
81
  })
76
- addresses
82
+ Core.getAddon().encodeAddresses(
83
+ ~ecosystem=(ecosystem :> string),
84
+ ~addresses,
85
+ )->Array.mapWithIndex((address, idx) => {
86
+ AddressRows.chainId: chainConfig.id,
87
+ address,
88
+ contractId: contractIds->Array.getUnsafe(idx),
89
+ registrationBlock: AddressRows.configRegistrationBlock,
90
+ })
77
91
  }
78
92
 
93
+ // Seeds an in-memory address table with every chain's config-declared
94
+ // addresses, the twin of what `PgStorage.initialize` writes in one insert.
95
+ let seedConfigAddresses = (
96
+ table: AddressRows.Table.t,
97
+ ~chainConfigs: array<Config.chain>,
98
+ ~ecosystem: Ecosystem.name,
99
+ ~contractMapping: ContractMapping.t,
100
+ ) =>
101
+ chainConfigs->Array.forEach(chainConfig =>
102
+ table->AddressRows.Table.insertMany(
103
+ chainConfig->configStorageRows(~ecosystem, ~contractMapping),
104
+ )
105
+ )
106
+
79
107
  let validateOnEventRegistrations = (
80
108
  ~chainId: ChainId.t,
81
109
  registrations: array<Internal.onEventRegistration>,
@@ -142,22 +170,12 @@ let make = (
142
170
  }
143
171
  }
144
172
 
145
- // Every chain's contracts, not just one chain's: `context.chain.X.add` accepts
146
- // any config contract, whichever chain declared it, so every chain's address
147
- // store has to hold the whole set.
148
- let configContractNames = (config: Config.t) => {
149
- let names = Utils.Set.make()
150
- config.chainMap
151
- ->ChainMap.values
152
- ->Array.forEach(chain =>
153
- chain.contracts->Array.forEach(contract => names->Utils.Set.add(contract.name)->ignore)
154
- )
155
- names->Utils.Set.toArray
156
- }
157
-
158
173
  let makeInternal = (
159
174
  ~chainConfig: Config.chain,
160
- ~indexingAddresses: array<Internal.indexingAddress>,
175
+ ~addressRows: AddressRows.seedRows,
176
+ // Passed rather than read off `config`, because a resume must use the mapping
177
+ // storage holds: the ids stored rows carry, not the ids this config derives.
178
+ ~contractMapping: ContractMapping.t,
161
179
  ~startBlock,
162
180
  ~endBlock,
163
181
  ~firstEventBlock=None,
@@ -199,16 +217,13 @@ let makeInternal = (
199
217
  let addressStore = AddressStore.make(
200
218
  ~ecosystem=config.ecosystem.name,
201
219
  ~shouldChecksum=!lowercaseAddresses,
202
- ~contracts=AddressStore.contractsOf(
203
- ~onEventRegistrations,
204
- ~configContractNames=config->configContractNames,
205
- ),
220
+ ~contracts=AddressStore.contractsOf(~onEventRegistrations, ~contractMapping),
206
221
  )
207
222
 
208
223
  let fetchState = FetchState.make(
209
224
  ~maxAddrInPartition=config.maxAddrInPartition,
210
225
  ~addressStore,
211
- ~addresses=indexingAddresses,
226
+ ~addressRows,
212
227
  ~progressBlockNumber,
213
228
  ~startBlock,
214
229
  ~endBlock,
@@ -295,7 +310,11 @@ let makeInternal = (
295
310
  chainId,
296
311
  endpointUrl: hypersyncUrl,
297
312
  apiToken,
298
- onEventRegistrations,
313
+ onEventRegistrations: onEventRegistrations->(
314
+ Utils.magic: array<Internal.onEventRegistration> => array<
315
+ Internal.svmOnEventRegistration,
316
+ >
317
+ ),
299
318
  clientTimeoutMillis: Env.hyperSyncClientTimeoutMillis,
300
319
  addressStore,
301
320
  }),
@@ -359,7 +378,7 @@ let makeInternal = (
359
378
  ~chainReorgCheckpoints,
360
379
  ),
361
380
  ~committedProgressBlockNumber=progressBlockNumber,
362
- ~perChainEntities=config.allEntities->EntityTables.perChain,
381
+ ~perChainEntities=config.userEntities->EntityTables.perChain,
363
382
  ~timestampCaughtUpToHeadOrEndblock,
364
383
  ~numEventsProcessed,
365
384
  ~transactionStore=TransactionStore.make(
@@ -373,33 +392,6 @@ let makeInternal = (
373
392
  )
374
393
  }
375
394
 
376
- let makeFromConfig = (
377
- chainConfig: Config.chain,
378
- ~config,
379
- ~registrationsByChainId,
380
- ~knownHeight,
381
- ) => {
382
- let logger = Logging.createChild(~params={"chainId": chainConfig.id})
383
-
384
- makeInternal(
385
- ~chainConfig,
386
- ~config,
387
- ~registrationsByChainId,
388
- ~startBlock=chainConfig.startBlock,
389
- ~endBlock=chainConfig.endBlock,
390
- ~reorgCheckpoints=[],
391
- ~maxReorgDepth=chainConfig.maxReorgDepth,
392
- ~progressBlockNumber=-1,
393
- ~timestampCaughtUpToHeadOrEndblock=None,
394
- ~numEventsProcessed=0.,
395
- ~logger,
396
- ~indexingAddresses=configAddresses(chainConfig),
397
- ~isInReorgThreshold=false,
398
- ~isRealtime=false,
399
- ~knownHeight,
400
- )
401
- }
402
-
403
395
  /**
404
396
  * This function allows a chain state to be created from metadata, in particular this is useful for restarting an indexer and making sure it fetches blocks from the same place.
405
397
  */
@@ -410,6 +402,7 @@ let makeFromDbState = (
410
402
  ~isInReorgThreshold,
411
403
  ~isRealtime,
412
404
  ~config,
405
+ ~contractMapping,
413
406
  ~registrationsByChainId,
414
407
  ~reducedPollingInterval=?,
415
408
  ) => {
@@ -423,7 +416,8 @@ let makeFromDbState = (
423
416
  : resumedChainState.startBlock - 1
424
417
 
425
418
  makeInternal(
426
- ~indexingAddresses=resumedChainState.indexingAddresses,
419
+ ~addressRows=resumedChainState.addressRows,
420
+ ~contractMapping,
427
421
  ~chainConfig,
428
422
  ~startBlock=resumedChainState.startBlock,
429
423
  ~endBlock=resumedChainState.endBlock,
@@ -507,7 +501,7 @@ let contractAddresses = (cs: t, ~contractName) =>
507
501
 
508
502
  // Hands over the dynamically registered addresses the database hasn't seen yet,
509
503
  // up to the block the caller is about to commit, each paired with the checkpoint
510
- // that owns its row. Destructive: the caller must persist them or take the
504
+ // that registered it. Destructive: the caller must persist them or take the
511
505
  // indexer down. Throws with nothing consumed when a registration's block has no
512
506
  // checkpoint in the batch.
513
507
  let drainAddressesForWrite = (cs: t, ~toBlockInclusive, ~checkpointBlockNumbers) =>
@@ -854,9 +848,9 @@ let applyTransactionGroups = async (store: TransactionStore.t, g: transactionGro
854
848
  g.payloadGroups->Array.forEachWithIndex((payloads, i) => {
855
849
  let tx = txs->Array.getUnsafe(i)
856
850
  payloads->Array.forEach(payload => payload->Internal.setPayloadTransaction(tx))
857
- switch (
858
- tx->(Utils.magic: Internal.eventTransaction => dict<unknown>)
859
- )->Dict.get("accountActivities") {
851
+ switch tx
852
+ ->(Utils.magic: Internal.eventTransaction => dict<unknown>)
853
+ ->Dict.get("accountActivities") {
860
854
  | Some(_) => payloads->Array.forEach(payload => Svm.attachAccountActivities(payload, tx))
861
855
  | None => ()
862
856
  }
@@ -1027,6 +1021,9 @@ let toMetrics = (cs: t): Metrics.chainMetrics => {
1027
1021
  startBlock: cs.fetchState.startBlock,
1028
1022
  endBlock: cs.fetchState.endBlock,
1029
1023
  numAddresses: cs.addressStore->AddressStore.size,
1024
+ addressesByContract: cs.addressStore
1025
+ ->AddressStore.contractCounts
1026
+ ->Array.map(({contractName, count}) => (contractName, count)),
1030
1027
  isReady: cs->isReady,
1031
1028
  sourceBlockNumber: cs.fetchState.knownHeight,
1032
1029
  progressBlockNumber: cs.committedProgressBlockNumber,
@@ -1212,27 +1209,40 @@ let markReady = (cs: t, ~readyAt) =>
1212
1209
  cs.timestampCaughtUpToHeadOrEndblock = Some(readyAt)
1213
1210
  }
1214
1211
 
1215
- // Roll a chain back to a reorg target. With a progress diff, restore fetch/
1216
- // safe-checkpoint/progress state to `newProgressBlockNumber`. A reorg chain
1217
- // with no diff entry still rewinds fetch state + stores to the target -
1218
- // otherwise the stale block hash stays in the store and re-triggers the same
1219
- // reorg.
1220
- let rollback = (
1221
- cs: t,
1222
- ~newProgressBlockNumber,
1223
- ~eventsProcessedDiff,
1224
- ~rollbackTargetBlockNumber,
1225
- ~isReorgChain,
1226
- ) => {
1227
- switch newProgressBlockNumber {
1228
- | Some(newProgressBlockNumber) =>
1229
- let newTotalEventsProcessed =
1230
- cs.numEventsProcessed -.
1231
- // Both dicts are populated together per progress-diff row, so a chain with
1232
- // a progress diff always has an events-processed diff too.
1233
- eventsProcessedDiff->Option.getOrThrow(
1234
- ~message="Missing events-processed diff for rolled-back chain",
1212
+ type progressDiff = {blockNumber: int, eventsProcessed: float}
1213
+
1214
+ type rolledBackTo =
1215
+ | RecomputedProgress(progressDiff)
1216
+ | ForkBlock(int)
1217
+ | Untouched
1218
+
1219
+ // Returns the address registrations the storage has to delete along with the
1220
+ // chain — the store is what decides which ones died, so the two halves of a
1221
+ // rollback can't disagree.
1222
+ let rollback = (cs: t, ~rolledBackTo: rolledBackTo): array<AddressRows.key> => {
1223
+ let rollbackTo = targetBlockNumber => {
1224
+ let {fetchState, rolledBackAddresses} =
1225
+ cs.fetchState->FetchState.rollback(
1226
+ ~rolledBackAddressStore=cs.addressStore,
1227
+ ~targetBlockNumber,
1235
1228
  )
1229
+ cs.fetchState = fetchState
1230
+ rolledBackAddresses->Array.map(({address, contractId}): AddressRows.key => {
1231
+ chainId: cs.chainConfig.id,
1232
+ address,
1233
+ contractId,
1234
+ })
1235
+ }
1236
+
1237
+ switch rolledBackTo {
1238
+ | RecomputedProgress({blockNumber, eventsProcessed}) =>
1239
+ // A rollback only ever takes a chain back. The diff recomputes progress from
1240
+ // the checkpoints still in the database, so a chain that an unwritten
1241
+ // rollback already took below them — to a fork block only that rollback
1242
+ // knew — would be handed their MIN back and moved forward onto blocks it
1243
+ // never re-indexed.
1244
+ let newProgressBlockNumber = Pervasives.min(blockNumber, cs.committedProgressBlockNumber)
1245
+ let newTotalEventsProcessed = cs.numEventsProcessed -. eventsProcessed
1236
1246
 
1237
1247
  switch cs.safeCheckpointTracking {
1238
1248
  | Some(safeCheckpointTracking) =>
@@ -1243,30 +1253,22 @@ let rollback = (
1243
1253
  )
1244
1254
  | None => ()
1245
1255
  }
1246
- cs.fetchState =
1247
- cs.fetchState->FetchState.rollback(
1248
- ~addressStore=cs.addressStore,
1249
- ~targetBlockNumber=newProgressBlockNumber,
1250
- )
1256
+ let rolledBackAddresses = rollbackTo(newProgressBlockNumber)
1251
1257
  cs.transactionStore->TransactionStore.rollback(newProgressBlockNumber)
1252
1258
  cs.blockStore->BlockStore.rollback(newProgressBlockNumber)
1253
1259
  cs.committedProgressBlockNumber = newProgressBlockNumber
1254
1260
  cs.processingBlockNumber = newProgressBlockNumber
1255
1261
  cs.numEventsProcessed = newTotalEventsProcessed
1256
- | None =>
1257
- if isReorgChain {
1258
- cs.fetchState =
1259
- cs.fetchState->FetchState.rollback(
1260
- ~addressStore=cs.addressStore,
1261
- ~targetBlockNumber=rollbackTargetBlockNumber,
1262
- )
1263
- cs.transactionStore->TransactionStore.rollback(rollbackTargetBlockNumber)
1264
- cs.blockStore->BlockStore.rollback(rollbackTargetBlockNumber)
1265
- cs.committedProgressBlockNumber = Pervasives.min(
1266
- cs.committedProgressBlockNumber,
1267
- rollbackTargetBlockNumber,
1268
- )
1269
- cs.processingBlockNumber = Pervasives.min(cs.processingBlockNumber, rollbackTargetBlockNumber)
1270
- }
1262
+ rolledBackAddresses
1263
+ | ForkBlock(forkBlock) =>
1264
+ // Rewinds the stores too, not just progress: otherwise the stale block hash
1265
+ // survives and re-triggers the same reorg.
1266
+ let rolledBackAddresses = rollbackTo(forkBlock)
1267
+ cs.transactionStore->TransactionStore.rollback(forkBlock)
1268
+ cs.blockStore->BlockStore.rollback(forkBlock)
1269
+ cs.committedProgressBlockNumber = Pervasives.min(cs.committedProgressBlockNumber, forkBlock)
1270
+ cs.processingBlockNumber = Pervasives.min(cs.processingBlockNumber, forkBlock)
1271
+ rolledBackAddresses
1272
+ | Untouched => []
1271
1273
  }
1272
1274
  }
@@ -2,15 +2,16 @@
2
2
 
3
3
  import * as Env from "./Env.res.mjs";
4
4
  import * as Svm from "./sources/Svm.res.mjs";
5
+ import * as Core from "./Core.res.mjs";
5
6
  import * as Batch from "./Batch.res.mjs";
6
7
  import * as Utils from "./Utils.res.mjs";
7
8
  import * as ChainId from "./ChainId.res.mjs";
8
9
  import * as Logging from "./Logging.res.mjs";
9
- import * as ChainMap from "./ChainMap.res.mjs";
10
10
  import * as EvmChain from "./sources/EvmChain.res.mjs";
11
11
  import * as FieldMask from "./sources/FieldMask.res.mjs";
12
12
  import * as BlockStore from "./sources/BlockStore.res.mjs";
13
13
  import * as FetchState from "./FetchState.res.mjs";
14
+ import * as AddressRows from "./AddressRows.res.mjs";
14
15
  import * as Stdlib_Null from "@rescript/runtime/lib/es6/Stdlib_Null.js";
15
16
  import * as AddressStore from "./sources/AddressStore.res.mjs";
16
17
  import * as EntityTables from "./EntityTables.res.mjs";
@@ -20,6 +21,7 @@ import * as SourceManager from "./sources/SourceManager.res.mjs";
20
21
  import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js";
21
22
  import * as SimulateSource from "./sources/SimulateSource.res.mjs";
22
23
  import * as Stdlib_JsError from "@rescript/runtime/lib/es6/Stdlib_JsError.js";
24
+ import * as ContractMapping from "./ContractMapping.res.mjs";
23
25
  import * as Primitive_float from "@rescript/runtime/lib/es6/Primitive_float.js";
24
26
  import * as Primitive_option from "@rescript/runtime/lib/es6/Primitive_option.js";
25
27
  import * as TransactionStore from "./sources/TransactionStore.res.mjs";
@@ -27,18 +29,26 @@ import * as SvmHyperSyncSource from "./sources/SvmHyperSyncSource.res.mjs";
27
29
  import * as FuelHyperSyncSource from "./sources/FuelHyperSyncSource.res.mjs";
28
30
  import * as SafeCheckpointTracking from "./SafeCheckpointTracking.res.mjs";
29
31
 
30
- function configAddresses(chainConfig) {
32
+ function configStorageRows(chainConfig, ecosystem, contractMapping) {
31
33
  let addresses = [];
34
+ let contractIds = [];
32
35
  chainConfig.contracts.forEach(contract => {
36
+ let contractId = ContractMapping.idOfOrThrow(contractMapping, contract.name, undefined);
33
37
  contract.addresses.forEach(address => {
34
- addresses.push({
35
- address: address,
36
- contractName: contract.name,
37
- registrationBlock: -1
38
- });
38
+ addresses.push(address);
39
+ contractIds.push(contractId);
39
40
  });
40
41
  });
41
- return addresses;
42
+ return Core.getAddon().encodeAddresses(ecosystem, addresses).map((address, idx) => ({
43
+ chainId: chainConfig.id,
44
+ address: address,
45
+ contractId: contractIds[idx],
46
+ registrationBlock: AddressRows.configRegistrationBlock
47
+ }));
48
+ }
49
+
50
+ function seedConfigAddresses(table, chainConfigs, ecosystem, contractMapping) {
51
+ chainConfigs.forEach(chainConfig => AddressRows.Table.insertMany(table, configStorageRows(chainConfig, ecosystem, contractMapping)));
42
52
  }
43
53
 
44
54
  function validateOnEventRegistrations(chainId, registrations) {
@@ -94,17 +104,21 @@ function make(chainConfig, fetchState, onEventRegistrationsOpt, addressStore, so
94
104
  };
95
105
  }
96
106
 
97
- function configContractNames(config) {
98
- let names = new Set();
99
- ChainMap.values(config.chainMap).forEach(chain => {
100
- chain.contracts.forEach(contract => {
101
- names.add(contract.name);
102
- });
107
+ function makeFromDbState(chainConfig, resumedChainState, reorgCheckpoints, isInReorgThreshold, isRealtime, config, contractMapping, registrationsByChainId, reducedPollingInterval) {
108
+ let chainId = chainConfig.id;
109
+ let logger = Logging.createChild({
110
+ chainId: chainId
103
111
  });
104
- return Array.from(names);
105
- }
106
-
107
- function makeInternal(chainConfig, indexingAddresses, startBlock, endBlock, firstEventBlockOpt, progressBlockNumber, config, registrationsByChainId, logger, timestampCaughtUpToHeadOrEndblock, numEventsProcessed, isInReorgThreshold, isRealtime, reorgCheckpoints, maxReorgDepth, knownHeightOpt, isResumedOpt, reducedPollingInterval) {
112
+ let progressBlockNumber = resumedChainState.progressBlockNumber >= 0 ? resumedChainState.progressBlockNumber : resumedChainState.startBlock - 1 | 0;
113
+ let addressRows = resumedChainState.addressRows;
114
+ let startBlock = resumedChainState.startBlock;
115
+ let endBlock = resumedChainState.endBlock;
116
+ let firstEventBlockOpt = Primitive_option.some(resumedChainState.firstEventBlockNumber);
117
+ let timestampCaughtUpToHeadOrEndblock = resumedChainState.timestampCaughtUpToHeadOrEndblock;
118
+ let numEventsProcessed = resumedChainState.numEventsProcessed;
119
+ let maxReorgDepth = resumedChainState.maxReorgDepth;
120
+ let knownHeightOpt = resumedChainState.sourceBlockNumber;
121
+ let isResumedOpt = true;
108
122
  let firstEventBlock = firstEventBlockOpt !== undefined ? Primitive_option.valFromOption(firstEventBlockOpt) : undefined;
109
123
  let knownHeight = knownHeightOpt !== undefined ? knownHeightOpt : 0;
110
124
  let isResumed = isResumedOpt !== undefined ? isResumedOpt : false;
@@ -120,7 +134,7 @@ function makeInternal(chainConfig, indexingAddresses, startBlock, endBlock, firs
120
134
  }
121
135
  });
122
136
  let lowercaseAddresses = config.lowercaseAddresses;
123
- let addressStore = AddressStore.make(config.ecosystem.name, !lowercaseAddresses, AddressStore.contractsOf(onEventRegistrations, configContractNames(config)));
137
+ let addressStore = AddressStore.make(config.ecosystem.name, !lowercaseAddresses, AddressStore.contractsOf(onEventRegistrations, contractMapping));
124
138
  let match$1 = config.ecosystem.name;
125
139
  let tmp;
126
140
  switch (match$1) {
@@ -132,13 +146,13 @@ function makeInternal(chainConfig, indexingAddresses, startBlock, endBlock, firs
132
146
  tmp = undefined;
133
147
  break;
134
148
  }
135
- let fetchState = FetchState.make(startBlock, endBlock, onEventRegistrations, addressStore, indexingAddresses, config.maxAddrInPartition, chainConfig.id, (config.batchSize << 1), knownHeight, progressBlockNumber, match.onBlockRegistrations, Primitive_int.max(!config.shouldRollbackOnReorg || isInReorgThreshold ? 0 : maxReorgDepth, chainConfig.blockLag), Primitive_option.some(firstEventBlock), Primitive_option.some(tmp), isResumed);
149
+ let fetchState = FetchState.make(startBlock, endBlock, onEventRegistrations, addressStore, addressRows, config.maxAddrInPartition, chainConfig.id, (config.batchSize << 1), knownHeight, progressBlockNumber, match.onBlockRegistrations, Primitive_int.max(!config.shouldRollbackOnReorg || isInReorgThreshold ? 0 : maxReorgDepth, chainConfig.blockLag), Primitive_option.some(firstEventBlock), Primitive_option.some(tmp), isResumed);
136
150
  let chainReorgCheckpoints = Stdlib_Array.filterMap(reorgCheckpoints, reorgCheckpoint => {
137
151
  if (reorgCheckpoint.chain_id === chainConfig.id) {
138
152
  return reorgCheckpoint;
139
153
  }
140
154
  });
141
- let chainId = chainConfig.id;
155
+ let chainId$1 = chainConfig.id;
142
156
  let sources = chainConfig.sourceConfig;
143
157
  let sources$1;
144
158
  switch (sources.TAG) {
@@ -155,11 +169,11 @@ function makeInternal(chainConfig, indexingAddresses, startBlock, endBlock, firs
155
169
  headers: headers
156
170
  };
157
171
  });
158
- sources$1 = EvmChain.makeSources(chainId, onEventRegistrations, sources.hypersync, evmRpcs, lowercaseAddresses, addressStore);
172
+ sources$1 = EvmChain.makeSources(chainId$1, onEventRegistrations, sources.hypersync, evmRpcs, lowercaseAddresses, addressStore);
159
173
  break;
160
174
  case "FuelSourceConfig" :
161
175
  sources$1 = [FuelHyperSyncSource.make({
162
- chainId: chainId,
176
+ chainId: chainId$1,
163
177
  endpointUrl: sources.hypersync,
164
178
  apiToken: Env.envioApiToken,
165
179
  onEventRegistrations: onEventRegistrations,
@@ -170,18 +184,18 @@ function makeInternal(chainConfig, indexingAddresses, startBlock, endBlock, firs
170
184
  let rpc = sources.rpc;
171
185
  let hypersync = sources.hypersync;
172
186
  sources$1 = hypersync !== undefined ? [SvmHyperSyncSource.make({
173
- chainId: chainId,
187
+ chainId: chainId$1,
174
188
  endpointUrl: hypersync,
175
189
  apiToken: Env.envioApiToken,
176
190
  onEventRegistrations: onEventRegistrations,
177
191
  clientTimeoutMillis: Env.hyperSyncClientTimeoutMillis,
178
192
  addressStore: addressStore
179
193
  })] : (
180
- rpc !== undefined ? [Svm.makeRPCSource(chainId, rpc, undefined)] : Stdlib_JsError.throwWithMessage(`Chain ` + ChainId.toString(chainId) + ` has no SVM data source`)
194
+ rpc !== undefined ? [Svm.makeRPCSource(chainId$1, rpc, undefined)] : Stdlib_JsError.throwWithMessage(`Chain ` + ChainId.toString(chainId$1) + ` has no SVM data source`)
181
195
  );
182
196
  break;
183
197
  case "SimulateSourceConfig" :
184
- sources$1 = [SimulateSource.make(sources.items, sources.endBlock, chainId, addressStore, config.ecosystem.name, Primitive_option.some(sources.transactionStore), Primitive_option.some(sources.blockStore))];
198
+ sources$1 = [SimulateSource.make(sources.items, sources.endBlock, chainId$1, addressStore, config.ecosystem.name, Primitive_option.some(sources.transactionStore), Primitive_option.some(sources.blockStore))];
185
199
  break;
186
200
  case "CustomSources" :
187
201
  sources$1 = sources._0;
@@ -197,23 +211,7 @@ function makeInternal(chainConfig, indexingAddresses, startBlock, endBlock, firs
197
211
  }
198
212
  let firstEventBlock$1 = fetchState.firstEventBlock;
199
213
  let chainDensity = firstEventBlock$1 !== undefined && progressBlockNumber > firstEventBlock$1 && numEventsProcessed > 0 ? numEventsProcessed / (progressBlockNumber - firstEventBlock$1 | 0) : undefined;
200
- return make(chainConfig, fetchState, onEventRegistrations, addressStore, SourceManager.make(sources$1, isRealtime, undefined, undefined, undefined, reducedPollingInterval, undefined, undefined), progressBlockNumber, Primitive_option.some(SafeCheckpointTracking.make(maxReorgDepth, config.shouldRollbackOnReorg, chainReorgCheckpoints)), config.shouldRollbackOnReorg, maxReorgDepth, numEventsProcessed, Primitive_option.some(timestampCaughtUpToHeadOrEndblock), undefined, Primitive_option.some(TransactionStore.make(config.ecosystem.name, !lowercaseAddresses)), Primitive_option.some(chainDensity), Primitive_option.some(blockStore), config.reorgThresholdReadyTolerance, EntityTables.perChain(config.allEntities), logger);
201
- }
202
-
203
- function makeFromConfig(chainConfig, config, registrationsByChainId, knownHeight) {
204
- let logger = Logging.createChild({
205
- chainId: chainConfig.id
206
- });
207
- return makeInternal(chainConfig, configAddresses(chainConfig), chainConfig.startBlock, chainConfig.endBlock, undefined, -1, config, registrationsByChainId, logger, undefined, 0, false, false, [], chainConfig.maxReorgDepth, knownHeight, undefined, undefined);
208
- }
209
-
210
- function makeFromDbState(chainConfig, resumedChainState, reorgCheckpoints, isInReorgThreshold, isRealtime, config, registrationsByChainId, reducedPollingInterval) {
211
- let chainId = chainConfig.id;
212
- let logger = Logging.createChild({
213
- chainId: chainId
214
- });
215
- let progressBlockNumber = resumedChainState.progressBlockNumber >= 0 ? resumedChainState.progressBlockNumber : resumedChainState.startBlock - 1 | 0;
216
- return makeInternal(chainConfig, resumedChainState.indexingAddresses, resumedChainState.startBlock, resumedChainState.endBlock, Primitive_option.some(resumedChainState.firstEventBlockNumber), progressBlockNumber, config, registrationsByChainId, logger, resumedChainState.timestampCaughtUpToHeadOrEndblock, resumedChainState.numEventsProcessed, isInReorgThreshold, isRealtime, reorgCheckpoints, resumedChainState.maxReorgDepth, resumedChainState.sourceBlockNumber, true, reducedPollingInterval);
214
+ return make(chainConfig, fetchState, onEventRegistrations, addressStore, SourceManager.make(sources$1, isRealtime, undefined, undefined, undefined, reducedPollingInterval, undefined, undefined), progressBlockNumber, Primitive_option.some(SafeCheckpointTracking.make(maxReorgDepth, config.shouldRollbackOnReorg, chainReorgCheckpoints)), config.shouldRollbackOnReorg, maxReorgDepth, numEventsProcessed, Primitive_option.some(timestampCaughtUpToHeadOrEndblock), undefined, Primitive_option.some(TransactionStore.make(config.ecosystem.name, !lowercaseAddresses)), Primitive_option.some(chainDensity), Primitive_option.some(blockStore), config.reorgThresholdReadyTolerance, EntityTables.perChain(config.userEntities), logger);
217
215
  }
218
216
 
219
217
  function logger(cs) {
@@ -721,6 +719,10 @@ function toMetrics(cs) {
721
719
  startBlock: cs.fetchState.startBlock,
722
720
  endBlock: cs.fetchState.endBlock,
723
721
  numAddresses: cs.addressStore.size(),
722
+ addressesByContract: cs.addressStore.contractCounts().map(param => [
723
+ param.contractName,
724
+ param.count
725
+ ]),
724
726
  isReady: cs.timestampCaughtUpToHeadOrEndblock !== undefined,
725
727
  sourceBlockNumber: cs.fetchState.knownHeight,
726
728
  progressBlockNumber: cs.committedProgressBlockNumber,
@@ -855,36 +857,48 @@ function markReady(cs, readyAt) {
855
857
  }
856
858
  }
857
859
 
858
- function rollback(cs, newProgressBlockNumber, eventsProcessedDiff, rollbackTargetBlockNumber, isReorgChain) {
859
- if (newProgressBlockNumber === undefined) {
860
- if (isReorgChain) {
861
- cs.fetchState = FetchState.rollback(cs.fetchState, cs.addressStore, rollbackTargetBlockNumber);
862
- cs.transactionStore.rollback(rollbackTargetBlockNumber);
863
- cs.blockStore.rollback(rollbackTargetBlockNumber);
864
- cs.committedProgressBlockNumber = Primitive_int.min(cs.committedProgressBlockNumber, rollbackTargetBlockNumber);
865
- cs.processingBlockNumber = Primitive_int.min(cs.processingBlockNumber, rollbackTargetBlockNumber);
866
- return;
867
- } else {
868
- return;
869
- }
860
+ function rollback(cs, rolledBackTo) {
861
+ let rollbackTo = targetBlockNumber => {
862
+ let match = FetchState.rollback(cs.fetchState, cs.addressStore, targetBlockNumber);
863
+ cs.fetchState = match.fetchState;
864
+ return match.rolledBackAddresses.map(param => ({
865
+ chainId: cs.chainConfig.id,
866
+ address: param.address,
867
+ contractId: param.contractId
868
+ }));
869
+ };
870
+ if (typeof rolledBackTo !== "object") {
871
+ return [];
870
872
  }
871
- let newTotalEventsProcessed = cs.numEventsProcessed - Stdlib_Option.getOrThrow(eventsProcessedDiff, "Missing events-processed diff for rolled-back chain");
872
- let safeCheckpointTracking = cs.safeCheckpointTracking;
873
- if (safeCheckpointTracking !== undefined) {
874
- cs.safeCheckpointTracking = SafeCheckpointTracking.rollback(safeCheckpointTracking, newProgressBlockNumber);
873
+ if (rolledBackTo.TAG === "RecomputedProgress") {
874
+ let match = rolledBackTo._0;
875
+ let newProgressBlockNumber = Primitive_int.min(match.blockNumber, cs.committedProgressBlockNumber);
876
+ let newTotalEventsProcessed = cs.numEventsProcessed - match.eventsProcessed;
877
+ let safeCheckpointTracking = cs.safeCheckpointTracking;
878
+ if (safeCheckpointTracking !== undefined) {
879
+ cs.safeCheckpointTracking = SafeCheckpointTracking.rollback(safeCheckpointTracking, newProgressBlockNumber);
880
+ }
881
+ let rolledBackAddresses = rollbackTo(newProgressBlockNumber);
882
+ cs.transactionStore.rollback(newProgressBlockNumber);
883
+ cs.blockStore.rollback(newProgressBlockNumber);
884
+ cs.committedProgressBlockNumber = newProgressBlockNumber;
885
+ cs.processingBlockNumber = newProgressBlockNumber;
886
+ cs.numEventsProcessed = newTotalEventsProcessed;
887
+ return rolledBackAddresses;
875
888
  }
876
- cs.fetchState = FetchState.rollback(cs.fetchState, cs.addressStore, newProgressBlockNumber);
877
- cs.transactionStore.rollback(newProgressBlockNumber);
878
- cs.blockStore.rollback(newProgressBlockNumber);
879
- cs.committedProgressBlockNumber = newProgressBlockNumber;
880
- cs.processingBlockNumber = newProgressBlockNumber;
881
- cs.numEventsProcessed = newTotalEventsProcessed;
889
+ let forkBlock = rolledBackTo._0;
890
+ let rolledBackAddresses$1 = rollbackTo(forkBlock);
891
+ cs.transactionStore.rollback(forkBlock);
892
+ cs.blockStore.rollback(forkBlock);
893
+ cs.committedProgressBlockNumber = Primitive_int.min(cs.committedProgressBlockNumber, forkBlock);
894
+ cs.processingBlockNumber = Primitive_int.min(cs.processingBlockNumber, forkBlock);
895
+ return rolledBackAddresses$1;
882
896
  }
883
897
 
884
898
  export {
885
- configAddresses,
899
+ configStorageRows,
900
+ seedConfigAddresses,
886
901
  make,
887
- makeFromConfig,
888
902
  makeFromDbState,
889
903
  logger,
890
904
  blockStore,
@@ -3,7 +3,18 @@
3
3
 
4
4
  type t
5
5
 
6
- let configAddresses: Config.chain => array<Internal.indexingAddress>
6
+ let configStorageRows: (
7
+ Config.chain,
8
+ ~ecosystem: Ecosystem.name,
9
+ ~contractMapping: ContractMapping.t,
10
+ ) => array<AddressRows.row>
11
+
12
+ let seedConfigAddresses: (
13
+ AddressRows.Table.t,
14
+ ~chainConfigs: array<Config.chain>,
15
+ ~ecosystem: Ecosystem.name,
16
+ ~contractMapping: ContractMapping.t,
17
+ ) => unit
7
18
 
8
19
  let make: (
9
20
  ~chainConfig: Config.chain,
@@ -26,13 +37,6 @@ let make: (
26
37
  ~logger: Pino.t,
27
38
  ) => t
28
39
 
29
- let makeFromConfig: (
30
- Config.chain,
31
- ~config: Config.t,
32
- ~registrationsByChainId: HandlerRegister.registrationsByChainId,
33
- ~knownHeight: int,
34
- ) => t
35
-
36
40
  let makeFromDbState: (
37
41
  Config.chain,
38
42
  ~resumedChainState: Persistence.initialChainState,
@@ -40,6 +44,7 @@ let makeFromDbState: (
40
44
  ~isInReorgThreshold: bool,
41
45
  ~isRealtime: bool,
42
46
  ~config: Config.t,
47
+ ~contractMapping: ContractMapping.t,
43
48
  ~registrationsByChainId: HandlerRegister.registrationsByChainId,
44
49
  ~reducedPollingInterval: int=?,
45
50
  ) => t
@@ -152,10 +157,18 @@ let enterReorgThreshold: t => unit
152
157
  let advanceAfterBatch: (t, ~batch: Batch.t, ~enteringReorgThreshold: bool) => unit
153
158
  let applyBatchProgress: (t, ~batch: Batch.t, ~blockTimestampName: string) => unit
154
159
  let markReady: (t, ~readyAt: Date.t) => unit
155
- let rollback: (
156
- t,
157
- ~newProgressBlockNumber: option<int>,
158
- ~eventsProcessedDiff: option<float>,
159
- ~rollbackTargetBlockNumber: int,
160
- ~isReorgChain: bool,
161
- ) => unit
160
+ // What a rollback's progress diff says about one chain: the block its surviving
161
+ // checkpoints put it back to, and the events the deleted ones carried away. One
162
+ // record because one diff row yields both — they are never known separately.
163
+ type progressDiff = {blockNumber: int, eventsProcessed: float}
164
+
165
+ // Where a rollback leaves one chain.
166
+ type rolledBackTo =
167
+ | RecomputedProgress(progressDiff)
168
+ // The reorg chain with no checkpoint left at or below the target, so only the
169
+ // fork block says where it goes.
170
+ | ForkBlock(int)
171
+ // A chain this rollback doesn't reach.
172
+ | Untouched
173
+
174
+ let rollback: (t, ~rolledBackTo: rolledBackTo) => array<AddressRows.key>