envio 3.2.1 → 3.3.0-alpha.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (106) hide show
  1. package/package.json +6 -7
  2. package/src/Batch.res +12 -15
  3. package/src/Batch.res.mjs +4 -5
  4. package/src/BatchProcessing.res +199 -0
  5. package/src/BatchProcessing.res.mjs +125 -0
  6. package/src/ChainFetching.res +375 -0
  7. package/src/ChainFetching.res.mjs +206 -0
  8. package/src/ChainMetadata.res +27 -0
  9. package/src/ChainMetadata.res.mjs +27 -0
  10. package/src/ChainState.res +618 -0
  11. package/src/ChainState.res.mjs +461 -0
  12. package/src/ChainState.resi +85 -0
  13. package/src/Config.res +13 -4
  14. package/src/Config.res.mjs +8 -3
  15. package/src/ContractRegisterContext.res +106 -0
  16. package/src/ContractRegisterContext.res.mjs +76 -0
  17. package/src/Core.res +3 -0
  18. package/src/CrossChainState.res +269 -0
  19. package/src/CrossChainState.res.mjs +197 -0
  20. package/src/CrossChainState.resi +47 -0
  21. package/src/Ecosystem.res +58 -0
  22. package/src/Ecosystem.res.mjs +43 -1
  23. package/src/Env.res +7 -2
  24. package/src/Env.res.mjs +5 -2
  25. package/src/EventConfigBuilder.res +2 -2
  26. package/src/EventProcessing.res +60 -46
  27. package/src/EventProcessing.res.mjs +33 -32
  28. package/src/EventUtils.res +0 -4
  29. package/src/EventUtils.res.mjs +0 -4
  30. package/src/ExitOnCaughtUp.res +10 -0
  31. package/src/ExitOnCaughtUp.res.mjs +22 -0
  32. package/src/FetchState.res +124 -62
  33. package/src/FetchState.res.mjs +159 -93
  34. package/src/InMemoryStore.res +36 -451
  35. package/src/InMemoryStore.res.mjs +20 -369
  36. package/src/IndexerLoop.res +44 -0
  37. package/src/IndexerLoop.res.mjs +46 -0
  38. package/src/IndexerState.res +544 -0
  39. package/src/IndexerState.res.mjs +547 -0
  40. package/src/IndexerState.resi +132 -0
  41. package/src/Internal.res +29 -8
  42. package/src/LoadLayer.res +20 -18
  43. package/src/LoadLayer.res.mjs +14 -12
  44. package/src/LoadLayer.resi +6 -3
  45. package/src/Logging.res +11 -44
  46. package/src/Logging.res.mjs +10 -42
  47. package/src/Main.res +54 -79
  48. package/src/Main.res.mjs +44 -56
  49. package/src/PgStorage.res +8 -71
  50. package/src/PgStorage.res.mjs +3 -47
  51. package/src/Prometheus.res +7 -8
  52. package/src/Prometheus.res.mjs +7 -4
  53. package/src/PruneStaleHistory.res +45 -0
  54. package/src/PruneStaleHistory.res.mjs +46 -0
  55. package/src/RawEvent.res +73 -0
  56. package/src/RawEvent.res.mjs +51 -0
  57. package/src/Rollback.res +204 -0
  58. package/src/Rollback.res.mjs +118 -0
  59. package/src/SimulateItems.res +12 -10
  60. package/src/SimulateItems.res.mjs +1 -1
  61. package/src/UserContext.res +26 -114
  62. package/src/UserContext.res.mjs +15 -78
  63. package/src/Writing.res +242 -0
  64. package/src/Writing.res.mjs +215 -0
  65. package/src/db/InternalTable.res +14 -27
  66. package/src/sources/Evm.res +40 -1
  67. package/src/sources/Evm.res.mjs +94 -84
  68. package/src/sources/Fuel.res +40 -1
  69. package/src/sources/Fuel.res.mjs +36 -26
  70. package/src/sources/HyperFuel.res +41 -120
  71. package/src/sources/HyperFuel.res.mjs +42 -80
  72. package/src/sources/HyperFuel.resi +1 -24
  73. package/src/sources/HyperFuelClient.res +16 -297
  74. package/src/sources/HyperFuelClient.res.mjs +5 -4
  75. package/src/sources/HyperFuelSource.res +33 -8
  76. package/src/sources/HyperFuelSource.res.mjs +56 -10
  77. package/src/sources/HyperSyncSource.res +5 -3
  78. package/src/sources/HyperSyncSource.res.mjs +1 -1
  79. package/src/sources/RpcSource.res +2 -2
  80. package/src/sources/RpcSource.res.mjs +1 -1
  81. package/src/sources/SourceManager.res +8 -18
  82. package/src/sources/SourceManager.res.mjs +8 -5
  83. package/src/sources/SourceManager.resi +4 -1
  84. package/src/sources/Svm.res +18 -1
  85. package/src/sources/Svm.res.mjs +30 -22
  86. package/src/sources/SvmHyperSyncSource.res +1 -1
  87. package/src/sources/SvmHyperSyncSource.res.mjs +1 -1
  88. package/src/tui/Tui.res +38 -36
  89. package/src/tui/Tui.res.mjs +51 -51
  90. package/src/ChainFetcher.res +0 -519
  91. package/src/ChainFetcher.res.mjs +0 -314
  92. package/src/ChainManager.res +0 -358
  93. package/src/ChainManager.res.mjs +0 -291
  94. package/src/Ctx.res +0 -6
  95. package/src/Ctx.res.mjs +0 -2
  96. package/src/GlobalState.res +0 -1056
  97. package/src/GlobalState.res.mjs +0 -1086
  98. package/src/GlobalStateManager.res +0 -57
  99. package/src/GlobalStateManager.res.mjs +0 -68
  100. package/src/GlobalStateManager.resi +0 -7
  101. package/src/Ports.res +0 -5
  102. package/src/Ports.res.mjs +0 -9
  103. package/src/adapters/MarkBatchProcessedAdapter.res +0 -5
  104. package/src/adapters/MarkBatchProcessedAdapter.res.mjs +0 -14
  105. package/src/sources/EnvioApiClient.res +0 -15
  106. package/src/sources/EnvioApiClient.res.mjs +0 -24
@@ -400,10 +400,13 @@ let IndexingAddresses = {
400
400
  set: set$4
401
401
  };
402
402
 
403
- let gauge$5 = makeOrThrow$1("envio_indexing_max_concurrency", "The maximum number of concurrent queries to the chain data-source.", chainIdLabelsSchema);
403
+ let gauge$5 = PromClient.Gauge.makeGauge({
404
+ name: "envio_indexing_max_concurrency",
405
+ help: "The maximum number of concurrent data-source queries across the whole indexer."
406
+ });
404
407
 
405
- function set$5(maxConcurrency, chainId) {
406
- handleInt$1(gauge$5, chainId, maxConcurrency);
408
+ function set$5(maxConcurrency) {
409
+ gauge$5.set(maxConcurrency);
407
410
  }
408
411
 
409
412
  let IndexingMaxConcurrency = {
@@ -464,7 +467,7 @@ let IndexingBufferSize = {
464
467
 
465
468
  let gauge$9 = PromClient.Gauge.makeGauge({
466
469
  name: "envio_indexing_target_buffer_size",
467
- help: "The target buffer size per chain for indexing. The actual number of items in the queue may exceed this value, but the indexer always tries to keep the buffer filled up to this target."
470
+ help: "The indexer-wide target buffer size shared across all chains. The actual number of items in the queue may exceed this value, but the indexer always tries to keep the buffer filled up to this target."
468
471
  });
469
472
 
470
473
  function set$9(targetBufferSize) {
@@ -0,0 +1,45 @@
1
+ // Throttled prune of stale entity-history rows below the safe checkpoint.
2
+
3
+ let runPrune = async (state: IndexerState.t) => {
4
+ switch state->IndexerState.getSafeCheckpointId {
5
+ | None => ()
6
+ | Some(safeCheckpointId) =>
7
+ let persistence = state->IndexerState.persistence
8
+ await persistence.storage.pruneStaleCheckpoints(~safeCheckpointId)
9
+
10
+ for idx in 0 to persistence.allEntities->Array.length - 1 {
11
+ if idx !== 0 {
12
+ // Add some delay between entities
13
+ // To unblock the pg client if it's needed for something else
14
+ await Utils.delay(1000)
15
+ }
16
+ let entityConfig = persistence.allEntities->Array.getUnsafe(idx)
17
+ let timeRef = Hrtime.makeTimer()
18
+ try {
19
+ let () = await persistence.storage.pruneStaleEntityHistory(
20
+ ~entityName=entityConfig.name,
21
+ ~entityIndex=entityConfig.index,
22
+ ~safeCheckpointId,
23
+ )
24
+ } catch {
25
+ | exn =>
26
+ exn->ErrorHandling.mkLogAndRaise(
27
+ ~msg=`Failed to prune stale entity history`,
28
+ ~logger=Logging.createChild(
29
+ ~params={
30
+ "entityName": entityConfig.name,
31
+ "safeCheckpointId": safeCheckpointId,
32
+ },
33
+ ),
34
+ )
35
+ }
36
+ Prometheus.RollbackHistoryPrune.increment(
37
+ ~timeSeconds=Hrtime.timeSince(timeRef)->Hrtime.toSecondsFloat,
38
+ ~entityName=entityConfig.name,
39
+ )
40
+ }
41
+ }
42
+ }
43
+
44
+ let schedule = (state: IndexerState.t) =>
45
+ state->IndexerState.pruneStaleEntityHistoryThrottler->Throttler.schedule(() => runPrune(state))
@@ -0,0 +1,46 @@
1
+ // Generated by ReScript, PLEASE EDIT WITH CARE
2
+
3
+ import * as Utils from "./Utils.res.mjs";
4
+ import * as Hrtime from "./bindings/Hrtime.res.mjs";
5
+ import * as Logging from "./Logging.res.mjs";
6
+ import * as Throttler from "./Throttler.res.mjs";
7
+ import * as Prometheus from "./Prometheus.res.mjs";
8
+ import * as IndexerState from "./IndexerState.res.mjs";
9
+ import * as ErrorHandling from "./ErrorHandling.res.mjs";
10
+ import * as Primitive_exceptions from "@rescript/runtime/lib/es6/Primitive_exceptions.js";
11
+
12
+ async function runPrune(state) {
13
+ let safeCheckpointId = IndexerState.getSafeCheckpointId(state);
14
+ if (safeCheckpointId === undefined) {
15
+ return;
16
+ }
17
+ let persistence = IndexerState.persistence(state);
18
+ await persistence.storage.pruneStaleCheckpoints(safeCheckpointId);
19
+ for (let idx = 0, idx_finish = persistence.allEntities.length; idx < idx_finish; ++idx) {
20
+ if (idx !== 0) {
21
+ await Utils.delay(1000);
22
+ }
23
+ let entityConfig = persistence.allEntities[idx];
24
+ let timeRef = Hrtime.makeTimer();
25
+ try {
26
+ await persistence.storage.pruneStaleEntityHistory(entityConfig.name, entityConfig.index, safeCheckpointId);
27
+ } catch (raw_exn) {
28
+ let exn = Primitive_exceptions.internalToException(raw_exn);
29
+ ErrorHandling.mkLogAndRaise(Logging.createChild({
30
+ entityName: entityConfig.name,
31
+ safeCheckpointId: safeCheckpointId
32
+ }), `Failed to prune stale entity history`, exn);
33
+ }
34
+ Prometheus.RollbackHistoryPrune.increment(Hrtime.toSecondsFloat(Hrtime.timeSince(timeRef)), entityConfig.name);
35
+ }
36
+ }
37
+
38
+ function schedule(state) {
39
+ Throttler.schedule(IndexerState.pruneStaleEntityHistoryThrottler(state), () => runPrune(state));
40
+ }
41
+
42
+ export {
43
+ runPrune,
44
+ schedule,
45
+ }
46
+ /* Utils Not a pure module */
@@ -0,0 +1,73 @@
1
+ let convertFieldsToJson = (fields: option<dict<unknown>>) => {
2
+ switch fields {
3
+ | None => %raw(`{}`)
4
+ | Some(fields) =>
5
+ // Convert bigint fields to string. There are no fields with nested
6
+ // bigints, so iterating only the top level is safe.
7
+ fields
8
+ ->Utils.Dict.mapValues(value =>
9
+ typeof(value) === #bigint
10
+ ? value
11
+ ->(Utils.magic: unknown => bigint)
12
+ ->BigInt.toString
13
+ ->(Utils.magic: string => unknown)
14
+ : value
15
+ )
16
+ ->(Utils.magic: dict<unknown> => JSON.t)
17
+ }
18
+ }
19
+
20
+ // Block and transaction are passed already extracted from the ecosystem's
21
+ // concrete payload (EVM or Fuel) — `RawEvent` stays payload-shape-agnostic and
22
+ // only needs them as opaque field bags to serialise.
23
+ let make = (
24
+ eventItem: Internal.eventItem,
25
+ ~block,
26
+ ~transaction,
27
+ ~params: Internal.eventParams,
28
+ ~srcAddress: Address.t,
29
+ ~cleanUpRawEventFieldsInPlace: JSON.t => unit,
30
+ ): Internal.rawEvent => {
31
+ let {eventConfig, chain, blockNumber, blockHash, timestamp: blockTimestamp, logIndex} = eventItem
32
+ let chainId = chain->ChainMap.Chain.toChainId
33
+ let eventId = EventUtils.packEventIndex(~logIndex, ~blockNumber)
34
+ let blockFields =
35
+ block
36
+ ->(Utils.magic: 'block => option<dict<unknown>>)
37
+ ->convertFieldsToJson
38
+ let transactionFields =
39
+ transaction
40
+ ->(Utils.magic: 'transaction => option<dict<unknown>>)
41
+ ->convertFieldsToJson
42
+
43
+ blockFields->cleanUpRawEventFieldsInPlace
44
+
45
+ // Serialize to unknown, because serializing to Js.Json.t fails for Bytes Fuel type, since it has unknown schema
46
+ let params =
47
+ params
48
+ ->S.reverseConvertOrThrow(eventConfig.paramsRawEventSchema)
49
+ ->(Utils.magic: unknown => JSON.t)
50
+ let params = if params === %raw(`null`) {
51
+ // Should probably make the params field nullable
52
+ // But this is currently needed to make events
53
+ // with empty params work
54
+ %raw(`"null"`)
55
+ } else {
56
+ params
57
+ }
58
+
59
+ {
60
+ chain_id: chainId,
61
+ event_id: eventId,
62
+ event_name: eventConfig.name,
63
+ contract_name: eventConfig.contractName,
64
+ block_number: blockNumber,
65
+ log_index: logIndex,
66
+ src_address: srcAddress,
67
+ block_hash: blockHash,
68
+ block_timestamp: blockTimestamp,
69
+ block_fields: blockFields,
70
+ transaction_fields: transactionFields,
71
+ params,
72
+ }
73
+ }
@@ -0,0 +1,51 @@
1
+ // Generated by ReScript, PLEASE EDIT WITH CARE
2
+
3
+ import * as Utils from "./Utils.res.mjs";
4
+ import * as EventUtils from "./EventUtils.res.mjs";
5
+ import * as S$RescriptSchema from "rescript-schema/src/S.res.mjs";
6
+
7
+ function convertFieldsToJson(fields) {
8
+ if (fields !== undefined) {
9
+ return Utils.Dict.mapValues(fields, value => {
10
+ if (typeof value === "bigint") {
11
+ return value.toString();
12
+ } else {
13
+ return value;
14
+ }
15
+ });
16
+ } else {
17
+ return {};
18
+ }
19
+ }
20
+
21
+ function make(eventItem, block, transaction, params, srcAddress, cleanUpRawEventFieldsInPlace) {
22
+ let logIndex = eventItem.logIndex;
23
+ let blockNumber = eventItem.blockNumber;
24
+ let eventConfig = eventItem.eventConfig;
25
+ let eventId = EventUtils.packEventIndex(blockNumber, logIndex);
26
+ let blockFields = convertFieldsToJson(block);
27
+ let transactionFields = convertFieldsToJson(transaction);
28
+ cleanUpRawEventFieldsInPlace(blockFields);
29
+ let params$1 = S$RescriptSchema.reverseConvertOrThrow(params, eventConfig.paramsRawEventSchema);
30
+ let params$2 = params$1 === null ? "null" : params$1;
31
+ return {
32
+ chain_id: eventItem.chain,
33
+ event_id: eventId,
34
+ event_name: eventConfig.name,
35
+ contract_name: eventConfig.contractName,
36
+ block_number: blockNumber,
37
+ log_index: logIndex,
38
+ src_address: srcAddress,
39
+ block_hash: eventItem.blockHash,
40
+ block_timestamp: eventItem.timestamp,
41
+ block_fields: blockFields,
42
+ transaction_fields: transactionFields,
43
+ params: params$2
44
+ };
45
+ }
46
+
47
+ export {
48
+ convertFieldsToJson,
49
+ make,
50
+ }
51
+ /* Utils Not a pure module */
@@ -0,0 +1,204 @@
1
+ // The reorg rollback state machine. Re-enters fetch/process/rollback through
2
+ // the injected schedule* effects.
3
+
4
+ /**
5
+ Finds the last known valid block number below the reorg block
6
+ If not found, returns the highest block below threshold
7
+ */
8
+ let getLastKnownValidBlock = async (
9
+ chainState: ChainState.t,
10
+ ~reorgBlockNumber: int,
11
+ ~isRealtime: bool,
12
+ ) => {
13
+ // Don't include the reorg block itself — different source instances
14
+ // may have mismatching hashes at the head, so we always rollback
15
+ // the block where we detected the reorg.
16
+ let scannedBlockNumbers =
17
+ chainState
18
+ ->ChainState.reorgDetection
19
+ ->ReorgDetection.getThresholdBlockNumbersBelowBlock(
20
+ ~blockNumber=reorgBlockNumber,
21
+ ~knownHeight=(chainState->ChainState.fetchState).knownHeight,
22
+ )
23
+
24
+ switch scannedBlockNumbers {
25
+ | [] => chainState->ChainState.getHighestBlockBelowThreshold
26
+ | _ => {
27
+ let blockNumbersAndHashes = await chainState
28
+ ->ChainState.sourceManager
29
+ ->SourceManager.getBlockHashes(~blockNumbers=scannedBlockNumbers, ~isRealtime)
30
+
31
+ switch chainState
32
+ ->ChainState.reorgDetection
33
+ ->ReorgDetection.getLatestValidScannedBlock(~blockNumbersAndHashes) {
34
+ | Some(blockNumber) => blockNumber
35
+ | None => chainState->ChainState.getHighestBlockBelowThreshold
36
+ }
37
+ }
38
+ }
39
+ }
40
+
41
+ let rec rollback = async (
42
+ state: IndexerState.t,
43
+ ~scheduleFetch,
44
+ ~scheduleProcessing,
45
+ ~scheduleRollback,
46
+ ) =>
47
+ // Owns its error boundary: launch doesn't catch, so a failure mid-rollback
48
+ // must stop the indexer.
49
+ try {
50
+ switch state->IndexerState.rollbackState {
51
+ | NoRollback | RollbackReady(_) =>
52
+ JsError.throwWithMessage("Internal error: Rollback initiated with invalid state")
53
+ | ReorgDetected({chain, blockNumber: reorgBlockNumber}) =>
54
+ let chainState = state->IndexerState.getChainState(~chain)
55
+
56
+ state->IndexerState.enterFindingReorgDepth
57
+ let rollbackTargetBlockNumber = await chainState->getLastKnownValidBlock(
58
+ ~reorgBlockNumber,
59
+ ~isRealtime=state->IndexerState.isRealtime,
60
+ )
61
+
62
+ chainState
63
+ ->ChainState.sourceManager
64
+ ->SourceManager.onReorg(~rollbackTargetBlock=rollbackTargetBlockNumber)
65
+
66
+ state->IndexerState.foundReorgDepth(~chain, ~rollbackTargetBlockNumber)
67
+ // Rendezvous with the processing loop: whichever of {depth found, loop
68
+ // idle} happens last triggers the rollback; the earlier one finds the
69
+ // other condition unmet and bails here.
70
+ scheduleRollback()
71
+ // Reached when a batch finished (loop idle) while the reorg depth wasn't
72
+ // found yet. Wait for the ReorgDetected branch above to find it and re-kick.
73
+ | FindingReorgDepth => ()
74
+ | FoundReorgDepth(_) if state->IndexerState.isProcessing =>
75
+ Logging.info("Waiting for batch to finish processing before executing rollback")
76
+ | FoundReorgDepth({chain: reorgChain, rollbackTargetBlockNumber}) =>
77
+ await executeRollback(
78
+ state,
79
+ ~reorgChain,
80
+ ~rollbackTargetBlockNumber,
81
+ ~scheduleFetch,
82
+ ~scheduleProcessing,
83
+ )
84
+ }
85
+ } catch {
86
+ | exn =>
87
+ IndexerState.errorExit(state, exn->ErrorHandling.make(~msg=IndexerState.unexpectedErrorMsg))
88
+ }
89
+
90
+ and executeRollback = async (
91
+ state: IndexerState.t,
92
+ ~reorgChain,
93
+ ~rollbackTargetBlockNumber,
94
+ ~scheduleFetch,
95
+ ~scheduleProcessing,
96
+ ) => {
97
+ let startTime = Hrtime.makeTimer()
98
+
99
+ let chainState = state->IndexerState.getChainState(~chain=reorgChain)
100
+
101
+ let logger = Logging.createChildFrom(
102
+ ~logger=chainState->ChainState.logger,
103
+ ~params={
104
+ "action": "Rollback",
105
+ "reorgChain": reorgChain,
106
+ "targetBlockNumber": rollbackTargetBlockNumber,
107
+ },
108
+ )
109
+ logger->Logging.childInfo("Started rollback on reorg")
110
+ Prometheus.RollbackTargetBlockNumber.set(
111
+ ~blockNumber=rollbackTargetBlockNumber,
112
+ ~chain=reorgChain,
113
+ )
114
+
115
+ let reorgChainId = reorgChain->ChainMap.Chain.toChainId
116
+
117
+ // Finish pending batch writes first: the target checkpoint, the progress
118
+ // diff and the rollback diff below must all be computed from the same db
119
+ // state. Otherwise an in-flight batch lands after the progress reads and
120
+ // its entity changes get reverted without the chain progress being
121
+ // rolled back, so the events are never reprocessed.
122
+ await state->Writing.flush
123
+
124
+ let rollbackTargetCheckpointId = {
125
+ switch await (state->IndexerState.persistence).storage.getRollbackTargetCheckpoint(
126
+ ~reorgChainId,
127
+ ~lastKnownValidBlockNumber=rollbackTargetBlockNumber,
128
+ ) {
129
+ | Some(checkpointId) => checkpointId
130
+ | None => 0n
131
+ }
132
+ }
133
+
134
+ let eventsProcessedDiffByChain = Dict.make()
135
+ let newProgressBlockNumberPerChain = Dict.make()
136
+ let rollbackedProcessedEvents = ref(0.)
137
+
138
+ {
139
+ let rollbackProgressDiff = await (
140
+ state->IndexerState.persistence
141
+ ).storage.getRollbackProgressDiff(~rollbackTargetCheckpointId)
142
+ for idx in 0 to rollbackProgressDiff->Array.length - 1 {
143
+ let diff = rollbackProgressDiff->Array.getUnsafe(idx)
144
+ eventsProcessedDiffByChain->Utils.Dict.setByInt(
145
+ diff["chain_id"],
146
+ {
147
+ let eventsProcessedDiff =
148
+ Float.fromString(diff["events_processed_diff"])->Option.getOrThrow
149
+ rollbackedProcessedEvents := rollbackedProcessedEvents.contents +. eventsProcessedDiff
150
+ eventsProcessedDiff
151
+ },
152
+ )
153
+ newProgressBlockNumberPerChain->Utils.Dict.setByInt(
154
+ diff["chain_id"],
155
+ if rollbackTargetCheckpointId === 0n && diff["chain_id"] === reorgChainId {
156
+ Pervasives.min(diff["new_progress_block_number"], rollbackTargetBlockNumber)
157
+ } else {
158
+ diff["new_progress_block_number"]
159
+ },
160
+ )
161
+ }
162
+ }
163
+
164
+ state
165
+ ->IndexerState.chainStates
166
+ ->Utils.Dict.forEach(cs => {
167
+ let chainId = (cs->ChainState.chainConfig).id
168
+ cs->ChainState.rollback(
169
+ ~newProgressBlockNumber=newProgressBlockNumberPerChain->Utils.Dict.dangerouslyGetByIntNonOption(
170
+ chainId,
171
+ ),
172
+ ~eventsProcessedDiff=eventsProcessedDiffByChain->Utils.Dict.dangerouslyGetByIntNonOption(
173
+ chainId,
174
+ ),
175
+ ~rollbackTargetBlockNumber,
176
+ ~isReorgChain=chainId === reorgChainId,
177
+ )
178
+ })
179
+
180
+ let diff = await state->InMemoryStore.prepareRollbackDiff(
181
+ ~rollbackTargetCheckpointId,
182
+ ~rollbackDiffCheckpointId=state->IndexerState.committedCheckpointId->BigInt.add(1n),
183
+ ~progressBlockNumberByChainId=newProgressBlockNumberPerChain,
184
+ )
185
+
186
+ logger->Logging.childTrace({
187
+ "msg": "Finished rollback on reorg",
188
+ "entityChanges": {
189
+ "deleted": diff["deletedEntities"],
190
+ "upserted": diff["setEntities"],
191
+ },
192
+ "rollbackedEvents": rollbackedProcessedEvents.contents,
193
+ "beforeCheckpointId": state->IndexerState.committedCheckpointId,
194
+ "targetCheckpointId": rollbackTargetCheckpointId,
195
+ })
196
+ Prometheus.RollbackSuccess.increment(
197
+ ~timeSeconds=Hrtime.timeSince(startTime)->Hrtime.toSecondsFloat,
198
+ ~rollbackedProcessedEvents=rollbackedProcessedEvents.contents,
199
+ )
200
+
201
+ state->IndexerState.completeRollback(~eventsProcessedDiffByChain)
202
+ scheduleFetch()
203
+ scheduleProcessing()
204
+ }
@@ -0,0 +1,118 @@
1
+ // Generated by ReScript, PLEASE EDIT WITH CARE
2
+
3
+ import * as Utils from "./Utils.res.mjs";
4
+ import * as Hrtime from "./bindings/Hrtime.res.mjs";
5
+ import * as Logging from "./Logging.res.mjs";
6
+ import * as Writing from "./Writing.res.mjs";
7
+ import * as ChainState from "./ChainState.res.mjs";
8
+ import * as Prometheus from "./Prometheus.res.mjs";
9
+ import * as IndexerState from "./IndexerState.res.mjs";
10
+ import * as Stdlib_Float from "@rescript/runtime/lib/es6/Stdlib_Float.js";
11
+ import * as ErrorHandling from "./ErrorHandling.res.mjs";
12
+ import * as InMemoryStore from "./InMemoryStore.res.mjs";
13
+ import * as Primitive_int from "@rescript/runtime/lib/es6/Primitive_int.js";
14
+ import * as SourceManager from "./sources/SourceManager.res.mjs";
15
+ import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js";
16
+ import * as ReorgDetection from "./ReorgDetection.res.mjs";
17
+ import * as Stdlib_JsError from "@rescript/runtime/lib/es6/Stdlib_JsError.js";
18
+ import * as Primitive_exceptions from "@rescript/runtime/lib/es6/Primitive_exceptions.js";
19
+
20
+ async function getLastKnownValidBlock(chainState, reorgBlockNumber, isRealtime) {
21
+ let scannedBlockNumbers = ReorgDetection.getThresholdBlockNumbersBelowBlock(ChainState.reorgDetection(chainState), reorgBlockNumber, ChainState.fetchState(chainState).knownHeight);
22
+ if (scannedBlockNumbers.length === 0) {
23
+ return ChainState.getHighestBlockBelowThreshold(chainState);
24
+ }
25
+ let blockNumbersAndHashes = await SourceManager.getBlockHashes(ChainState.sourceManager(chainState), scannedBlockNumbers, isRealtime);
26
+ let blockNumber = ReorgDetection.getLatestValidScannedBlock(ChainState.reorgDetection(chainState), blockNumbersAndHashes);
27
+ if (blockNumber !== undefined) {
28
+ return blockNumber;
29
+ } else {
30
+ return ChainState.getHighestBlockBelowThreshold(chainState);
31
+ }
32
+ }
33
+
34
+ async function executeRollback(state, reorgChain, rollbackTargetBlockNumber, scheduleFetch, scheduleProcessing) {
35
+ let startTime = Hrtime.makeTimer();
36
+ let chainState = IndexerState.getChainState(state, reorgChain);
37
+ let logger = Logging.createChildFrom(ChainState.logger(chainState), {
38
+ action: "Rollback",
39
+ reorgChain: reorgChain,
40
+ targetBlockNumber: rollbackTargetBlockNumber
41
+ });
42
+ Logging.childInfo(logger, "Started rollback on reorg");
43
+ Prometheus.RollbackTargetBlockNumber.set(rollbackTargetBlockNumber, reorgChain);
44
+ await Writing.flush(state);
45
+ let checkpointId = await IndexerState.persistence(state).storage.getRollbackTargetCheckpoint(reorgChain, rollbackTargetBlockNumber);
46
+ let rollbackTargetCheckpointId = checkpointId !== undefined ? checkpointId : 0n;
47
+ let eventsProcessedDiffByChain = {};
48
+ let newProgressBlockNumberPerChain = {};
49
+ let rollbackedProcessedEvents = 0;
50
+ let rollbackProgressDiff = await IndexerState.persistence(state).storage.getRollbackProgressDiff(rollbackTargetCheckpointId);
51
+ for (let idx = 0, idx_finish = rollbackProgressDiff.length; idx < idx_finish; ++idx) {
52
+ let diff = rollbackProgressDiff[idx];
53
+ let eventsProcessedDiff = Stdlib_Option.getOrThrow(Stdlib_Float.fromString(diff.events_processed_diff), undefined);
54
+ rollbackedProcessedEvents = rollbackedProcessedEvents + eventsProcessedDiff;
55
+ eventsProcessedDiffByChain[diff.chain_id] = eventsProcessedDiff;
56
+ newProgressBlockNumberPerChain[diff.chain_id] = rollbackTargetCheckpointId === 0n && diff.chain_id === reorgChain ? Primitive_int.min(diff.new_progress_block_number, rollbackTargetBlockNumber) : diff.new_progress_block_number;
57
+ }
58
+ Utils.Dict.forEach(IndexerState.chainStates(state), cs => {
59
+ let chainId = ChainState.chainConfig(cs).id;
60
+ ChainState.rollback(cs, newProgressBlockNumberPerChain[chainId], eventsProcessedDiffByChain[chainId], rollbackTargetBlockNumber, chainId === reorgChain);
61
+ });
62
+ let diff$1 = await InMemoryStore.prepareRollbackDiff(state, rollbackTargetCheckpointId, IndexerState.committedCheckpointId(state) + 1n, newProgressBlockNumberPerChain);
63
+ Logging.childTrace(logger, {
64
+ msg: "Finished rollback on reorg",
65
+ entityChanges: {
66
+ deleted: diff$1.deletedEntities,
67
+ upserted: diff$1.setEntities
68
+ },
69
+ rollbackedEvents: rollbackedProcessedEvents,
70
+ beforeCheckpointId: IndexerState.committedCheckpointId(state),
71
+ targetCheckpointId: rollbackTargetCheckpointId
72
+ });
73
+ Prometheus.RollbackSuccess.increment(Hrtime.toSecondsFloat(Hrtime.timeSince(startTime)), rollbackedProcessedEvents);
74
+ IndexerState.completeRollback(state, eventsProcessedDiffByChain);
75
+ scheduleFetch();
76
+ return scheduleProcessing();
77
+ }
78
+
79
+ async function rollback(state, scheduleFetch, scheduleProcessing, scheduleRollback) {
80
+ try {
81
+ let match = IndexerState.rollbackState(state);
82
+ if (typeof match !== "object") {
83
+ if (match === "NoRollback") {
84
+ return Stdlib_JsError.throwWithMessage("Internal error: Rollback initiated with invalid state");
85
+ } else {
86
+ return;
87
+ }
88
+ }
89
+ switch (match.TAG) {
90
+ case "ReorgDetected" :
91
+ let chain = match.chain;
92
+ let chainState = IndexerState.getChainState(state, chain);
93
+ IndexerState.enterFindingReorgDepth(state);
94
+ let rollbackTargetBlockNumber = await getLastKnownValidBlock(chainState, match.blockNumber, IndexerState.isRealtime(state));
95
+ SourceManager.onReorg(ChainState.sourceManager(chainState), rollbackTargetBlockNumber);
96
+ IndexerState.foundReorgDepth(state, chain, rollbackTargetBlockNumber);
97
+ return scheduleRollback();
98
+ case "FoundReorgDepth" :
99
+ if (IndexerState.isProcessing(state)) {
100
+ return Logging.info("Waiting for batch to finish processing before executing rollback");
101
+ } else {
102
+ return await executeRollback(state, match.chain, match.rollbackTargetBlockNumber, scheduleFetch, scheduleProcessing);
103
+ }
104
+ case "RollbackReady" :
105
+ return Stdlib_JsError.throwWithMessage("Internal error: Rollback initiated with invalid state");
106
+ }
107
+ } catch (raw_exn) {
108
+ let exn = Primitive_exceptions.internalToException(raw_exn);
109
+ return IndexerState.errorExit(state, ErrorHandling.make(exn, undefined, IndexerState.unexpectedErrorMsg));
110
+ }
111
+ }
112
+
113
+ export {
114
+ getLastKnownValidBlock,
115
+ rollback,
116
+ executeRollback,
117
+ }
118
+ /* Utils Not a pure module */
@@ -288,16 +288,18 @@ let parse = (~simulateItems: array<JSON.t>, ~config: Config.t, ~chainConfig: Con
288
288
  blockNumber,
289
289
  blockHash,
290
290
  logIndex,
291
- event: {
292
- contractName: eventConfig.contractName,
293
- eventName: eventConfig.name,
294
- params,
295
- chainId,
296
- srcAddress,
297
- logIndex,
298
- transaction,
299
- block,
300
- }->Internal.fromGenericEvent,
291
+ payload: (
292
+ {
293
+ contractName: eventConfig.contractName,
294
+ eventName: eventConfig.name,
295
+ params,
296
+ chainId,
297
+ srcAddress,
298
+ logIndex,
299
+ transaction,
300
+ block,
301
+ }: Evm.payload
302
+ )->Evm.fromPayload,
301
303
  }),
302
304
  )
303
305
  ->ignore
@@ -241,7 +241,7 @@ function parse(simulateItems, config, chainConfig) {
241
241
  blockNumber: blockNumber,
242
242
  blockHash: match$3[3],
243
243
  logIndex: logIndex,
244
- event: {
244
+ payload: {
245
245
  contractName: eventConfig.contractName,
246
246
  eventName: eventConfig.name,
247
247
  params: params,