envio 3.5.0-alpha.3 → 3.5.0-rc.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (127) hide show
  1. package/package.json +6 -6
  2. package/src/Batch.res +8 -8
  3. package/src/Batch.res.mjs +4 -3
  4. package/src/BatchProcessing.res +17 -7
  5. package/src/BatchProcessing.res.mjs +8 -4
  6. package/src/ChainFetching.res +20 -19
  7. package/src/ChainFetching.res.mjs +14 -14
  8. package/src/ChainId.res +58 -0
  9. package/src/ChainId.res.mjs +49 -0
  10. package/src/ChainId.resi +27 -0
  11. package/src/ChainMap.res +12 -26
  12. package/src/ChainMap.res.mjs +5 -41
  13. package/src/ChainMap.resi +5 -20
  14. package/src/ChainMetadata.res +1 -1
  15. package/src/ChainMetadata.res.mjs +3 -2
  16. package/src/ChainState.res +35 -20
  17. package/src/ChainState.res.mjs +34 -11
  18. package/src/ChainState.resi +2 -0
  19. package/src/Config.res +41 -24
  20. package/src/Config.res.mjs +17 -14
  21. package/src/ContractRegisterContext.res +1 -1
  22. package/src/ContractRegisterContext.res.mjs +1 -1
  23. package/src/Core.res +1 -0
  24. package/src/CrossChainState.res +61 -16
  25. package/src/CrossChainState.res.mjs +37 -13
  26. package/src/CrossChainState.resi +3 -2
  27. package/src/Env.res +0 -3
  28. package/src/Env.res.mjs +0 -3
  29. package/src/EventConfigBuilder.res +1 -1
  30. package/src/EventProcessing.res +4 -4
  31. package/src/EventProcessing.res.mjs +5 -5
  32. package/src/FetchState.res +261 -89
  33. package/src/FetchState.res.mjs +187 -63
  34. package/src/FinalizeBackfill.res +33 -6
  35. package/src/FinalizeBackfill.res.mjs +19 -1
  36. package/src/HandlerRegister.res +8 -8
  37. package/src/HandlerRegister.res.mjs +5 -4
  38. package/src/HandlerRegister.resi +1 -1
  39. package/src/InMemoryStore.res +4 -0
  40. package/src/InMemoryStore.res.mjs +1 -1
  41. package/src/IndexerLoop.res +9 -2
  42. package/src/IndexerLoop.res.mjs +8 -1
  43. package/src/IndexerState.res +41 -24
  44. package/src/IndexerState.res.mjs +36 -17
  45. package/src/IndexerState.resi +9 -7
  46. package/src/Internal.res +19 -17
  47. package/src/Internal.res.mjs +18 -9
  48. package/src/LoadLayer.res +1 -1
  49. package/src/LoadLayer.res.mjs +2 -1
  50. package/src/LogSelection.res +2 -2
  51. package/src/Main.res +7 -11
  52. package/src/Main.res.mjs +4 -6
  53. package/src/Metrics.res +6 -6
  54. package/src/Metrics.res.mjs +4 -3
  55. package/src/Persistence.res +15 -7
  56. package/src/PgStorage.res +284 -258
  57. package/src/PgStorage.res.mjs +181 -208
  58. package/src/RawEvent.res +1 -2
  59. package/src/RawEvent.res.mjs +1 -1
  60. package/src/Rollback.res +18 -20
  61. package/src/Rollback.res.mjs +9 -9
  62. package/src/RollbackCommit.res +2 -2
  63. package/src/RollbackCommit.res.mjs +3 -3
  64. package/src/SafeCheckpointTracking.res +2 -2
  65. package/src/SimulateDeadInputTracker.res +17 -21
  66. package/src/SimulateDeadInputTracker.res.mjs +9 -8
  67. package/src/SimulateItems.res +8 -10
  68. package/src/SimulateItems.res.mjs +9 -10
  69. package/src/Sink.res +2 -2
  70. package/src/Sink.res.mjs +3 -2
  71. package/src/TestIndexer.res +11 -14
  72. package/src/TestIndexer.res.mjs +14 -10
  73. package/src/UserContext.res +2 -2
  74. package/src/UserContext.res.mjs +2 -1
  75. package/src/Utils.res +0 -4
  76. package/src/Utils.res.mjs +1 -6
  77. package/src/bindings/ClickHouse.res +20 -2
  78. package/src/bindings/ClickHouse.res.mjs +25 -14
  79. package/src/db/EntityHistory.res +1 -1
  80. package/src/db/IndexCatalog.res +248 -0
  81. package/src/db/IndexCatalog.res.mjs +222 -0
  82. package/src/db/IndexDefinition.res +131 -0
  83. package/src/db/IndexDefinition.res.mjs +138 -0
  84. package/src/db/IndexManager.res +145 -0
  85. package/src/db/IndexManager.res.mjs +130 -0
  86. package/src/db/InternalTable.res +50 -26
  87. package/src/db/InternalTable.res.mjs +32 -24
  88. package/src/db/Table.res +11 -1
  89. package/src/db/Table.res.mjs +9 -4
  90. package/src/sources/AddressSet.res +5 -4
  91. package/src/sources/AddressStore.res +25 -14
  92. package/src/sources/AddressStore.res.mjs +10 -12
  93. package/src/sources/Evm.res +2 -2
  94. package/src/sources/Evm.res.mjs +1 -1
  95. package/src/sources/EvmChain.res +3 -3
  96. package/src/sources/EvmChain.res.mjs +3 -3
  97. package/src/sources/EvmHyperSyncSource.res +7 -7
  98. package/src/sources/EvmHyperSyncSource.res.mjs +6 -6
  99. package/src/sources/Fuel.res +2 -2
  100. package/src/sources/Fuel.res.mjs +1 -1
  101. package/src/sources/FuelHyperSyncClient.res +3 -0
  102. package/src/sources/FuelHyperSyncClient.res.mjs +1 -0
  103. package/src/sources/FuelHyperSyncSource.res +5 -7
  104. package/src/sources/FuelHyperSyncSource.res.mjs +5 -5
  105. package/src/sources/HyperSync.resi +2 -2
  106. package/src/sources/HyperSyncClient.res +5 -0
  107. package/src/sources/HyperSyncClient.res.mjs +1 -0
  108. package/src/sources/RpcSource.res +13 -14
  109. package/src/sources/RpcSource.res.mjs +13 -12
  110. package/src/sources/SimulateSource.res +42 -15
  111. package/src/sources/SimulateSource.res.mjs +27 -9
  112. package/src/sources/Source.res +1 -5
  113. package/src/sources/SourceManager.res +12 -12
  114. package/src/sources/SourceManager.res.mjs +11 -10
  115. package/src/sources/SourceManager.resi +2 -2
  116. package/src/sources/Svm.res +4 -5
  117. package/src/sources/Svm.res.mjs +5 -4
  118. package/src/sources/SvmHyperSyncClient.res +3 -0
  119. package/src/sources/SvmHyperSyncClient.res.mjs +1 -0
  120. package/src/sources/SvmHyperSyncSource.res +4 -4
  121. package/src/sources/SvmHyperSyncSource.res.mjs +3 -3
  122. package/src/tui/Tui.res +1 -1
  123. package/src/tui/Tui.res.mjs +2 -1
  124. package/src/tui/components/CustomHooks.res +5 -4
  125. package/src/tui/components/CustomHooks.res.mjs +4 -2
  126. package/src/db/IndexRegistry.res +0 -219
  127. package/src/db/IndexRegistry.res.mjs +0 -195
@@ -1,78 +1,42 @@
1
1
  // Generated by ReScript, PLEASE EDIT WITH CARE
2
2
 
3
3
  import * as Belt_Id from "@rescript/runtime/lib/es6/Belt_Id.js";
4
+ import * as ChainId from "./ChainId.res.mjs";
4
5
  import * as Belt_Map from "@rescript/runtime/lib/es6/Belt_Map.js";
5
- import * as Primitive_int from "@rescript/runtime/lib/es6/Primitive_int.js";
6
- import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js";
7
6
  import * as Stdlib_JsError from "@rescript/runtime/lib/es6/Stdlib_JsError.js";
8
7
  import * as Primitive_option from "@rescript/runtime/lib/es6/Primitive_option.js";
9
8
 
10
- function toString(chainId) {
11
- return chainId.toString();
12
- }
13
-
14
- function makeUnsafe(chainId) {
15
- return chainId;
16
- }
17
-
18
- let Chain = {
19
- toString: toString,
20
- makeUnsafe: makeUnsafe
21
- };
22
-
23
- function cmp(a, b) {
24
- return Primitive_int.compare(a, b) | 0;
25
- }
26
-
27
9
  let ChainIdCmp = Belt_Id.MakeComparable({
28
- cmp: cmp
10
+ cmp: ChainId.compare
29
11
  });
30
12
 
31
13
  function fromArrayUnsafe(arr) {
32
14
  return Belt_Map.fromArray(arr, ChainIdCmp);
33
15
  }
34
16
 
35
- function get(self, chain) {
36
- let v = Belt_Map.get(self, chain);
17
+ function get(self, chainId) {
18
+ let v = Belt_Map.get(self, chainId);
37
19
  if (v !== undefined) {
38
20
  return Primitive_option.valFromOption(v);
39
21
  } else {
40
- return Stdlib_JsError.throwWithMessage("No chain with id " + chain.toString() + " found in chain map");
22
+ return Stdlib_JsError.throwWithMessage("No chain with id " + ChainId.toString(chainId) + " found in chain map");
41
23
  }
42
24
  }
43
25
 
44
- let set = Belt_Map.set;
45
-
46
26
  let values = Belt_Map.valuesToArray;
47
27
 
48
28
  let keys = Belt_Map.keysToArray;
49
29
 
50
- let entries = Belt_Map.toArray;
51
-
52
30
  let has = Belt_Map.has;
53
31
 
54
- let map = Belt_Map.map;
55
-
56
32
  let mapWithKey = Belt_Map.mapWithKey;
57
33
 
58
- let size = Belt_Map.size;
59
-
60
- function update(map, chain, updateFn) {
61
- return Belt_Map.update(map, chain, opt => Stdlib_Option.map(opt, updateFn));
62
- }
63
-
64
34
  export {
65
- Chain,
66
35
  fromArrayUnsafe,
67
36
  get,
68
- set,
69
37
  values,
70
38
  keys,
71
- entries,
72
39
  has,
73
- map,
74
40
  mapWithKey,
75
- size,
76
- update,
77
41
  }
78
42
  /* ChainIdCmp Not a pure module */
package/src/ChainMap.resi CHANGED
@@ -1,22 +1,7 @@
1
- module Chain: {
2
- type t
3
-
4
- external toChainId: t => int = "%identity"
5
-
6
- let toString: t => string
7
-
8
- let makeUnsafe: (~chainId: int) => t
9
- }
10
-
11
1
  type t<'a>
12
- let fromArrayUnsafe: array<(Chain.t, 'a)> => t<'a>
13
- let get: (t<'a>, Chain.t) => 'a
14
- let set: (t<'a>, Chain.t, 'a) => t<'a>
2
+ let fromArrayUnsafe: array<(ChainId.t, 'a)> => t<'a>
3
+ let get: (t<'a>, ChainId.t) => 'a
15
4
  let values: t<'a> => array<'a>
16
- let keys: t<'a> => array<Chain.t>
17
- let entries: t<'a> => array<(Chain.t, 'a)>
18
- let has: (t<'a>, Chain.t) => bool
19
- let map: (t<'a>, 'a => 'b) => t<'b>
20
- let mapWithKey: (t<'a>, (Chain.t, 'a) => 'b) => t<'b>
21
- let size: t<'a> => int
22
- let update: (t<'a>, Chain.t, 'a => 'a) => t<'a>
5
+ let keys: t<'a> => array<ChainId.t>
6
+ let has: (t<'a>, ChainId.t) => bool
7
+ let mapWithKey: (t<'a>, (ChainId.t, 'a) => 'b) => t<'b>
@@ -8,7 +8,7 @@ let stage = (state: IndexerState.t) => {
8
8
  ->Dict.valuesToArray
9
9
  ->Array.forEach(cs => {
10
10
  chainsData->Dict.set(
11
- (cs->ChainState.chainConfig).id->Int.toString,
11
+ (cs->ChainState.chainConfig).id->ChainId.toString,
12
12
  cs->ChainState.toChainMetadata,
13
13
  )
14
14
  })
@@ -1,5 +1,6 @@
1
1
  // Generated by ReScript, PLEASE EDIT WITH CARE
2
2
 
3
+ import * as ChainId from "./ChainId.res.mjs";
3
4
  import * as Writing from "./Writing.res.mjs";
4
5
  import * as ChainState from "./ChainState.res.mjs";
5
6
  import * as IndexerState from "./IndexerState.res.mjs";
@@ -7,7 +8,7 @@ import * as IndexerState from "./IndexerState.res.mjs";
7
8
  function stage(state) {
8
9
  let chainsData = {};
9
10
  Object.values(IndexerState.chainStates(state)).forEach(cs => {
10
- chainsData[ChainState.chainConfig(cs).id.toString()] = ChainState.toChainMetadata(cs);
11
+ chainsData[ChainId.toString(ChainState.chainConfig(cs).id)] = ChainState.toChainMetadata(cs);
11
12
  });
12
13
  Writing.setChainMeta(state, chainsData);
13
14
  }
@@ -15,4 +16,4 @@ function stage(state) {
15
16
  export {
16
17
  stage,
17
18
  }
18
- /* Writing Not a pure module */
19
+ /* ChainId Not a pure module */
@@ -69,13 +69,13 @@ let configAddresses = (chainConfig: Config.chain): array<Internal.indexingAddres
69
69
  }
70
70
 
71
71
  let validateOnEventRegistrations = (
72
- ~chainId: int,
72
+ ~chainId: ChainId.t,
73
73
  registrations: array<Internal.onEventRegistration>,
74
74
  ) =>
75
75
  registrations->Array.forEachWithIndex((registration, expectedIndex) => {
76
76
  if registration.index !== expectedIndex {
77
77
  JsError.throwWithMessage(
78
- `Invalid onEvent registration index for chain ${chainId->Int.toString}: ${registration.eventConfig.contractName}.${registration.eventConfig.name} has index ${registration.index->Int.toString}, but its ChainState position is ${expectedIndex->Int.toString}.`,
78
+ `Invalid onEvent registration index for chain ${chainId->ChainId.toString}: ${registration.eventConfig.contractName}.${registration.eventConfig.name} has index ${registration.index->Int.toString}, but its ChainState position is ${expectedIndex->Int.toString}.`,
79
79
  )
80
80
  }
81
81
  })
@@ -155,7 +155,7 @@ let makeInternal = (
155
155
  // chain - this just looks up this chain's slice.
156
156
  let {onEventRegistrations, onBlockRegistrations} =
157
157
  registrationsByChainId
158
- ->Utils.Dict.dangerouslyGetNonOption(chainConfig.id->Int.toString)
158
+ ->Utils.Dict.dangerouslyGetNonOption(chainConfig.id->ChainId.toString)
159
159
  ->Option.getOr({onEventRegistrations: [], onBlockRegistrations: []})
160
160
 
161
161
  chainConfig.contracts->Array.forEach(contract => {
@@ -216,7 +216,7 @@ let makeInternal = (
216
216
  })
217
217
 
218
218
  // Create sources lazily here - this is where API token validation happens
219
- let chain = ChainMap.Chain.makeUnsafe(~chainId=chainConfig.id)
219
+ let chainId = chainConfig.id
220
220
  let sources = switch chainConfig.sourceConfig {
221
221
  | Config.EvmSourceConfig({hypersync, rpcs}) =>
222
222
  let evmRpcs: array<EvmChain.rpc> = rpcs->Array.map((rpc): EvmChain.rpc => {
@@ -232,7 +232,7 @@ let makeInternal = (
232
232
  }
233
233
  })
234
234
  EvmChain.makeSources(
235
- ~chain,
235
+ ~chainId,
236
236
  ~onEventRegistrations=onEventRegistrations->(
237
237
  Utils.magic: array<Internal.onEventRegistration> => array<Internal.evmOnEventRegistration>
238
238
  ),
@@ -243,7 +243,7 @@ let makeInternal = (
243
243
  )
244
244
  | Config.FuelSourceConfig({hypersync}) => [
245
245
  FuelHyperSyncSource.make({
246
- chain,
246
+ chainId,
247
247
  endpointUrl: hypersync,
248
248
  apiToken: Env.envioApiToken,
249
249
  onEventRegistrations,
@@ -253,17 +253,15 @@ let makeInternal = (
253
253
  | Config.SvmSourceConfig({hypersync, rpc}) =>
254
254
  switch (hypersync, rpc) {
255
255
  | (None, None) =>
256
- JsError.throwWithMessage(
257
- `Chain ${chain->ChainMap.Chain.toChainId->Int.toString} has no SVM data source`,
258
- )
259
- | (None, Some(rpc)) => [Svm.makeRPCSource(~chain, ~rpc)]
256
+ JsError.throwWithMessage(`Chain ${chainId->ChainId.toString} has no SVM data source`)
257
+ | (None, Some(rpc)) => [Svm.makeRPCSource(~chainId, ~rpc)]
260
258
  | (Some(hypersyncUrl), _) =>
261
259
  // HyperSync drives instruction sync. A configured RPC is ignored for now
262
260
  // (RPC fallback isn't wired up yet).
263
261
  let apiToken = Env.envioApiToken
264
262
  [
265
263
  SvmHyperSyncSource.make({
266
- chain,
264
+ chainId,
267
265
  endpointUrl: hypersyncUrl,
268
266
  apiToken,
269
267
  onEventRegistrations,
@@ -272,6 +270,9 @@ let makeInternal = (
272
270
  }),
273
271
  ]
274
272
  }
273
+ | Config.SimulateSourceConfig({items, endBlock}) => [
274
+ SimulateSource.make(~items, ~endBlock, ~chainId, ~addressStore),
275
+ ]
275
276
  // For tests: use ready-to-use sources directly
276
277
  | Config.CustomSources(sources) => sources
277
278
  }
@@ -377,9 +378,7 @@ let makeFromDbState = (
377
378
  ~maxReorgDepth=resumedChainState.maxReorgDepth,
378
379
  ~firstEventBlock=resumedChainState.firstEventBlockNumber,
379
380
  ~progressBlockNumber,
380
- ~timestampCaughtUpToHeadOrEndblock=Env.updateSyncTimeOnRestart
381
- ? None
382
- : resumedChainState.timestampCaughtUpToHeadOrEndblock,
381
+ ~timestampCaughtUpToHeadOrEndblock=resumedChainState.timestampCaughtUpToHeadOrEndblock,
383
382
  ~numEventsProcessed=resumedChainState.numEventsProcessed,
384
383
  ~logger,
385
384
  ~isInReorgThreshold,
@@ -615,6 +614,24 @@ let hasProcessedToEndblock = (cs: t) => {
615
614
  }
616
615
  }
617
616
 
617
+ // Caught up as judged by persisted values alone: progress reached the endBlock,
618
+ // or the head the previous run had already observed (less the lag that holds the
619
+ // tip back). Unlike `isFetchingAtHead` this doesn't move when a fresh height
620
+ // lands, so a run resumed at the head still knows it was caught up even once the
621
+ // head has run away from it while the indexer was down.
622
+ let isDurablyCaughtUp = (cs: t) => {
623
+ let {committedProgressBlockNumber, fetchState} = cs
624
+ switch fetchState.endBlock {
625
+ | Some(endBlock) => committedProgressBlockNumber >= endBlock
626
+ | None =>
627
+ // The configured lag, not the fetch state's: pre-threshold that one also
628
+ // carries maxReorgDepth, which would read a chain a whole reorg depth behind
629
+ // the head as caught up.
630
+ fetchState.knownHeight > 0 &&
631
+ committedProgressBlockNumber >= fetchState.knownHeight - cs.chainConfig.blockLag
632
+ }
633
+ }
634
+
618
635
  let getHighestBlockBelowThreshold = (cs: t): int => {
619
636
  let highestBlockBelowThreshold = cs.fetchState.knownHeight - cs.chainConfig.maxReorgDepth
620
637
  highestBlockBelowThreshold < 0 ? 0 : highestBlockBelowThreshold
@@ -911,7 +928,7 @@ let toChainMetadata = (cs: t): InternalTable.Chains.metaFields => {
911
928
  }
912
929
 
913
930
  let toMetrics = (cs: t): Metrics.chainMetrics => {
914
- chainId: cs.chainConfig.id->Int.toFloat,
931
+ chainId: cs.chainConfig.id,
915
932
  poweredByHyperSync: (cs.sourceManager->SourceManager.getActiveSource).poweredByHyperSync,
916
933
  firstEventBlockNumber: cs.fetchState.firstEventBlock,
917
934
  latestProcessedBlock: cs.committedProgressBlockNumber === -1
@@ -961,7 +978,7 @@ let toChainBeforeBatch = (cs: t): Batch.chainBeforeBatch => {
961
978
  // Whether the chain's post-batch fetch frontier is ready to cross into the reorg
962
979
  // threshold, using the batch's progressed frontier when this chain advanced.
963
980
  let isReadyToEnterReorgThresholdAfterBatch = (cs: t, ~batch: Batch.t) => {
964
- let fetchState = switch batch.progressedChainsById->Utils.Dict.dangerouslyGetByIntNonOption(
981
+ let fetchState = switch batch.progressedChainsById->ChainId.Dict.dangerouslyGetNonOption(
965
982
  cs.fetchState.chainId,
966
983
  ) {
967
984
  | Some(chainAfterBatch) => chainAfterBatch.fetchState
@@ -973,9 +990,7 @@ let isReadyToEnterReorgThresholdAfterBatch = (cs: t, ~batch: Batch.t) => {
973
990
  // Commit the post-batch fetch frontier for a chain that progressed in the batch,
974
991
  // applying blockLag when this batch also crosses into the reorg threshold.
975
992
  let advanceAfterBatch = (cs: t, ~batch: Batch.t, ~enteringReorgThreshold) =>
976
- switch batch.progressedChainsById->Utils.Dict.dangerouslyGetByIntNonOption(
977
- cs.fetchState.chainId,
978
- ) {
993
+ switch batch.progressedChainsById->ChainId.Dict.dangerouslyGetNonOption(cs.fetchState.chainId) {
979
994
  | Some(chainAfterBatch) =>
980
995
  cs.fetchState = enteringReorgThreshold
981
996
  ? chainAfterBatch.fetchState->FetchState.updateInternal(~blockLag=cs.chainConfig.blockLag)
@@ -995,7 +1010,7 @@ let advanceAfterBatch = (cs: t, ~batch: Batch.t, ~enteringReorgThreshold) =>
995
1010
  let applyBatchProgress = (cs: t, ~batch: Batch.t, ~blockTimestampName: string) => {
996
1011
  let chainId = cs.chainConfig.id
997
1012
 
998
- switch batch.progressedChainsById->Utils.Dict.dangerouslyGetByIntNonOption(chainId) {
1013
+ switch batch.progressedChainsById->ChainId.Dict.dangerouslyGetNonOption(chainId) {
999
1014
  | Some(chainAfterBatch) => {
1000
1015
  // Calculate and set latency metrics. The payload block is materialised or
1001
1016
  // inline by processing time; its timestamp may still be absent (e.g. an
@@ -4,8 +4,8 @@ import * as Env from "./Env.res.mjs";
4
4
  import * as Svm from "./sources/Svm.res.mjs";
5
5
  import * as Batch from "./Batch.res.mjs";
6
6
  import * as Utils from "./Utils.res.mjs";
7
+ import * as ChainId from "./ChainId.res.mjs";
7
8
  import * as Logging from "./Logging.res.mjs";
8
- import * as ChainMap from "./ChainMap.res.mjs";
9
9
  import * as EvmChain from "./sources/EvmChain.res.mjs";
10
10
  import * as FieldMask from "./sources/FieldMask.res.mjs";
11
11
  import * as BlockStore from "./sources/BlockStore.res.mjs";
@@ -17,6 +17,7 @@ import * as Primitive_int from "@rescript/runtime/lib/es6/Primitive_int.js";
17
17
  import * as SourceManager from "./sources/SourceManager.res.mjs";
18
18
  import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js";
19
19
  import * as ReorgDetection from "./ReorgDetection.res.mjs";
20
+ import * as SimulateSource from "./sources/SimulateSource.res.mjs";
20
21
  import * as Stdlib_JsError from "@rescript/runtime/lib/es6/Stdlib_JsError.js";
21
22
  import * as Primitive_float from "@rescript/runtime/lib/es6/Primitive_float.js";
22
23
  import * as Primitive_option from "@rescript/runtime/lib/es6/Primitive_option.js";
@@ -42,7 +43,7 @@ function configAddresses(chainConfig) {
42
43
  function validateOnEventRegistrations(chainId, registrations) {
43
44
  registrations.forEach((registration, expectedIndex) => {
44
45
  if (registration.index !== expectedIndex) {
45
- return Stdlib_JsError.throwWithMessage(`Invalid onEvent registration index for chain ` + chainId.toString() + `: ` + registration.eventConfig.contractName + `.` + registration.eventConfig.name + ` has index ` + registration.index.toString() + `, but its ChainState position is ` + expectedIndex.toString() + `.`);
46
+ return Stdlib_JsError.throwWithMessage(`Invalid onEvent registration index for chain ` + ChainId.toString(chainId) + `: ` + registration.eventConfig.contractName + `.` + registration.eventConfig.name + ` has index ` + registration.index.toString() + `, but its ChainState position is ` + expectedIndex.toString() + `.`);
46
47
  }
47
48
  });
48
49
  }
@@ -93,7 +94,7 @@ function makeInternal(chainConfig, indexingAddresses, startBlock, endBlock, firs
93
94
  let firstEventBlock = firstEventBlockOpt !== undefined ? Primitive_option.valFromOption(firstEventBlockOpt) : undefined;
94
95
  let knownHeight = knownHeightOpt !== undefined ? knownHeightOpt : 0;
95
96
  let isResumed = isResumedOpt !== undefined ? isResumedOpt : false;
96
- let match = Stdlib_Option.getOr(registrationsByChainId[chainConfig.id.toString()], {
97
+ let match = Stdlib_Option.getOr(registrationsByChainId[ChainId.toString(chainConfig.id)], {
97
98
  onEventRegistrations: [],
98
99
  onBlockRegistrations: []
99
100
  });
@@ -123,7 +124,7 @@ function makeInternal(chainConfig, indexingAddresses, startBlock, endBlock, firs
123
124
  return reorgCheckpoint;
124
125
  }
125
126
  });
126
- let chain = ChainMap.Chain.makeUnsafe(chainConfig.id);
127
+ let chainId = chainConfig.id;
127
128
  let sources = chainConfig.sourceConfig;
128
129
  let sources$1;
129
130
  switch (sources.TAG) {
@@ -140,11 +141,11 @@ function makeInternal(chainConfig, indexingAddresses, startBlock, endBlock, firs
140
141
  headers: headers
141
142
  };
142
143
  });
143
- sources$1 = EvmChain.makeSources(chain, onEventRegistrations, sources.hypersync, evmRpcs, lowercaseAddresses, addressStore);
144
+ sources$1 = EvmChain.makeSources(chainId, onEventRegistrations, sources.hypersync, evmRpcs, lowercaseAddresses, addressStore);
144
145
  break;
145
146
  case "FuelSourceConfig" :
146
147
  sources$1 = [FuelHyperSyncSource.make({
147
- chain: chain,
148
+ chainId: chainId,
148
149
  endpointUrl: sources.hypersync,
149
150
  apiToken: Env.envioApiToken,
150
151
  onEventRegistrations: onEventRegistrations,
@@ -155,16 +156,19 @@ function makeInternal(chainConfig, indexingAddresses, startBlock, endBlock, firs
155
156
  let rpc = sources.rpc;
156
157
  let hypersync = sources.hypersync;
157
158
  sources$1 = hypersync !== undefined ? [SvmHyperSyncSource.make({
158
- chain: chain,
159
+ chainId: chainId,
159
160
  endpointUrl: hypersync,
160
161
  apiToken: Env.envioApiToken,
161
162
  onEventRegistrations: onEventRegistrations,
162
163
  clientTimeoutMillis: Env.hyperSyncClientTimeoutMillis,
163
164
  addressStore: addressStore
164
165
  })] : (
165
- rpc !== undefined ? [Svm.makeRPCSource(chain, rpc, undefined)] : Stdlib_JsError.throwWithMessage(`Chain ` + chain.toString() + ` has no SVM data source`)
166
+ rpc !== undefined ? [Svm.makeRPCSource(chainId, rpc, undefined)] : Stdlib_JsError.throwWithMessage(`Chain ` + ChainId.toString(chainId) + ` has no SVM data source`)
166
167
  );
167
168
  break;
169
+ case "SimulateSourceConfig" :
170
+ sources$1 = [SimulateSource.make(sources.items, sources.endBlock, chainId, addressStore)];
171
+ break;
168
172
  case "CustomSources" :
169
173
  sources$1 = sources._0;
170
174
  break;
@@ -187,7 +191,7 @@ function makeFromDbState(chainConfig, resumedChainState, reorgCheckpoints, isInR
187
191
  chainId: chainId
188
192
  });
189
193
  let progressBlockNumber = resumedChainState.progressBlockNumber >= 0 ? resumedChainState.progressBlockNumber : resumedChainState.startBlock - 1 | 0;
190
- return makeInternal(chainConfig, resumedChainState.indexingAddresses, resumedChainState.startBlock, resumedChainState.endBlock, Primitive_option.some(resumedChainState.firstEventBlockNumber), progressBlockNumber, config, registrationsByChainId, logger, Env.updateSyncTimeOnRestart ? undefined : resumedChainState.timestampCaughtUpToHeadOrEndblock, resumedChainState.numEventsProcessed, isInReorgThreshold, isRealtime, reorgCheckpoints, resumedChainState.maxReorgDepth, resumedChainState.sourceBlockNumber, true, reducedPollingInterval);
194
+ return makeInternal(chainConfig, resumedChainState.indexingAddresses, resumedChainState.startBlock, resumedChainState.endBlock, Primitive_option.some(resumedChainState.firstEventBlockNumber), progressBlockNumber, config, registrationsByChainId, logger, resumedChainState.timestampCaughtUpToHeadOrEndblock, resumedChainState.numEventsProcessed, isInReorgThreshold, isRealtime, reorgCheckpoints, resumedChainState.maxReorgDepth, resumedChainState.sourceBlockNumber, true, reducedPollingInterval);
191
195
  }
192
196
 
193
197
  function logger(cs) {
@@ -284,6 +288,11 @@ function startFetchingQueries(cs, queries) {
284
288
  cs.pendingBudget = cs.pendingBudget + Stdlib_Array.reduce(queries, 0, (acc, query) => acc + query.itemsEst);
285
289
  }
286
290
 
291
+ function resetPendingQueries(cs) {
292
+ cs.fetchState = FetchState.resetPendingQueries(cs.fetchState);
293
+ cs.pendingBudget = 0;
294
+ }
295
+
287
296
  function isReady(cs) {
288
297
  return cs.timestampCaughtUpToHeadOrEndblock !== undefined;
289
298
  }
@@ -390,6 +399,19 @@ function hasProcessedToEndblock(cs) {
390
399
  }
391
400
  }
392
401
 
402
+ function isDurablyCaughtUp(cs) {
403
+ let fetchState = cs.fetchState;
404
+ let committedProgressBlockNumber = cs.committedProgressBlockNumber;
405
+ let endBlock = fetchState.endBlock;
406
+ if (endBlock !== undefined) {
407
+ return committedProgressBlockNumber >= endBlock;
408
+ } else if (fetchState.knownHeight > 0) {
409
+ return committedProgressBlockNumber >= (fetchState.knownHeight - cs.chainConfig.blockLag | 0);
410
+ } else {
411
+ return false;
412
+ }
413
+ }
414
+
393
415
  function getHighestBlockBelowThreshold(cs) {
394
416
  let highestBlockBelowThreshold = cs.fetchState.knownHeight - cs.chainConfig.maxReorgDepth | 0;
395
417
  if (highestBlockBelowThreshold < 0) {
@@ -565,8 +587,7 @@ function prepareReorg(cs, eventsProcessedDiff) {
565
587
  if (eventsProcessedDiff !== undefined) {
566
588
  cs.numEventsProcessed = cs.numEventsProcessed + eventsProcessedDiff;
567
589
  }
568
- cs.fetchState = FetchState.resetPendingQueries(cs.fetchState);
569
- cs.pendingBudget = 0;
590
+ resetPendingQueries(cs);
570
591
  }
571
592
 
572
593
  function updateKnownHeight(cs, knownHeight) {
@@ -832,6 +853,7 @@ export {
832
853
  toChainBeforeBatch,
833
854
  isReadyToEnterReorgThresholdAfterBatch,
834
855
  hasProcessedToEndblock,
856
+ isDurablyCaughtUp,
835
857
  getHighestBlockBelowThreshold,
836
858
  isActivelyIndexing,
837
859
  isReady,
@@ -842,6 +864,7 @@ export {
842
864
  materializePageItems,
843
865
  registerReorgGuard,
844
866
  prepareReorg,
867
+ resetPendingQueries,
845
868
  updateKnownHeight,
846
869
  setEndBlockToFirstEvent,
847
870
  enterReorgThreshold,
@@ -98,6 +98,7 @@ let isReadyToEnterReorgThresholdAfterBatch: (t, ~batch: Batch.t) => bool
98
98
 
99
99
  // Derived (pure).
100
100
  let hasProcessedToEndblock: t => bool
101
+ let isDurablyCaughtUp: t => bool
101
102
  let getHighestBlockBelowThreshold: t => int
102
103
  let isActivelyIndexing: t => bool
103
104
  let isReady: t => bool
@@ -132,6 +133,7 @@ let registerReorgGuard: (
132
133
  ~knownHeight: int,
133
134
  ) => ReorgDetection.reorgResult
134
135
  let prepareReorg: (t, ~eventsProcessedDiff: option<float>) => unit
136
+ let resetPendingQueries: t => unit
135
137
  let updateKnownHeight: (t, ~knownHeight: int) => unit
136
138
  let setEndBlockToFirstEvent: (t, ~blockNumber: int) => unit
137
139
  let enterReorgThreshold: t => unit
package/src/Config.res CHANGED
@@ -30,12 +30,16 @@ type sourceConfig =
30
30
  | EvmSourceConfig({hypersync: option<string>, rpcs: array<evmRpcConfig>})
31
31
  | FuelSourceConfig({hypersync: string})
32
32
  | SvmSourceConfig({hypersync: option<string>, rpc: option<string>})
33
+ // A `simulate` run: the items the test fed in, parsed against the chain's
34
+ // registrations. The source itself is built with the chain's address store,
35
+ // like every other source, so it can apply the same gates.
36
+ | SimulateSourceConfig({items: array<Internal.item>, endBlock: int})
33
37
  // For tests: pass custom sources directly
34
38
  | CustomSources(array<Source.t>)
35
39
 
36
40
  type chain = {
37
41
  name: string,
38
- id: int,
42
+ id: ChainId.t,
39
43
  startBlock: int,
40
44
  endBlock?: int,
41
45
  maxReorgDepth: int,
@@ -73,6 +77,10 @@ type t = {
73
77
  shouldRollbackOnReorg: bool,
74
78
  shouldSaveFullHistory: bool,
75
79
  storage: storage,
80
+ // Widest scalar the internal chain-id columns need, resolved by the CLI from
81
+ // the maximum active chain id. Older configs predate the field, and every id
82
+ // they can express fits an INTEGER.
83
+ chainIdMode: ChainId.mode,
76
84
  chainMap: ChainMap.t<chain>,
77
85
  defaultChain: option<chain>,
78
86
  ecosystem: Ecosystem.t,
@@ -98,13 +106,13 @@ module EnvioAddresses = {
98
106
  let name = "envio_addresses"
99
107
  let index = -1
100
108
 
101
- let makeId = (~chainId, ~address) => {
102
- chainId->Int.toString ++ "-" ++ address->Address.toString
109
+ let makeId = (~chainId: ChainId.t, ~address) => {
110
+ chainId->ChainId.toString ++ "-" ++ address->Address.toString
103
111
  }
104
112
 
105
113
  type t = {
106
114
  id: string,
107
- @as("chain_id") chainId: int,
115
+ @as("chain_id") chainId: ChainId.t,
108
116
  @as("registration_block") registrationBlock: int,
109
117
  // -1 when the address was registered from a block handler (no log index)
110
118
  @as("registration_log_index") registrationLogIndex: int,
@@ -123,7 +131,7 @@ module EnvioAddresses = {
123
131
 
124
132
  let schema = S.schema(s => {
125
133
  id: s.matches(S.string),
126
- chainId: s.matches(S.int),
134
+ chainId: s.matches(ChainId.schema),
127
135
  registrationBlock: s.matches(S.int),
128
136
  registrationLogIndex: s.matches(S.int),
129
137
  contractName: s.matches(S.string),
@@ -133,7 +141,7 @@ module EnvioAddresses = {
133
141
  name,
134
142
  ~fields=[
135
143
  Table.mkField("id", String, ~isPrimaryKey=true, ~fieldSchema=S.string),
136
- Table.mkField("chain_id", Int32, ~fieldSchema=S.int),
144
+ Table.mkField("chain_id", ChainId, ~fieldSchema=ChainId.schema),
137
145
  Table.mkField("registration_block", Int32, ~fieldSchema=S.int),
138
146
  // -1 sentinel when registered from a block handler (no log index)
139
147
  Table.mkField("registration_log_index", Int32, ~fieldSchema=S.int),
@@ -185,7 +193,7 @@ let chainContractSchema = S.schema(s =>
185
193
 
186
194
  let publicConfigChainSchema = S.schema(s =>
187
195
  {
188
- "id": s.matches(S.int),
196
+ "id": s.matches(ChainId.schema),
189
197
  "startBlock": s.matches(S.int),
190
198
  "endBlock": s.matches(S.option(S.int)),
191
199
  "maxReorgDepth": s.matches(S.option(S.int)),
@@ -548,6 +556,7 @@ let publicConfigSchema = S.schema(s =>
548
556
  "rollbackOnReorg": s.matches(S.option(S.bool)),
549
557
  "saveFullHistory": s.matches(S.option(S.bool)),
550
558
  "rawEvents": s.matches(S.option(S.bool)),
559
+ "chainIdMode": s.matches(S.option(ChainId.modeSchema)),
551
560
  "storage": s.matches(publicConfigStorageSchema),
552
561
  "evm": s.matches(S.option(publicConfigEvmSchema)),
553
562
  "fuel": s.matches(S.option(publicConfigEcosystemSchema)),
@@ -690,7 +699,7 @@ let fromPublic = (publicConfigJson: JSON.t) => {
690
699
  ~contractName,
691
700
  ~events: option<array<_>>,
692
701
  ~abi,
693
- ~chainId: int,
702
+ ~chainId: ChainId.t,
694
703
  ~addresses: array<string>,
695
704
  ~svmDefinedTypes: JSON.t=JSON.Null,
696
705
  ) => {
@@ -724,11 +733,11 @@ let fromPublic = (publicConfigJson: JSON.t) => {
724
733
  | [pid] => pid->SvmTypes.Pubkey.fromStringUnsafe
725
734
  | [] =>
726
735
  JsError.throwWithMessage(
727
- `SVM program ${contractName} on chain ${chainId->Int.toString} is missing a program_id`,
736
+ `SVM program ${contractName} on chain ${chainId->ChainId.toString} is missing a program_id`,
728
737
  )
729
738
  | _ =>
730
739
  JsError.throwWithMessage(
731
- `SVM program ${contractName} on chain ${chainId->Int.toString} has multiple addresses; a program is uniquely identified by a single program_id`,
740
+ `SVM program ${contractName} on chain ${chainId->ChainId.toString} has multiple addresses; a program is uniquely identified by a single program_id`,
732
741
  )
733
742
  }
734
743
  let widenedEventItem =
@@ -878,8 +887,8 @@ let fromPublic = (publicConfigJson: JSON.t) => {
878
887
  | Some(existingContractName) =>
879
888
  JsError.throwWithMessage(
880
889
  existingContractName === contract.name
881
- ? `Address ${addressString} is listed multiple times for the contract ${contract.name} on chain ${chainId->Int.toString}. Please remove the duplicate from your config.`
882
- : `Address ${addressString} on chain ${chainId->Int.toString} is configured for multiple contracts: ${existingContractName} and ${contract.name}. Indexing the same address with multiple contract definitions is not supported. Please define the events on a single contract definition instead.`,
890
+ ? `Address ${addressString} is listed multiple times for the contract ${contract.name} on chain ${chainId->ChainId.toString}. Please remove the duplicate from your config.`
891
+ : `Address ${addressString} on chain ${chainId->ChainId.toString} is configured for multiple contracts: ${existingContractName} and ${contract.name}. Indexing the same address with multiple contract definitions is not supported. Please define the events on a single contract definition instead.`,
883
892
  )
884
893
  | None => contractNameByAddress->Dict.set(addressString, contract.name)
885
894
  }
@@ -969,7 +978,7 @@ let fromPublic = (publicConfigJson: JSON.t) => {
969
978
  let chainMap =
970
979
  chains
971
980
  ->Array.map(chain => {
972
- (ChainMap.Chain.makeUnsafe(~chainId=chain.id), chain)
981
+ (chain.id, chain)
973
982
  })
974
983
  ->ChainMap.fromArrayUnsafe
975
984
 
@@ -1027,6 +1036,7 @@ let fromPublic = (publicConfigJson: JSON.t) => {
1027
1036
  shouldRollbackOnReorg: publicConfig["rollbackOnReorg"]->Option.getOr(true),
1028
1037
  shouldSaveFullHistory: publicConfig["saveFullHistory"]->Option.getOr(false),
1029
1038
  storage: globalStorage,
1039
+ chainIdMode: publicConfig["chainIdMode"]->Option.getOr(Int32),
1030
1040
  chainMap,
1031
1041
  defaultChain: chains->Array.get(0),
1032
1042
  enableRawEvents: publicConfig["rawEvents"]->Option.getOr(false),
@@ -1092,15 +1102,14 @@ let normalizeSimulateAddress = (config: t, address: Address.t): Address.t =>
1092
1102
  // returns that chain's per-chain event config (matters for where-callback
1093
1103
  // probe detection, which runs with the chain's real id). Without `chainId`,
1094
1104
  // falls back to the first chain that declares this event.
1095
- let getEventConfig = (config: t, ~contractName, ~eventName, ~chainId: option<int>=?) => {
1105
+ let getEventConfig = (config: t, ~contractName, ~eventName, ~chainId: option<ChainId.t>=?) => {
1096
1106
  let chains = switch chainId {
1097
1107
  | Some(chainId) =>
1098
- let chain = ChainMap.Chain.makeUnsafe(~chainId)
1099
- switch config.chainMap->ChainMap.get(chain) {
1108
+ switch config.chainMap->ChainMap.get(chainId) {
1100
1109
  | chainConfig => [chainConfig]
1101
1110
  | exception _ =>
1102
1111
  JsError.throwWithMessage(
1103
- `Chain ${chainId->Int.toString} is not configured. Add it to config.yaml or pass a configured chain.`,
1112
+ `Chain ${chainId->ChainId.toString} is not configured. Add it to config.yaml or pass a configured chain.`,
1104
1113
  )
1105
1114
  }
1106
1115
  | None => config.chainMap->ChainMap.values
@@ -1122,14 +1131,12 @@ let shouldSaveHistory = (config, ~isInReorgThreshold) =>
1122
1131
  let shouldPruneHistory = (config, ~isInReorgThreshold) =>
1123
1132
  !config.shouldSaveFullHistory && (config.shouldRollbackOnReorg && isInReorgThreshold)
1124
1133
 
1125
- let getChain = (config, ~chainId) => {
1126
- let chain = ChainMap.Chain.makeUnsafe(~chainId)
1127
- config.chainMap->ChainMap.has(chain)
1128
- ? chain
1134
+ let getChain = (config, ~chainId) =>
1135
+ config.chainMap->ChainMap.has(chainId)
1136
+ ? chainId
1129
1137
  : JsError.throwWithMessage(
1130
- "No chain with id " ++ chain->ChainMap.Chain.toString ++ " found in config.yaml",
1138
+ "No chain with id " ++ chainId->ChainId.toString ++ " found in config.yaml",
1131
1139
  )
1132
- }
1133
1140
 
1134
1141
  // A CLI command payload already contains the resolved JSON; priming lets
1135
1142
  // downstream callers skip the NAPI `getConfigJson` round-trip. Calling
@@ -1267,7 +1274,17 @@ let diffPaths = (~stored: JSON.t, ~current: JSON.t): array<string> => {
1267
1274
 
1268
1275
  switch (stored, current) {
1269
1276
  | (Object(sObj), Object(cObj)) =>
1270
- let tiers = [["version"], ["name"], ["storage"], ["evm", "fuel", "svm"], ["entities"]]
1277
+ // chainIdMode sits right after version: it decides the physical type of
1278
+ // every chain-id column, so a change to it is reported on its own rather
1279
+ // than buried under the chain diffs that always accompany it.
1280
+ let tiers = [
1281
+ ["version"],
1282
+ ["chainIdMode"],
1283
+ ["name"],
1284
+ ["storage"],
1285
+ ["evm", "fuel", "svm"],
1286
+ ["entities"],
1287
+ ]
1271
1288
  let firstHit = tiers->Array.reduce(None, (acc, tier) =>
1272
1289
  switch acc {
1273
1290
  | Some(_) => acc