envio 3.5.0-alpha.3 → 3.5.0-rc.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (127) hide show
  1. package/package.json +6 -6
  2. package/src/Batch.res +8 -8
  3. package/src/Batch.res.mjs +4 -3
  4. package/src/BatchProcessing.res +17 -7
  5. package/src/BatchProcessing.res.mjs +8 -4
  6. package/src/ChainFetching.res +20 -19
  7. package/src/ChainFetching.res.mjs +14 -14
  8. package/src/ChainId.res +58 -0
  9. package/src/ChainId.res.mjs +49 -0
  10. package/src/ChainId.resi +27 -0
  11. package/src/ChainMap.res +12 -26
  12. package/src/ChainMap.res.mjs +5 -41
  13. package/src/ChainMap.resi +5 -20
  14. package/src/ChainMetadata.res +1 -1
  15. package/src/ChainMetadata.res.mjs +3 -2
  16. package/src/ChainState.res +35 -20
  17. package/src/ChainState.res.mjs +34 -11
  18. package/src/ChainState.resi +2 -0
  19. package/src/Config.res +41 -24
  20. package/src/Config.res.mjs +17 -14
  21. package/src/ContractRegisterContext.res +1 -1
  22. package/src/ContractRegisterContext.res.mjs +1 -1
  23. package/src/Core.res +1 -0
  24. package/src/CrossChainState.res +61 -16
  25. package/src/CrossChainState.res.mjs +37 -13
  26. package/src/CrossChainState.resi +3 -2
  27. package/src/Env.res +0 -3
  28. package/src/Env.res.mjs +0 -3
  29. package/src/EventConfigBuilder.res +1 -1
  30. package/src/EventProcessing.res +4 -4
  31. package/src/EventProcessing.res.mjs +5 -5
  32. package/src/FetchState.res +261 -89
  33. package/src/FetchState.res.mjs +187 -63
  34. package/src/FinalizeBackfill.res +33 -6
  35. package/src/FinalizeBackfill.res.mjs +19 -1
  36. package/src/HandlerRegister.res +8 -8
  37. package/src/HandlerRegister.res.mjs +5 -4
  38. package/src/HandlerRegister.resi +1 -1
  39. package/src/InMemoryStore.res +4 -0
  40. package/src/InMemoryStore.res.mjs +1 -1
  41. package/src/IndexerLoop.res +9 -2
  42. package/src/IndexerLoop.res.mjs +8 -1
  43. package/src/IndexerState.res +41 -24
  44. package/src/IndexerState.res.mjs +36 -17
  45. package/src/IndexerState.resi +9 -7
  46. package/src/Internal.res +19 -17
  47. package/src/Internal.res.mjs +18 -9
  48. package/src/LoadLayer.res +1 -1
  49. package/src/LoadLayer.res.mjs +2 -1
  50. package/src/LogSelection.res +2 -2
  51. package/src/Main.res +7 -11
  52. package/src/Main.res.mjs +4 -6
  53. package/src/Metrics.res +6 -6
  54. package/src/Metrics.res.mjs +4 -3
  55. package/src/Persistence.res +15 -7
  56. package/src/PgStorage.res +284 -258
  57. package/src/PgStorage.res.mjs +181 -208
  58. package/src/RawEvent.res +1 -2
  59. package/src/RawEvent.res.mjs +1 -1
  60. package/src/Rollback.res +18 -20
  61. package/src/Rollback.res.mjs +9 -9
  62. package/src/RollbackCommit.res +2 -2
  63. package/src/RollbackCommit.res.mjs +3 -3
  64. package/src/SafeCheckpointTracking.res +2 -2
  65. package/src/SimulateDeadInputTracker.res +17 -21
  66. package/src/SimulateDeadInputTracker.res.mjs +9 -8
  67. package/src/SimulateItems.res +8 -10
  68. package/src/SimulateItems.res.mjs +9 -10
  69. package/src/Sink.res +2 -2
  70. package/src/Sink.res.mjs +3 -2
  71. package/src/TestIndexer.res +11 -14
  72. package/src/TestIndexer.res.mjs +14 -10
  73. package/src/UserContext.res +2 -2
  74. package/src/UserContext.res.mjs +2 -1
  75. package/src/Utils.res +0 -4
  76. package/src/Utils.res.mjs +1 -6
  77. package/src/bindings/ClickHouse.res +20 -2
  78. package/src/bindings/ClickHouse.res.mjs +25 -14
  79. package/src/db/EntityHistory.res +1 -1
  80. package/src/db/IndexCatalog.res +248 -0
  81. package/src/db/IndexCatalog.res.mjs +222 -0
  82. package/src/db/IndexDefinition.res +131 -0
  83. package/src/db/IndexDefinition.res.mjs +138 -0
  84. package/src/db/IndexManager.res +145 -0
  85. package/src/db/IndexManager.res.mjs +130 -0
  86. package/src/db/InternalTable.res +50 -26
  87. package/src/db/InternalTable.res.mjs +32 -24
  88. package/src/db/Table.res +11 -1
  89. package/src/db/Table.res.mjs +9 -4
  90. package/src/sources/AddressSet.res +5 -4
  91. package/src/sources/AddressStore.res +25 -14
  92. package/src/sources/AddressStore.res.mjs +10 -12
  93. package/src/sources/Evm.res +2 -2
  94. package/src/sources/Evm.res.mjs +1 -1
  95. package/src/sources/EvmChain.res +3 -3
  96. package/src/sources/EvmChain.res.mjs +3 -3
  97. package/src/sources/EvmHyperSyncSource.res +7 -7
  98. package/src/sources/EvmHyperSyncSource.res.mjs +6 -6
  99. package/src/sources/Fuel.res +2 -2
  100. package/src/sources/Fuel.res.mjs +1 -1
  101. package/src/sources/FuelHyperSyncClient.res +3 -0
  102. package/src/sources/FuelHyperSyncClient.res.mjs +1 -0
  103. package/src/sources/FuelHyperSyncSource.res +5 -7
  104. package/src/sources/FuelHyperSyncSource.res.mjs +5 -5
  105. package/src/sources/HyperSync.resi +2 -2
  106. package/src/sources/HyperSyncClient.res +5 -0
  107. package/src/sources/HyperSyncClient.res.mjs +1 -0
  108. package/src/sources/RpcSource.res +13 -14
  109. package/src/sources/RpcSource.res.mjs +13 -12
  110. package/src/sources/SimulateSource.res +42 -15
  111. package/src/sources/SimulateSource.res.mjs +27 -9
  112. package/src/sources/Source.res +1 -5
  113. package/src/sources/SourceManager.res +12 -12
  114. package/src/sources/SourceManager.res.mjs +11 -10
  115. package/src/sources/SourceManager.resi +2 -2
  116. package/src/sources/Svm.res +4 -5
  117. package/src/sources/Svm.res.mjs +5 -4
  118. package/src/sources/SvmHyperSyncClient.res +3 -0
  119. package/src/sources/SvmHyperSyncClient.res.mjs +1 -0
  120. package/src/sources/SvmHyperSyncSource.res +4 -4
  121. package/src/sources/SvmHyperSyncSource.res.mjs +3 -3
  122. package/src/tui/Tui.res +1 -1
  123. package/src/tui/Tui.res.mjs +2 -1
  124. package/src/tui/components/CustomHooks.res +5 -4
  125. package/src/tui/components/CustomHooks.res.mjs +4 -2
  126. package/src/db/IndexRegistry.res +0 -219
  127. package/src/db/IndexRegistry.res.mjs +0 -195
@@ -19,10 +19,10 @@ let start = (state: IndexerState.t) => {
19
19
  launch(state, () =>
20
20
  state
21
21
  ->IndexerState.crossChainState
22
- ->CrossChainState.checkAndFetch(~dispatchChain=(~chain, ~action) =>
22
+ ->CrossChainState.checkAndFetch(~dispatchChain=(~chainId, ~action) =>
23
23
  ChainFetching.fetchChain(
24
24
  state,
25
- chain,
25
+ chainId,
26
26
  ~action,
27
27
  ~stateId=state->IndexerState.epoch,
28
28
  ~scheduleFetch,
@@ -38,6 +38,13 @@ let start = (state: IndexerState.t) => {
38
38
  Rollback.rollback(state, ~scheduleFetch, ~scheduleProcessing, ~scheduleRollback)
39
39
  )
40
40
 
41
+ // Resuming already ready means the FinalizingIndexes phase is behind us and
42
+ // won't run again, so this is the only pass that can restore an index the
43
+ // database lost while the indexer was down.
44
+ if state->IndexerState.isRealtime {
45
+ launch(state, () => FinalizeBackfill.repairSchemaIndexes(state))
46
+ }
47
+
41
48
  scheduleFetch()
42
49
  scheduleProcessing()
43
50
  }
@@ -5,6 +5,7 @@ import * as IndexerState from "./IndexerState.res.mjs";
5
5
  import * as ChainFetching from "./ChainFetching.res.mjs";
6
6
  import * as BatchProcessing from "./BatchProcessing.res.mjs";
7
7
  import * as CrossChainState from "./CrossChainState.res.mjs";
8
+ import * as FinalizeBackfill from "./FinalizeBackfill.res.mjs";
8
9
 
9
10
  function launch(state, work) {
10
11
  if (!IndexerState.isStopped(state)) {
@@ -15,7 +16,7 @@ function launch(state, work) {
15
16
 
16
17
  function start(state) {
17
18
  let scheduleFetch = () => {
18
- let work = () => CrossChainState.checkAndFetch(IndexerState.crossChainState(state), (chain, action) => ChainFetching.fetchChain(state, chain, action, IndexerState.epoch(state), scheduleFetch, scheduleProcessing, scheduleRollback));
19
+ let work = () => CrossChainState.checkAndFetch(IndexerState.crossChainState(state), (chainId, action) => ChainFetching.fetchChain(state, chainId, action, IndexerState.epoch(state), scheduleFetch, scheduleProcessing, scheduleRollback));
19
20
  if (!IndexerState.isStopped(state)) {
20
21
  work();
21
22
  return;
@@ -35,6 +36,12 @@ function start(state) {
35
36
  return;
36
37
  }
37
38
  };
39
+ if (IndexerState.isRealtime(state)) {
40
+ let work = () => FinalizeBackfill.repairSchemaIndexes(state);
41
+ if (!IndexerState.isStopped(state)) {
42
+ work();
43
+ }
44
+ }
38
45
  scheduleFetch();
39
46
  scheduleProcessing();
40
47
  }
@@ -1,9 +1,8 @@
1
- type chain = ChainMap.Chain.t
2
1
  type rollbackState =
3
2
  | NoRollback
4
- | ReorgDetected({chain: chain, blockNumber: int})
3
+ | ReorgDetected({chainId: ChainId.t, blockNumber: int})
5
4
  | FindingReorgDepth
6
- | FoundReorgDepth({chain: chain, rollbackTargetBlockNumber: int})
5
+ | FoundReorgDepth({chainId: ChainId.t, rollbackTargetBlockNumber: int})
7
6
  | RollbackReady({eventsProcessedDiffByChain: dict<float>})
8
7
 
9
8
  module EntityTables = {
@@ -90,6 +89,10 @@ type t = {
90
89
  mutable processedBatchesCount: int,
91
90
  // The single in-flight write loop, None when idle.
92
91
  mutable writeFiber: option<promise<unit>>,
92
+ // The single in-flight finalization, None when none is running. Every path
93
+ // that reaches the FinalizingIndexes phase awaits this one instead of
94
+ // starting a second pass over the same indexes.
95
+ mutable finalizeFiber: option<promise<unit>>,
93
96
  // Set once a write throws, to stop the loop. The error itself goes to onError.
94
97
  mutable hasFailedWrite: bool,
95
98
  // Resolved after every commit so capacity/flush waiters can re-evaluate.
@@ -191,6 +194,7 @@ let make = (
191
194
  processedBatches: [],
192
195
  processedBatchesCount: 0,
193
196
  writeFiber: None,
197
+ finalizeFiber: None,
194
198
  hasFailedWrite: false,
195
199
  commitWaiters: [],
196
200
  chainMeta: Dict.make(),
@@ -256,27 +260,26 @@ let makeFromDbState = (
256
260
  false
257
261
  } else {
258
262
  // Check if any chain is in reorg threshold by comparing progress with sourceBlock - maxReorgDepth.
259
- initialState.chains->Array.some(chain =>
263
+ initialState.chains->Array.some(resumedChainState =>
260
264
  isProgressInReorgThreshold(
261
- ~progressBlockNumber=chain.progressBlockNumber,
262
- ~sourceBlockNumber=chain.sourceBlockNumber,
263
- ~maxReorgDepth=chain.maxReorgDepth,
265
+ ~progressBlockNumber=resumedChainState.progressBlockNumber,
266
+ ~sourceBlockNumber=resumedChainState.sourceBlockNumber,
267
+ ~maxReorgDepth=resumedChainState.maxReorgDepth,
264
268
  )
265
269
  )
266
270
  }
267
271
 
268
- // updateSyncTimeOnRestart wipes the saved timestamp so a restart re-enters
269
- // backfill mode for all chains.
272
+ // `ready_at` is durable: a chain that once caught up resumes realtime, and the
273
+ // deferred indexes committed alongside that stamp are not owed again.
270
274
  let isRealtime =
271
- !Env.updateSyncTimeOnRestart &&
272
275
  initialState.chains->Array.length > 0 &&
273
- initialState.chains->Array.every(c => c.timestampCaughtUpToHeadOrEndblock->Option.isSome)
276
+ initialState.chains->Array.every(c => c.timestampCaughtUpToHeadOrEndblock->Option.isSome)
274
277
 
275
278
  let chainStates = Dict.make()
276
279
  initialState.chains->Array.forEach((resumedChainState: Persistence.initialChainState) => {
277
- let chain = Config.getChain(config, ~chainId=resumedChainState.id)
278
- let chainConfig = config.chainMap->ChainMap.get(chain)
279
- chainStates->Utils.Dict.setByInt(
280
+ let chainId = Config.getChain(config, ~chainId=resumedChainState.id)
281
+ let chainConfig = config.chainMap->ChainMap.get(chainId)
282
+ chainStates->ChainId.Dict.set(
280
283
  resumedChainState.id,
281
284
  chainConfig->ChainState.makeFromDbState(
282
285
  ~resumedChainState,
@@ -304,6 +307,7 @@ let makeFromDbState = (
304
307
  ~onError,
305
308
  ~onExit?,
306
309
  )
310
+ state.crossChainState->CrossChainState.markCaughtUpOnResume
307
311
  initialState.cache->Utils.Dict.forEach(({effectName, count, scope}) => {
308
312
  state.effectState->EffectState.setUnregisteredCacheCount(~effectName, ~scope, ~count)
309
313
  })
@@ -358,15 +362,15 @@ let stop = (state: t) => {
358
362
  state.isStopped = true
359
363
  }
360
364
 
361
- let getChainState = (state: t, ~chain: chain): ChainState.t =>
365
+ let getChainState = (state: t, ~chainId: ChainId.t): ChainState.t =>
362
366
  switch state.crossChainState
363
367
  ->CrossChainState.chainStates
364
- ->Utils.Dict.dangerouslyGetByIntNonOption(chain->ChainMap.Chain.toChainId) {
368
+ ->ChainId.Dict.dangerouslyGetNonOption(chainId) {
365
369
  | Some(cs) => cs
366
370
  | None =>
367
- // Should be unreachable, since we validate on Chain.t creation
371
+ // Should be unreachable: every configured chain gets a state at startup
368
372
  JsError.throwWithMessage(
369
- "No chain with id " ++ chain->ChainMap.Chain.toString ++ " found in chain states",
373
+ "No chain with id " ++ chainId->ChainId.toString ++ " found in chain states",
370
374
  )
371
375
  }
372
376
 
@@ -390,18 +394,18 @@ let enterReorgThreshold = (state: t) => state.crossChainState->CrossChainState.e
390
394
  // ReorgDetected state as one step, so the epoch bump can never be left out. The
391
395
  // caller has already mutated the chain states (restored counters, reset pending
392
396
  // queries). isResolvingReorg derives from rollbackState.
393
- let beginReorg = (state: t, ~chain, ~blockNumber) => {
397
+ let beginReorg = (state: t, ~chainId, ~blockNumber) => {
394
398
  // Settle here, or the rollback that follows would be folded into the stall on
395
399
  // the next beginProcessing — time envio_rollback_seconds already counts.
396
400
  state->settleStalledOnFetch
397
401
  state.epoch = state.epoch + 1
398
- state.rollbackState = ReorgDetected({chain, blockNumber})
402
+ state.rollbackState = ReorgDetected({chainId, blockNumber})
399
403
  }
400
404
 
401
405
  let enterFindingReorgDepth = (state: t) => state.rollbackState = FindingReorgDepth
402
406
 
403
- let foundReorgDepth = (state: t, ~chain, ~rollbackTargetBlockNumber) =>
404
- state.rollbackState = FoundReorgDepth({chain, rollbackTargetBlockNumber})
407
+ let foundReorgDepth = (state: t, ~chainId, ~rollbackTargetBlockNumber) =>
408
+ state.rollbackState = FoundReorgDepth({chainId, rollbackTargetBlockNumber})
405
409
 
406
410
  // Finish a rollback. The caller has already rolled the chain states back in
407
411
  // place; this leaves the diff ready for the next batch to consume.
@@ -414,7 +418,15 @@ let clearRollback = (state: t) => state.rollbackState = NoRollback
414
418
 
415
419
  // Invalidate in-flight fetches/waiters without starting a rollback, eg on the
416
420
  // realtime transition where the parked waiter is bound to the pre-realtime source.
417
- let invalidateInflight = (state: t) => state.epoch = state.epoch + 1
421
+ // Drops the pending queries with it, the way the reorg path does: a response
422
+ // carrying the old epoch is discarded before handleQueryResult can retire its
423
+ // query, and a partition that still holds one never asks for another range.
424
+ let invalidateInflight = (state: t) => {
425
+ state.epoch = state.epoch + 1
426
+ state.crossChainState
427
+ ->CrossChainState.chainStates
428
+ ->Utils.Dict.forEach(ChainState.resetPendingQueries)
429
+ }
418
430
 
419
431
  let applyBatchProgress = (state: t, ~batch: Batch.t) =>
420
432
  state.crossChainState->CrossChainState.applyBatchProgress(
@@ -461,6 +473,7 @@ let processedCheckpointId = (state: t) => state.processedCheckpointId
461
473
  let processedBatches = (state: t) => state.processedBatches
462
474
  let processedBatchesCount = (state: t) => state.processedBatchesCount
463
475
  let writeFiber = (state: t) => state.writeFiber
476
+ let finalizeFiber = (state: t) => state.finalizeFiber
464
477
  let hasFailedWrite = (state: t) => state.hasFailedWrite
465
478
  let chainMetaDirty = (state: t) => state.chainMetaDirty
466
479
  let chainMetaThrottler = (state: t) => state.chainMetaThrottler
@@ -476,7 +489,8 @@ let isFinalizingIndexes = (state: t) =>
476
489
  state.crossChainState->CrossChainState.isCaughtUp &&
477
490
  !(state.crossChainState->CrossChainState.isRealtime)
478
491
 
479
- let markCaughtUp = (state: t) => state.crossChainState->CrossChainState.markCaughtUp
492
+ let markCaughtUpIfSettled = (state: t) =>
493
+ state.crossChainState->CrossChainState.markCaughtUpIfSettled
480
494
 
481
495
  let markReady = (state: t, ~readyAt) => state.crossChainState->CrossChainState.markReady(~readyAt)
482
496
 
@@ -806,6 +820,9 @@ let recordWriteFailure = (state: t, exn) => {
806
820
  let beginWriteFiber = (state: t, fiber) => state.writeFiber = Some(fiber)
807
821
  let endWriteFiber = (state: t) => state.writeFiber = None
808
822
 
823
+ let beginFinalizeFiber = (state: t, fiber) => state.finalizeFiber = Some(fiber)
824
+ let endFinalizeFiber = (state: t) => state.finalizeFiber = None
825
+
809
826
  // Resolve and clear everyone waiting on a commit so they can re-evaluate.
810
827
  let wakeCommitWaiters = (state: t) => {
811
828
  let waiters = state.commitWaiters
@@ -3,6 +3,7 @@
3
3
  import * as Env from "./Env.res.mjs";
4
4
  import * as Utils from "./Utils.res.mjs";
5
5
  import * as Config from "./Config.res.mjs";
6
+ import * as ChainId from "./ChainId.res.mjs";
6
7
  import * as Logging from "./Logging.res.mjs";
7
8
  import * as ChainMap from "./ChainMap.res.mjs";
8
9
  import * as Internal from "./Internal.res.mjs";
@@ -67,6 +68,7 @@ function make$1(config, persistence, chainStates, isInReorgThreshold, isRealtime
67
68
  processedBatches: [],
68
69
  processedBatchesCount: 0,
69
70
  writeFiber: undefined,
71
+ finalizeFiber: undefined,
70
72
  hasFailedWrite: false,
71
73
  commitWaiters: [],
72
74
  chainMeta: {},
@@ -106,24 +108,25 @@ function makeFromDbState(config, persistence, initialState, registrationsByChain
106
108
  let shouldUseTui = shouldUseTuiOpt !== undefined ? shouldUseTuiOpt : false;
107
109
  let exitAfterFirstEventBlock = exitAfterFirstEventBlockOpt !== undefined ? exitAfterFirstEventBlockOpt : false;
108
110
  let targetBufferSize = targetBufferSizeOpt !== undefined ? targetBufferSizeOpt : CrossChainState.calculateTargetBufferSize();
109
- let isInReorgThreshold = initialState.cleanRun ? false : initialState.chains.some(chain => {
110
- let progressBlockNumber = chain.progressBlockNumber;
111
- let sourceBlockNumber = chain.sourceBlockNumber;
112
- let maxReorgDepth = chain.maxReorgDepth;
111
+ let isInReorgThreshold = initialState.cleanRun ? false : initialState.chains.some(resumedChainState => {
112
+ let progressBlockNumber = resumedChainState.progressBlockNumber;
113
+ let sourceBlockNumber = resumedChainState.sourceBlockNumber;
114
+ let maxReorgDepth = resumedChainState.maxReorgDepth;
113
115
  if (maxReorgDepth > 0 && sourceBlockNumber > 0) {
114
116
  return progressBlockNumber > (sourceBlockNumber - maxReorgDepth | 0);
115
117
  } else {
116
118
  return false;
117
119
  }
118
120
  });
119
- let isRealtime = !Env.updateSyncTimeOnRestart && initialState.chains.length !== 0 && initialState.chains.every(c => Stdlib_Option.isSome(c.timestampCaughtUpToHeadOrEndblock));
121
+ let isRealtime = initialState.chains.length !== 0 && initialState.chains.every(c => Stdlib_Option.isSome(c.timestampCaughtUpToHeadOrEndblock));
120
122
  let chainStates = {};
121
123
  initialState.chains.forEach(resumedChainState => {
122
- let chain = Config.getChain(config, resumedChainState.id);
123
- let chainConfig = ChainMap.get(config.chainMap, chain);
124
+ let chainId = Config.getChain(config, resumedChainState.id);
125
+ let chainConfig = ChainMap.get(config.chainMap, chainId);
124
126
  chainStates[resumedChainState.id] = ChainState.makeFromDbState(chainConfig, resumedChainState, initialState.reorgCheckpoints, isInReorgThreshold, isRealtime, config, registrationsByChainId, reducedPollingInterval);
125
127
  });
126
128
  let state = make$1(config, persistence, chainStates, isInReorgThreshold, isRealtime, targetBufferSize, initialState.checkpointId, isDevelopmentMode, shouldUseTui, exitAfterFirstEventBlock, onError, onExit);
129
+ CrossChainState.markCaughtUpOnResume(state.crossChainState);
127
130
  Utils.Dict.forEach(initialState.cache, param => {
128
131
  let count = param.count;
129
132
  EffectState.setUnregisteredCacheCount(state.effectState, param.effectName, param.scope, count);
@@ -170,12 +173,12 @@ function stop(state) {
170
173
  state.isStopped = true;
171
174
  }
172
175
 
173
- function getChainState(state, chain) {
174
- let cs = CrossChainState.chainStates(state.crossChainState)[chain];
176
+ function getChainState(state, chainId) {
177
+ let cs = CrossChainState.chainStates(state.crossChainState)[chainId];
175
178
  if (cs !== undefined) {
176
179
  return Primitive_option.valFromOption(cs);
177
180
  } else {
178
- return Stdlib_JsError.throwWithMessage("No chain with id " + ChainMap.Chain.toString(chain) + " found in chain states");
181
+ return Stdlib_JsError.throwWithMessage("No chain with id " + ChainId.toString(chainId) + " found in chain states");
179
182
  }
180
183
  }
181
184
 
@@ -191,7 +194,7 @@ function enterReorgThreshold(state) {
191
194
  CrossChainState.enterReorgThreshold(state.crossChainState);
192
195
  }
193
196
 
194
- function beginReorg(state, chain, blockNumber) {
197
+ function beginReorg(state, chainId, blockNumber) {
195
198
  let since = state.processingStalledOnFetchSince;
196
199
  if (since !== undefined) {
197
200
  state.processingStalledOnFetchSeconds = state.processingStalledOnFetchSeconds + Performance.secondsSince(since);
@@ -200,7 +203,7 @@ function beginReorg(state, chain, blockNumber) {
200
203
  state.epoch = state.epoch + 1 | 0;
201
204
  state.rollbackState = {
202
205
  TAG: "ReorgDetected",
203
- chain: chain,
206
+ chainId: chainId,
204
207
  blockNumber: blockNumber
205
208
  };
206
209
  }
@@ -209,10 +212,10 @@ function enterFindingReorgDepth(state) {
209
212
  state.rollbackState = "FindingReorgDepth";
210
213
  }
211
214
 
212
- function foundReorgDepth(state, chain, rollbackTargetBlockNumber) {
215
+ function foundReorgDepth(state, chainId, rollbackTargetBlockNumber) {
213
216
  state.rollbackState = {
214
217
  TAG: "FoundReorgDepth",
215
- chain: chain,
218
+ chainId: chainId,
216
219
  rollbackTargetBlockNumber: rollbackTargetBlockNumber
217
220
  };
218
221
  }
@@ -230,6 +233,7 @@ function clearRollback(state) {
230
233
 
231
234
  function invalidateInflight(state) {
232
235
  state.epoch = state.epoch + 1 | 0;
236
+ Utils.Dict.forEach(CrossChainState.chainStates(state.crossChainState), ChainState.resetPendingQueries);
233
237
  }
234
238
 
235
239
  function applyBatchProgress(state, batch) {
@@ -308,6 +312,10 @@ function writeFiber(state) {
308
312
  return state.writeFiber;
309
313
  }
310
314
 
315
+ function finalizeFiber(state) {
316
+ return state.finalizeFiber;
317
+ }
318
+
311
319
  function hasFailedWrite(state) {
312
320
  return state.hasFailedWrite;
313
321
  }
@@ -344,8 +352,8 @@ function isFinalizingIndexes(state) {
344
352
  }
345
353
  }
346
354
 
347
- function markCaughtUp(state) {
348
- CrossChainState.markCaughtUp(state.crossChainState);
355
+ function markCaughtUpIfSettled(state) {
356
+ CrossChainState.markCaughtUpIfSettled(state.crossChainState);
349
357
  }
350
358
 
351
359
  function markReady(state, readyAt) {
@@ -692,6 +700,14 @@ function endWriteFiber(state) {
692
700
  state.writeFiber = undefined;
693
701
  }
694
702
 
703
+ function beginFinalizeFiber(state, fiber) {
704
+ state.finalizeFiber = fiber;
705
+ }
706
+
707
+ function endFinalizeFiber(state) {
708
+ state.finalizeFiber = undefined;
709
+ }
710
+
695
711
  function wakeCommitWaiters(state) {
696
712
  let waiters = state.commitWaiters;
697
713
  state.commitWaiters = [];
@@ -771,6 +787,7 @@ export {
771
787
  processedBatches,
772
788
  processedBatchesCount,
773
789
  writeFiber,
790
+ finalizeFiber,
774
791
  hasFailedWrite,
775
792
  chainMetaDirty,
776
793
  chainMetaThrottler,
@@ -779,7 +796,7 @@ export {
779
796
  isInReorgThreshold,
780
797
  isRealtime,
781
798
  isFinalizingIndexes,
782
- markCaughtUp,
799
+ markCaughtUpIfSettled,
783
800
  markReady,
784
801
  rollbackState,
785
802
  indexerStartTime,
@@ -810,6 +827,8 @@ export {
810
827
  recordWriteFailure,
811
828
  beginWriteFiber,
812
829
  endWriteFiber,
830
+ beginFinalizeFiber,
831
+ endFinalizeFiber,
813
832
  wakeCommitWaiters,
814
833
  addCommitWaiter,
815
834
  stageChainMeta,
@@ -1,13 +1,12 @@
1
1
  // The indexer state. `t` is opaque: other modules read it through the accessors
2
2
  // and change it only through the transitions and setters exposed here.
3
3
 
4
- type chain = ChainMap.Chain.t
5
4
 
6
5
  type rollbackState =
7
6
  | NoRollback
8
- | ReorgDetected({chain: chain, blockNumber: int})
7
+ | ReorgDetected({chainId: ChainId.t, blockNumber: int})
9
8
  | FindingReorgDepth
10
- | FoundReorgDepth({chain: chain, rollbackTargetBlockNumber: int})
9
+ | FoundReorgDepth({chainId: ChainId.t, rollbackTargetBlockNumber: int})
11
10
  | RollbackReady({eventsProcessedDiffByChain: dict<float>})
12
11
 
13
12
  module EntityTables: {
@@ -54,11 +53,11 @@ let isStale: (t, ~stateId: int) => bool
54
53
  let isResolvingReorg: t => bool
55
54
  let errorExit: (t, ErrorHandling.t) => unit
56
55
  let stop: t => unit
57
- let getChainState: (t, ~chain: chain) => ChainState.t
56
+ let getChainState: (t, ~chainId: ChainId.t) => ChainState.t
58
57
  let enterReorgThreshold: t => unit
59
- let beginReorg: (t, ~chain: chain, ~blockNumber: int) => unit
58
+ let beginReorg: (t, ~chainId: ChainId.t, ~blockNumber: int) => unit
60
59
  let enterFindingReorgDepth: t => unit
61
- let foundReorgDepth: (t, ~chain: chain, ~rollbackTargetBlockNumber: int) => unit
60
+ let foundReorgDepth: (t, ~chainId: ChainId.t, ~rollbackTargetBlockNumber: int) => unit
62
61
  let completeRollback: (t, ~eventsProcessedDiffByChain: dict<float>) => unit
63
62
  let clearRollback: t => unit
64
63
  let invalidateInflight: t => unit
@@ -87,6 +86,7 @@ let processedCheckpointId: t => Internal.checkpointId
87
86
  let processedBatches: t => array<Batch.t>
88
87
  let processedBatchesCount: t => int
89
88
  let writeFiber: t => option<promise<unit>>
89
+ let finalizeFiber: t => option<promise<unit>>
90
90
  let hasFailedWrite: t => bool
91
91
  let chainMetaDirty: t => bool
92
92
  let chainMetaThrottler: t => Throttler.t
@@ -95,7 +95,7 @@ let chainStates: t => dict<ChainState.t>
95
95
  let isInReorgThreshold: t => bool
96
96
  let isRealtime: t => bool
97
97
  let isFinalizingIndexes: t => bool
98
- let markCaughtUp: t => unit
98
+ let markCaughtUpIfSettled: t => unit
99
99
  let markReady: (t, ~readyAt: Date.t) => unit
100
100
  let rollbackState: t => rollbackState
101
101
  let indexerStartTime: t => Date.t
@@ -145,6 +145,8 @@ let beginRollbackDiff: (
145
145
  let recordWriteFailure: (t, exn) => unit
146
146
  let beginWriteFiber: (t, promise<unit>) => unit
147
147
  let endWriteFiber: t => unit
148
+ let beginFinalizeFiber: (t, promise<unit>) => unit
149
+ let endFinalizeFiber: t => unit
148
150
  let wakeCommitWaiters: t => unit
149
151
  let addCommitWaiter: (t, unit => unit) => unit
150
152
  let stageChainMeta: (t, dict<InternalTable.Chains.metaFields>) => unit
package/src/Internal.res CHANGED
@@ -301,7 +301,7 @@ type genericEvent<'params, 'block, 'transaction> = {
301
301
  contractName: string,
302
302
  eventName: string,
303
303
  params: 'params,
304
- chainId: int,
304
+ chainId: ChainId.t,
305
305
  srcAddress: Address.t,
306
306
  logIndex: int,
307
307
  transaction: 'transaction,
@@ -369,7 +369,7 @@ type entityHandlerContext<'entity> = {
369
369
  }
370
370
 
371
371
  type chainInfo = {
372
- id: int,
372
+ id: ChainId.t,
373
373
  // True once every chain has caught up to head/endBlock and entered real-time
374
374
  // indexing mode. False while any chain is still backfilling.
375
375
  isRealtime: bool,
@@ -624,7 +624,7 @@ type dcs = array<indexingAddress>
624
624
  type eventItem = private {
625
625
  kind: [#0],
626
626
  onEventRegistration: onEventRegistration,
627
- chain: ChainMap.Chain.t,
627
+ chainId: ChainId.t,
628
628
  blockNumber: int,
629
629
  logIndex: int,
630
630
  // Within-block transaction index — the key into the per-chain transaction
@@ -637,7 +637,7 @@ type eventItem = private {
637
637
  // `InternalTable`) so the ecosystem's `toRawEvent` can reference it without
638
638
  // pulling in `InternalTable`'s dependency on `Config`.
639
639
  type rawEvent = {
640
- chain_id: int,
640
+ chain_id: ChainId.t,
641
641
  event_id: bigint,
642
642
  event_name: string,
643
643
  contract_name: string,
@@ -665,7 +665,7 @@ type onBlockRegistration = {
665
665
  // we want to use the order they are defined for sorting
666
666
  index: int,
667
667
  name: string,
668
- chainId: int,
668
+ chainId: ChainId.t,
669
669
  startBlock: option<int>,
670
670
  endBlock: option<int>,
671
671
  interval: int,
@@ -677,7 +677,7 @@ type item =
677
677
  | @as(0)
678
678
  Event({
679
679
  onEventRegistration: onEventRegistration,
680
- chain: ChainMap.Chain.t,
680
+ chainId: ChainId.t,
681
681
  blockNumber: int,
682
682
  logIndex: int,
683
683
  transactionIndex: int,
@@ -694,7 +694,7 @@ external getItemLogIndex: item => int = "logIndex"
694
694
 
695
695
  let getItemChainId = item =>
696
696
  switch item {
697
- | Event({chain}) => chain->ChainMap.Chain.toChainId
697
+ | Event({chainId})
698
698
  | Block({onBlockRegistration: {chainId}}) => chainId
699
699
  }
700
700
 
@@ -796,7 +796,7 @@ type effect = {
796
796
  @unboxed
797
797
  type chainScope =
798
798
  | @as("crossChain") CrossChain
799
- | Chain(int)
799
+ | Chain(ChainId.t)
800
800
 
801
801
  let cacheTablePrefix = "envio_effect_"
802
802
 
@@ -804,27 +804,29 @@ let cacheTablePrefix = "envio_effect_"
804
804
  // canonical Postgres cache-table name and .envio/cache file path. Everything
805
805
  // that needs a cache address goes through here instead of slicing prefixes.
806
806
  // CrossChain -> envio_effect_<name> <name>.tsv
807
- // Chain(1) -> envio_1_effect_<name> 1/<name>.tsv
808
- // Chain(137) -> envio_137_effect_<name> 137/<name>.tsv
807
+ // Chain(1->ChainId.fromInt) -> envio_1_effect_<name> 1/<name>.tsv
808
+ // Chain(137->ChainId.fromInt) -> envio_137_effect_<name> 137/<name>.tsv
809
809
  module EffectCache = {
810
810
  let toTableName = (~effectName, ~scope) =>
811
811
  switch scope {
812
812
  | CrossChain => cacheTablePrefix ++ effectName
813
- | Chain(chainId) => `envio_${chainId->Int.toString}_effect_${effectName}`
813
+ | Chain(chainId) => `envio_${chainId->ChainId.toString}_effect_${effectName}`
814
814
  }
815
815
 
816
816
  // "crossChain" or the decimal chain id. Used as the `scope` Prometheus label.
817
817
  let scopeToString = scope =>
818
818
  switch scope {
819
819
  | CrossChain => "crossChain"
820
- | Chain(chainId) => chainId->Int.toString
820
+ | Chain(chainId) => chainId->ChainId.toString
821
821
  }
822
822
 
823
823
  // Only accepts a canonical decimal chain id ("7", not "007" or "1foo") —
824
- // Int.fromString alone follows parseInt semantics and accepts both.
824
+ // the schema's parser follows parseFloat semantics and accepts both.
825
825
  let parseChainId = str =>
826
- switch Int.fromString(str) {
827
- | Some(chainId) if chainId >= 0 && chainId->Int.toString === str => Some(chainId)
826
+ switch try Some(str->ChainId.normalizeOrThrow) catch {
827
+ | _ => None
828
+ } {
829
+ | Some(chainId) if chainId->ChainId.toString === str => Some(chainId)
828
830
  | _ => None
829
831
  }
830
832
 
@@ -864,7 +866,7 @@ module EffectCache = {
864
866
  let toCachePath = (~effectName, ~scope) =>
865
867
  switch scope {
866
868
  | CrossChain => effectName ++ ".tsv"
867
- | Chain(chainId) => `${chainId->Int.toString}/${effectName}.tsv`
869
+ | Chain(chainId) => `${chainId->ChainId.toString}/${effectName}.tsv`
868
870
  }
869
871
  }
870
872
 
@@ -893,7 +895,7 @@ type reorgCheckpoint = {
893
895
  @as("id")
894
896
  checkpointId: bigint,
895
897
  @as("chain_id")
896
- chainId: int,
898
+ chainId: ChainId.t,
897
899
  @as("block_number")
898
900
  blockNumber: int,
899
901
  @as("block_hash")
@@ -3,7 +3,7 @@
3
3
  import * as Table from "./db/Table.res.mjs";
4
4
  import * as Utils from "./Utils.res.mjs";
5
5
  import * as Address from "./Address.res.mjs";
6
- import * as Stdlib_Int from "@rescript/runtime/lib/es6/Stdlib_Int.js";
6
+ import * as ChainId from "./ChainId.res.mjs";
7
7
  import * as Primitive_option from "@rescript/runtime/lib/es6/Primitive_option.js";
8
8
  import * as S$RescriptSchema from "rescript-schema/src/S.res.mjs";
9
9
 
@@ -149,7 +149,7 @@ function dependsOnAddresses(isWildcard, filterByAddresses) {
149
149
 
150
150
  function getItemChainId(item) {
151
151
  if (item.kind === 0) {
152
- return item.chain;
152
+ return item.chainId;
153
153
  } else {
154
154
  return item.onBlockRegistration.chainId;
155
155
  }
@@ -172,7 +172,7 @@ function toTableName(effectName, scope) {
172
172
  if (scope === "crossChain") {
173
173
  return cacheTablePrefix + effectName;
174
174
  } else {
175
- return `envio_` + scope.toString() + `_effect_` + effectName;
175
+ return `envio_` + ChainId.toString(scope) + `_effect_` + effectName;
176
176
  }
177
177
  }
178
178
 
@@ -180,14 +180,23 @@ function scopeToString(scope) {
180
180
  if (scope === "crossChain") {
181
181
  return "crossChain";
182
182
  } else {
183
- return scope.toString();
183
+ return ChainId.toString(scope);
184
184
  }
185
185
  }
186
186
 
187
187
  function parseChainId(str) {
188
- let chainId = Stdlib_Int.fromString(str, undefined);
189
- if (chainId !== undefined && chainId >= 0 && chainId.toString() === str) {
190
- return chainId;
188
+ let chainId;
189
+ try {
190
+ chainId = Primitive_option.some(ChainId.normalizeOrThrow(str));
191
+ } catch (exn) {
192
+ chainId = undefined;
193
+ }
194
+ if (chainId === undefined) {
195
+ return;
196
+ }
197
+ let chainId$1 = Primitive_option.valFromOption(chainId);
198
+ if (ChainId.toString(chainId$1) === str) {
199
+ return Primitive_option.some(chainId$1);
191
200
  }
192
201
  }
193
202
 
@@ -236,7 +245,7 @@ function fromTableName(tableName) {
236
245
  if (chainId !== undefined) {
237
246
  return [
238
247
  effectName$1,
239
- chainId
248
+ Primitive_option.valFromOption(chainId)
240
249
  ];
241
250
  }
242
251
  }
@@ -245,7 +254,7 @@ function toCachePath(effectName, scope) {
245
254
  if (scope === "crossChain") {
246
255
  return effectName + ".tsv";
247
256
  } else {
248
- return scope.toString() + `/` + effectName + `.tsv`;
257
+ return ChainId.toString(scope) + `/` + effectName + `.tsv`;
249
258
  }
250
259
  }
251
260
 
package/src/LoadLayer.res CHANGED
@@ -226,7 +226,7 @@ let loadEffect = (
226
226
  // the storage-load metric stays stable.
227
227
  let key = switch scope {
228
228
  | CrossChain => `${effectName}.effect`
229
- | Chain(chainId) => `${effectName}.effect.${chainId->Int.toString}`
229
+ | Chain(chainId) => `${effectName}.effect.${chainId->ChainId.toString}`
230
230
  }
231
231
 
232
232
  let load = async (args, ~onError) => {
@@ -2,6 +2,7 @@
2
2
 
3
3
  import * as Table from "./db/Table.res.mjs";
4
4
  import * as Utils from "./Utils.res.mjs";
5
+ import * as ChainId from "./ChainId.res.mjs";
5
6
  import * as Logging from "./Logging.res.mjs";
6
7
  import * as Ecosystem from "./Ecosystem.res.mjs";
7
8
  import * as EffectState from "./EffectState.res.mjs";
@@ -113,7 +114,7 @@ function loadEffect(loadManager, persistence, effect, effectArgs, scope, indexer
113
114
  let table = inMemTable.table;
114
115
  let tableName = table.tableName;
115
116
  let key;
116
- key = scope === "crossChain" ? effectName + `.effect` : effectName + `.effect.` + scope.toString();
117
+ key = scope === "crossChain" ? effectName + `.effect` : effectName + `.effect.` + ChainId.toString(scope);
117
118
  let load = async (args, onError) => {
118
119
  let idsToLoad = args.map(arg => arg.cacheKey);
119
120
  let idsFromCache = new Set();