envio 3.9.0 → 3.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (110) hide show
  1. package/evm.schema.json +26 -0
  2. package/fuel.schema.json +26 -0
  3. package/index.d.ts +35 -3
  4. package/package.json +6 -7
  5. package/src/Batch.res +9 -0
  6. package/src/ChainState.res +35 -39
  7. package/src/ChainState.res.mjs +14 -11
  8. package/src/ChainState.resi +15 -7
  9. package/src/Config.res +52 -44
  10. package/src/Config.res.mjs +51 -17
  11. package/src/Core.res +4 -0
  12. package/src/CrossChainState.res +15 -23
  13. package/src/CrossChainState.res.mjs +10 -18
  14. package/src/CrossChainState.resi +4 -1
  15. package/src/Ecosystem.res +7 -2
  16. package/src/Envio.res +6 -4
  17. package/src/EventConfigBuilder.res +169 -26
  18. package/src/EventConfigBuilder.res.mjs +124 -15
  19. package/src/HandlerRegister.res +21 -4
  20. package/src/HandlerRegister.res.mjs +24 -5
  21. package/src/InMemoryStore.res +9 -7
  22. package/src/InMemoryStore.res.mjs +3 -3
  23. package/src/IndexerState.res +43 -6
  24. package/src/IndexerState.res.mjs +36 -7
  25. package/src/IndexerState.resi +4 -4
  26. package/src/Internal.res +16 -17
  27. package/src/Metrics.res +72 -6
  28. package/src/Metrics.res.mjs +30 -2
  29. package/src/Persistence.res +36 -30
  30. package/src/Persistence.res.mjs +1 -20
  31. package/src/PgStorage.res +104 -56
  32. package/src/PgStorage.res.mjs +57 -35
  33. package/src/PruneStaleHistory.res +85 -52
  34. package/src/PruneStaleHistory.res.mjs +63 -37
  35. package/src/Rollback.res +68 -45
  36. package/src/Rollback.res.mjs +38 -19
  37. package/src/RollbackCommit.res +7 -9
  38. package/src/RollbackCommit.res.mjs +5 -6
  39. package/src/SafeCheckpointTracking.res +11 -11
  40. package/src/SafeCheckpointTracking.res.mjs +7 -6
  41. package/src/SimulateItems.res +33 -18
  42. package/src/SimulateItems.res.mjs +26 -22
  43. package/src/Sink.res +38 -26
  44. package/src/Sink.res.mjs +21 -16
  45. package/src/TestIndexer.res +5 -5
  46. package/src/TestIndexer.res.mjs +1 -1
  47. package/src/Utils.res +170 -0
  48. package/src/Utils.res.mjs +153 -0
  49. package/src/Writing.res +2 -2
  50. package/src/Writing.res.mjs +1 -1
  51. package/src/bindings/ClickHouse.res +366 -698
  52. package/src/bindings/ClickHouse.res.mjs +371 -421
  53. package/src/bindings/ClickHouseSink.res +337 -0
  54. package/src/bindings/ClickHouseSink.res.mjs +246 -0
  55. package/src/bindings/EventSource.res +8 -2
  56. package/src/bindings/NodeJs.res +6 -0
  57. package/src/bindings/Vitest.res +23 -0
  58. package/src/bindings/Vitest.res.mjs +3 -0
  59. package/src/bindings/WebSocket.res +9 -10
  60. package/src/db/CheckpointBounds.res +53 -0
  61. package/src/db/CheckpointBounds.res.mjs +44 -0
  62. package/src/db/EntityFilter.res +36 -17
  63. package/src/db/EntityFilter.res.mjs +25 -14
  64. package/src/db/EntityHistory.res +47 -22
  65. package/src/db/EntityHistory.res.mjs +19 -12
  66. package/src/db/InternalTable.res +84 -35
  67. package/src/db/InternalTable.res.mjs +56 -23
  68. package/src/db/RollbackFloors.res +55 -0
  69. package/src/db/RollbackFloors.res.mjs +92 -0
  70. package/src/db/Table.res +22 -4
  71. package/src/db/Table.res.mjs +30 -9
  72. package/src/sources/Evm.res +2 -0
  73. package/src/sources/Evm.res.mjs +2 -0
  74. package/src/sources/EvmHyperSyncSource.res +3 -3
  75. package/src/sources/EvmHyperSyncSource.res.mjs +3 -3
  76. package/src/sources/EvmRpcWs.res +124 -0
  77. package/src/sources/EvmRpcWs.res.mjs +117 -0
  78. package/src/sources/Fuel.res +2 -0
  79. package/src/sources/Fuel.res.mjs +2 -0
  80. package/src/sources/FuelHyperSyncSource.res +1 -1
  81. package/src/sources/FuelHyperSyncSource.res.mjs +2 -1
  82. package/src/sources/HeightFeed.res +568 -0
  83. package/src/sources/HeightFeed.res.mjs +452 -0
  84. package/src/sources/HeightStream.res +257 -0
  85. package/src/sources/HeightStream.res.mjs +194 -0
  86. package/src/sources/HyperSync.res +5 -0
  87. package/src/sources/HyperSync.res.mjs +3 -0
  88. package/src/sources/HyperSync.resi +4 -0
  89. package/src/sources/HyperSyncSSE.res +49 -0
  90. package/src/sources/HyperSyncSSE.res.mjs +79 -0
  91. package/src/sources/RpcSource.res +1 -1
  92. package/src/sources/RpcSource.res.mjs +2 -2
  93. package/src/sources/Source.res +54 -1
  94. package/src/sources/Source.res.mjs +25 -0
  95. package/src/sources/SourceManager.res +220 -227
  96. package/src/sources/SourceManager.res.mjs +139 -164
  97. package/src/sources/SourceManager.resi +8 -0
  98. package/src/sources/Svm.res +2 -0
  99. package/src/sources/Svm.res.mjs +2 -0
  100. package/src/sources/SvmHyperSyncClient.res +70 -127
  101. package/src/sources/SvmHyperSyncClient.res.mjs +51 -25
  102. package/src/sources/SvmHyperSyncSource.res +18 -15
  103. package/src/sources/SvmHyperSyncSource.res.mjs +5 -4
  104. package/svm.schema.json +1 -75
  105. package/src/MemoryStorage.res +0 -729
  106. package/src/MemoryStorage.res.mjs +0 -586
  107. package/src/sources/HyperSyncHeightStream.res +0 -129
  108. package/src/sources/HyperSyncHeightStream.res.mjs +0 -110
  109. package/src/sources/RpcWebSocketHeightStream.res +0 -175
  110. package/src/sources/RpcWebSocketHeightStream.res.mjs +0 -180
@@ -154,6 +154,7 @@ let buildOnEventRegistrationWith = (
154
154
  ~isWildcard,
155
155
  ~handler,
156
156
  ~contractRegister,
157
+ ~where,
157
158
  ~fieldSelection?,
158
159
  ~startBlock?,
159
160
  ) :> Internal.onEventRegistration)
@@ -320,7 +321,7 @@ let addOnEventRegistration = (
320
321
  )
321
322
  | Fuel =>
322
323
  JsError.throwWithMessage(
323
- `The fields option of the "${eventName}" event registration on contract "${contractName}" is only supported on EVM. Select the fields in your config instead.`,
324
+ `The fields option of the "${eventName}" event registration on contract "${contractName}" is not supported on Fuel. Select the fields in your config instead.`,
324
325
  )
325
326
  }
326
327
  }
@@ -335,6 +336,21 @@ let addOnEventRegistration = (
335
336
  | None => ()
336
337
  | Some(eventConfig) =>
337
338
  matched := true
339
+ switch (registration.config.ecosystem.name, fieldSelection) {
340
+ | (Svm, Some(fieldSelection)) if fieldSelection.instructionFields->Utils.Set.has("args") =>
341
+ let svmEventConfig =
342
+ eventConfig->(Utils.magic: Internal.eventConfig => Internal.svmInstructionEventConfig)
343
+ let declaresArgs = switch svmEventConfig.args {
344
+ | JSON.Array(args) => args->Array.length > 0
345
+ | _ => false
346
+ }
347
+ if !declaresArgs {
348
+ JsError.throwWithMessage(
349
+ `Invalid "args" field in the fields.instruction option of the "${eventName}" instruction on program "${contractName}". The instruction declares no args in config.yaml, so there is nothing to decode. Remove "args" from the selection, or declare the instruction's args.`,
350
+ )
351
+ }
352
+ | _ => ()
353
+ }
338
354
  let reg = buildOnEventRegistrationWith(
339
355
  ~config=registration.config,
340
356
  ~chainId=chainConfig.id,
@@ -359,17 +375,18 @@ let addOnEventRegistration = (
359
375
  if !matched.contents {
360
376
  let (contractNames, eventNames) = registration->describeConfigured(~contractName)
361
377
  let listOr = (names, empty) =>
362
- names->Utils.Array.isEmpty ? empty : names->Array.joinUnsafe(", ")
378
+ names->Utils.Array.isEmpty ? empty : names->Utils.Array.quotedJoin
379
+ let {contractNoun, eventNoun} = registration.config.ecosystem
363
380
  if eventNames->Utils.Array.isEmpty {
364
381
  JsError.throwWithMessage(
365
- `Contract "${contractName}" is not configured on any chain, so its handler for "${eventName}" would never run. Add it to your config, or remove the registration. Configured contracts: ${listOr(
382
+ `${contractNoun->Utils.String.capitalize} "${contractName}" is not configured on any chain, so its handler for "${eventName}" would never run. Add it to your config, or remove the registration. Configured ${contractNoun}s: ${listOr(
366
383
  contractNames,
367
384
  "none",
368
385
  )}.`,
369
386
  )
370
387
  } else {
371
388
  JsError.throwWithMessage(
372
- `Event "${eventName}" is not configured on contract "${contractName}", so its handler would never run. Add it to your config, or remove the registration. Configured events on "${contractName}": ${listOr(
389
+ `${eventNoun->Utils.String.capitalize} "${eventName}" is not configured on ${contractNoun} "${contractName}", so its handler would never run. Add it to your config, or remove the registration. Configured ${eventNoun}s on "${contractName}": ${listOr(
373
390
  eventNames,
374
391
  "none",
375
392
  )}.`,
@@ -102,7 +102,7 @@ function buildOnEventRegistrationWith(config, chainId, eventConfig, isWildcard,
102
102
  case "fuel" :
103
103
  return EventConfigBuilder.buildFuelOnEventRegistration(eventConfig, isWildcard, handler, contractRegister, startBlock);
104
104
  case "svm" :
105
- return EventConfigBuilder.buildSvmOnEventRegistration(eventConfig, isWildcard, handler, contractRegister, fieldSelection, startBlock);
105
+ return EventConfigBuilder.buildSvmOnEventRegistration(eventConfig, isWildcard, handler, contractRegister, where, fieldSelection, startBlock);
106
106
  }
107
107
  }
108
108
 
@@ -209,7 +209,7 @@ function addOnEventRegistration(registration, contractName, eventName, handler,
209
209
  fieldSelection = EventConfigBuilder.resolveInlineFieldSelection(fields$1, contractName, eventName, registration.config.enableRawEvents);
210
210
  break;
211
211
  case "fuel" :
212
- fieldSelection = Stdlib_JsError.throwWithMessage(`The fields option of the "` + eventName + `" event registration on contract "` + contractName + `" is only supported on EVM. Select the fields in your config instead.`);
212
+ fieldSelection = Stdlib_JsError.throwWithMessage(`The fields option of the "` + eventName + `" event registration on contract "` + contractName + `" is not supported on Fuel. Select the fields in your config instead.`);
213
213
  break;
214
214
  case "svm" :
215
215
  fieldSelection = EventConfigBuilder.resolveSvmInlineFieldSelection(fields$1, contractName, eventName);
@@ -231,6 +231,22 @@ function addOnEventRegistration(registration, contractName, eventName, handler,
231
231
  return;
232
232
  }
233
233
  matched.contents = true;
234
+ let match = registration.config.ecosystem.name;
235
+ switch (match) {
236
+ case "evm" :
237
+ case "fuel" :
238
+ break;
239
+ case "svm" :
240
+ if (fieldSelection !== undefined && fieldSelection.instructionFields.has("args")) {
241
+ let args = eventConfig.args;
242
+ let declaresArgs;
243
+ declaresArgs = Array.isArray(args) ? args.length !== 0 : false;
244
+ if (!declaresArgs) {
245
+ Stdlib_JsError.throwWithMessage(`Invalid "args" field in the fields.instruction option of the "` + eventName + `" instruction on program "` + contractName + `". The instruction declares no args in config.yaml, so there is nothing to decode. Remove "args" from the selection, or declare the instruction's args.`);
246
+ }
247
+ }
248
+ break;
249
+ }
234
250
  let reg = buildOnEventRegistrationWith(registration.config, chainConfig.id, eventConfig, isWildcard, handler, contractRegister, where, fieldSelection, contract.startBlock);
235
251
  getChainRegistrations(registration, chainConfig.id).onEventRegistrations.push(reg);
236
252
  });
@@ -243,13 +259,16 @@ function addOnEventRegistration(registration, contractName, eventName, handler,
243
259
  if (Utils.$$Array.isEmpty(names)) {
244
260
  return empty;
245
261
  } else {
246
- return names.join(", ");
262
+ return Utils.$$Array.quotedJoin(names);
247
263
  }
248
264
  };
265
+ let match$2 = registration.config.ecosystem;
266
+ let eventNoun = match$2.eventNoun;
267
+ let contractNoun = match$2.contractNoun;
249
268
  if (Utils.$$Array.isEmpty(eventNames)) {
250
- return Stdlib_JsError.throwWithMessage(`Contract "` + contractName + `" is not configured on any chain, so its handler for "` + eventName + `" would never run. Add it to your config, or remove the registration. Configured contracts: ` + listOr(match$1[0], "none") + `.`);
269
+ return Stdlib_JsError.throwWithMessage(Utils.$$String.capitalize(contractNoun) + ` "` + contractName + `" is not configured on any chain, so its handler for "` + eventName + `" would never run. Add it to your config, or remove the registration. Configured ` + contractNoun + `s: ` + listOr(match$1[0], "none") + `.`);
251
270
  } else {
252
- return Stdlib_JsError.throwWithMessage(`Event "` + eventName + `" is not configured on contract "` + contractName + `", so its handler would never run. Add it to your config, or remove the registration. Configured events on "` + contractName + `": ` + listOr(eventNames, "none") + `.`);
271
+ return Stdlib_JsError.throwWithMessage(Utils.$$String.capitalize(eventNoun) + ` "` + eventName + `" is not configured on ` + contractNoun + ` "` + contractName + `", so its handler would never run. Add it to your config, or remove the registration. Configured ` + eventNoun + `s on "` + contractName + `": ` + listOr(eventNames, "none") + `.`);
253
272
  }
254
273
  }
255
274
 
@@ -127,15 +127,15 @@ let dropCommittedEffects = (
127
127
 
128
128
  let prepareRollbackDiff = async (
129
129
  state: IndexerState.t,
130
- ~rollbackTargetCheckpointId,
130
+ ~floors: RollbackFloors.t,
131
131
  ~rollbackDiffCheckpointId,
132
- ~progressBlockNumberByChainId,
132
+ ~progressedChains,
133
133
  ~rolledBackAddresses,
134
134
  ) => {
135
135
  state->IndexerState.beginRollbackDiff(
136
- ~targetCheckpointId=rollbackTargetCheckpointId,
137
136
  ~diffCheckpointId=rollbackDiffCheckpointId,
138
- ~progressBlockNumberByChainId,
137
+ ~floors,
138
+ ~progressedChains,
139
139
  ~rolledBackAddresses,
140
140
  )
141
141
  let persistence = state->IndexerState.persistence
@@ -152,7 +152,7 @@ let prepareRollbackDiff = async (
152
152
  ->Array.map(async entityConfig => {
153
153
  let (removals, restoredEntities) = await persistence.storage.getRollbackData(
154
154
  ~entityConfig,
155
- ~rollbackTargetCheckpointId,
155
+ ~floors,
156
156
  )
157
157
 
158
158
  removals->Array.forEach(({entityId, scope}: Persistence.rollbackRemoval) => {
@@ -215,7 +215,8 @@ let setBatchDcs = (state: IndexerState.t, ~batch: Batch.t) => {
215
215
  }
216
216
  }
217
217
 
218
- batch.registeredAddresses->Array.pushMany(
218
+ batch.registeredAddresses
219
+ ->Array.pushMany(
219
220
  chainState
220
221
  ->ChainState.drainAddressesForWrite(
221
222
  ~toBlockInclusive=progressedChain.progressBlockNumber,
@@ -230,7 +231,8 @@ let setBatchDcs = (state: IndexerState.t, ~batch: Batch.t) => {
230
231
  },
231
232
  checkpointId: checkpointIds->Array.getUnsafe(dc.checkpointIdx),
232
233
  }),
233
- )->ignore
234
+ )
235
+ ->ignore
234
236
  }
235
237
  })
236
238
  }
@@ -106,14 +106,14 @@ function dropCommittedEffects(inMemTable, committedCheckpointId, keepLoadedFromD
106
106
  inMemTable.changesCount = inMemTable.changesCount - keysToDelete.length;
107
107
  }
108
108
 
109
- async function prepareRollbackDiff(state, rollbackTargetCheckpointId, rollbackDiffCheckpointId, progressBlockNumberByChainId, rolledBackAddresses) {
110
- IndexerState.beginRollbackDiff(state, rollbackTargetCheckpointId, rollbackDiffCheckpointId, progressBlockNumberByChainId, rolledBackAddresses);
109
+ async function prepareRollbackDiff(state, floors, rollbackDiffCheckpointId, progressedChains, rolledBackAddresses) {
110
+ IndexerState.beginRollbackDiff(state, rollbackDiffCheckpointId, floors, progressedChains, rolledBackAddresses);
111
111
  let persistence = IndexerState.persistence(state);
112
112
  let committedCheckpointId = IndexerState.committedCheckpointId(state);
113
113
  let deletedEntities = {};
114
114
  let setEntities = {};
115
115
  await Promise.all(persistence.allEntities.filter(entityConfig => entityConfig.storage.postgres).map(async entityConfig => {
116
- let match = await persistence.storage.getRollbackData(entityConfig, rollbackTargetCheckpointId);
116
+ let match = await persistence.storage.getRollbackData(entityConfig, floors);
117
117
  match[0].forEach(param => {
118
118
  let entityId = param.entityId;
119
119
  Utils.Dict.push(deletedEntities, entityConfig.name, entityId);
@@ -355,7 +355,10 @@ let getChainState = (state: t, ~chainId: ChainId.t): ChainState.t =>
355
355
  )
356
356
  }
357
357
 
358
- let getSafeCheckpointId = (state: t) => state.crossChainState->CrossChainState.getSafeCheckpointId
358
+ let getSafeCheckpointIdByChain = (state: t) =>
359
+ state.crossChainState->CrossChainState.getSafeCheckpointIdByChain(
360
+ ~committedCheckpointId=state.committedCheckpointId,
361
+ )
359
362
 
360
363
  let createBatch = (
361
364
  state: t,
@@ -452,7 +455,14 @@ let entities = (state: t) => state.entities
452
455
  // Every in-memory entity table across all scopes, cross-chain first. The size,
453
456
  // drop and flush passes walk the whole store this way instead of assuming a
454
457
  // single partition.
455
- let eachEntityTable = (state: t, fn: (~entityConfig: Internal.entityConfig, ~scope: Internal.chainScope, ~table: InMemoryTable.Entity.t) => unit) => {
458
+ let eachEntityTable = (
459
+ state: t,
460
+ fn: (
461
+ ~entityConfig: Internal.entityConfig,
462
+ ~scope: Internal.chainScope,
463
+ ~table: InMemoryTable.Entity.t,
464
+ ) => unit,
465
+ ) => {
456
466
  let chainStates = state.crossChainState->CrossChainState.chainStates
457
467
  state.allEntities->Array.forEach(entityConfig =>
458
468
  if entityConfig.crossChain {
@@ -516,6 +526,7 @@ let toMetrics = (state: t): Metrics.t => {
516
526
  let chainStates = state.crossChainState->CrossChainState.chainStates
517
527
  let sourceRequests = []
518
528
  let sourceHeights = []
529
+ let sourceHeightStreams = []
519
530
  chainStates->Utils.Dict.forEach(cs => {
520
531
  let sourceManager = cs->ChainState.sourceManager
521
532
  sourceManager
@@ -538,6 +549,16 @@ let toMetrics = (state: t): Metrics.t => {
538
549
  height: s.height,
539
550
  })
540
551
  )
552
+ sourceManager
553
+ ->SourceManager.getHeightStreamSamples
554
+ ->Array.forEach(s =>
555
+ sourceHeightStreams->Array.push({
556
+ Metrics.source: s.sourceName,
557
+ chainId: s.chainId,
558
+ connectCount: s.stream.connectCount,
559
+ disconnectsByReason: s.stream.disconnectsByReason,
560
+ })
561
+ )
541
562
  })
542
563
  let historyPrunes = []
543
564
  state.historyPruneStats->Utils.Dict.forEachWithKey((s, entityName) =>
@@ -600,6 +621,7 @@ let toMetrics = (state: t): Metrics.t => {
600
621
  historyPrunes,
601
622
  sourceRequests,
602
623
  sourceHeights,
624
+ sourceHeightStreams,
603
625
  }
604
626
  }
605
627
 
@@ -793,6 +815,8 @@ let drainBatchRun = (state: t): Batch.t => {
793
815
  }
794
816
  }
795
817
 
818
+ let pendingRollback = (state: t): option<Persistence.rollback> => state.rollback
819
+
796
820
  // Take the pending rollback diff to write, clearing it from the store.
797
821
  let takeRollback = (state: t): option<Persistence.rollback> => {
798
822
  let rollback = state.rollback
@@ -812,9 +836,9 @@ let markCommitted = (state: t, ~upToCheckpointId) => {
812
836
  // Reset the in-memory tables and arm the rollback diff that the next write commits.
813
837
  let beginRollbackDiff = (
814
838
  state: t,
815
- ~targetCheckpointId,
816
839
  ~diffCheckpointId,
817
- ~progressBlockNumberByChainId,
840
+ ~floors,
841
+ ~progressedChains: array<InternalTable.Chains.progressedChain>,
818
842
  ~rolledBackAddresses,
819
843
  ) => {
820
844
  let perChainEntities = state.allEntities->EntityTables.perChain
@@ -829,10 +853,23 @@ let beginRollbackDiff = (
829
853
  | Some({rolledBackAddresses: pending}) => pending->Array.concat(rolledBackAddresses)
830
854
  | None => rolledBackAddresses
831
855
  }
856
+ // Same for the chains: this rollback recomputes progress from the checkpoints,
857
+ // so a chain the pending diff already moved can land on exactly the block it
858
+ // is now at and go unreported here. Its stored progress still needs
859
+ // correcting, so the pending rows carry over — this rollback's row for a
860
+ // chain wins, being the later reading of the same chain state.
861
+ let progressedChains = switch state.rollback {
862
+ | Some({progressedChains: pending}) =>
863
+ let byChainId = Dict.make()
864
+ pending->Array.forEach(chain => byChainId->ChainId.Dict.set(chain.chainId, chain))
865
+ progressedChains->Array.forEach(chain => byChainId->ChainId.Dict.set(chain.chainId, chain))
866
+ byChainId->Dict.valuesToArray
867
+ | None => progressedChains
868
+ }
832
869
  state.rollback = Some({
833
- targetCheckpointId,
834
870
  diffCheckpointId,
835
- progressBlockNumberByChainId,
871
+ floors,
872
+ progressedChains,
836
873
  rolledBackAddresses,
837
874
  })
838
875
  }
@@ -159,8 +159,8 @@ function getChainState(state, chainId) {
159
159
  }
160
160
  }
161
161
 
162
- function getSafeCheckpointId(state) {
163
- return CrossChainState.getSafeCheckpointId(state.crossChainState);
162
+ function getSafeCheckpointIdByChain(state) {
163
+ return CrossChainState.getSafeCheckpointIdByChain(state.crossChainState, state.committedCheckpointId);
164
164
  }
165
165
 
166
166
  function createBatch(state, processedCheckpointId, batchSizeTarget, isRollback) {
@@ -392,6 +392,7 @@ function toMetrics(state) {
392
392
  let chainStates = CrossChainState.chainStates(state.crossChainState);
393
393
  let sourceRequests = [];
394
394
  let sourceHeights = [];
395
+ let sourceHeightStreams = [];
395
396
  Utils.Dict.forEach(chainStates, cs => {
396
397
  let sourceManager = ChainState.sourceManager(cs);
397
398
  SourceManager.getRequestStatSamples(sourceManager).forEach(s => {
@@ -410,6 +411,14 @@ function toMetrics(state) {
410
411
  height: s.height
411
412
  });
412
413
  });
414
+ SourceManager.getHeightStreamSamples(sourceManager).forEach(s => {
415
+ sourceHeightStreams.push({
416
+ source: s.sourceName,
417
+ chainId: s.chainId,
418
+ connectCount: s.stream.connectCount,
419
+ disconnectsByReason: s.stream.disconnectsByReason
420
+ });
421
+ });
413
422
  });
414
423
  let historyPrunes = [];
415
424
  Utils.Dict.forEachWithKey(state.historyPruneStats, (s, entityName) => {
@@ -464,7 +473,8 @@ function toMetrics(state) {
464
473
  })),
465
474
  historyPrunes: historyPrunes,
466
475
  sourceRequests: sourceRequests,
467
- sourceHeights: sourceHeights
476
+ sourceHeights: sourceHeights,
477
+ sourceHeightStreams: sourceHeightStreams
468
478
  };
469
479
  }
470
480
 
@@ -658,6 +668,10 @@ function drainBatchRun(state) {
658
668
  };
659
669
  }
660
670
 
671
+ function pendingRollback(state) {
672
+ return state.rollback;
673
+ }
674
+
661
675
  function takeRollback(state) {
662
676
  let rollback = state.rollback;
663
677
  state.rollback = undefined;
@@ -668,18 +682,32 @@ function markCommitted(state, upToCheckpointId) {
668
682
  state.committedCheckpointId = upToCheckpointId;
669
683
  }
670
684
 
671
- function beginRollbackDiff(state, targetCheckpointId, diffCheckpointId, progressBlockNumberByChainId, rolledBackAddresses) {
685
+ function beginRollbackDiff(state, diffCheckpointId, floors, progressedChains, rolledBackAddresses) {
672
686
  let perChainEntities = EntityTables.perChain(state.allEntities);
673
687
  state.entities = EntityTables.make(EntityTables.crossChain(state.allEntities));
674
688
  Utils.Dict.forEach(CrossChainState.chainStates(state.crossChainState), cs => ChainState.resetEntities(cs, perChainEntities));
675
689
  EffectState.resetForRollback(state.effectState);
676
690
  let match = state.rollback;
677
691
  let rolledBackAddresses$1 = match !== undefined ? match.rolledBackAddresses.concat(rolledBackAddresses) : rolledBackAddresses;
692
+ let match$1 = state.rollback;
693
+ let progressedChains$1;
694
+ if (match$1 !== undefined) {
695
+ let byChainId = {};
696
+ match$1.progressedChains.forEach(chain => {
697
+ byChainId[chain.chainId] = chain;
698
+ });
699
+ progressedChains.forEach(chain => {
700
+ byChainId[chain.chainId] = chain;
701
+ });
702
+ progressedChains$1 = Object.values(byChainId);
703
+ } else {
704
+ progressedChains$1 = progressedChains;
705
+ }
678
706
  state.rollback = {
679
- targetCheckpointId: targetCheckpointId,
680
707
  diffCheckpointId: diffCheckpointId,
708
+ floors: floors,
681
709
  rolledBackAddresses: rolledBackAddresses$1,
682
- progressBlockNumberByChainId: progressBlockNumberByChainId
710
+ progressedChains: progressedChains$1
683
711
  };
684
712
  }
685
713
 
@@ -766,7 +794,7 @@ export {
766
794
  markProcessingStalledOnFetch,
767
795
  recordProcessedBatch,
768
796
  createBatch,
769
- getSafeCheckpointId,
797
+ getSafeCheckpointIdByChain,
770
798
  config,
771
799
  persistence,
772
800
  allEntities,
@@ -812,6 +840,7 @@ export {
812
840
  recordRollbackSuccess,
813
841
  queueProcessedBatch,
814
842
  drainBatchRun,
843
+ pendingRollback,
815
844
  takeRollback,
816
845
  markCommitted,
817
846
  beginRollbackDiff,
@@ -1,7 +1,6 @@
1
1
  // The indexer state. `t` is opaque: other modules read it through the accessors
2
2
  // and change it only through the transitions and setters exposed here.
3
3
 
4
-
5
4
  type rollbackState =
6
5
  | NoRollback
7
6
  | ReorgDetected({chainId: ChainId.t, blockNumber: int})
@@ -67,7 +66,7 @@ let createBatch: (
67
66
  ~batchSizeTarget: int,
68
67
  ~isRollback: bool,
69
68
  ) => Batch.t
70
- let getSafeCheckpointId: t => option<Internal.checkpointId>
69
+ let getSafeCheckpointIdByChain: t => array<(ChainId.t, option<Internal.checkpointId>)>
71
70
 
72
71
  // Accessors.
73
72
  let config: t => Config.t
@@ -136,13 +135,14 @@ let recordRollbackSuccess: (t, ~timeSeconds: float, ~rollbackedProcessedEvents:
136
135
  // Store domain operations.
137
136
  let queueProcessedBatch: (t, ~batch: Batch.t) => unit
138
137
  let drainBatchRun: t => Batch.t
138
+ let pendingRollback: t => option<Persistence.rollback>
139
139
  let takeRollback: t => option<Persistence.rollback>
140
140
  let markCommitted: (t, ~upToCheckpointId: Internal.checkpointId) => unit
141
141
  let beginRollbackDiff: (
142
142
  t,
143
- ~targetCheckpointId: Internal.checkpointId,
144
143
  ~diffCheckpointId: Internal.checkpointId,
145
- ~progressBlockNumberByChainId: dict<int>,
144
+ ~floors: RollbackFloors.t,
145
+ ~progressedChains: array<InternalTable.Chains.progressedChain>,
146
146
  ~rolledBackAddresses: array<AddressRows.key>,
147
147
  ) => unit
148
148
  let recordWriteFailure: (t, exn) => unit
package/src/Internal.res CHANGED
@@ -573,15 +573,6 @@ type svmInstructionEventConfig = {
573
573
  programId: SvmTypes.Pubkey.t,
574
574
  /** Hex-encoded discriminator. `None` matches every instruction in the program. */
575
575
  discriminator: option<string>,
576
- /** Length of the discriminator in bytes (0 / 1 / 2 / 4 / 8). Drives the
577
- `dN` selector at query time and the dispatch-key precomputation in the
578
- router. */
579
- discriminatorByteLen: int,
580
- /** Disjunctive normal form: outer array is OR of AND-groups, inner array is
581
- AND across positions. Empty outer array means "no account filter". */
582
- accountFilters: array<svmAccountFilterGroup>,
583
- /** `None` matches both outer and inner (CPI-invoked) instructions. */
584
- isInner: option<bool>,
585
576
  /** Positional account names from the Borsh schema, in declared order.
586
577
  `[]` means no schema is attached for this instruction. */
587
578
  accounts: array<string>,
@@ -597,10 +588,11 @@ type svmInstructionEventConfig = {
597
588
  // Per-(event, chain) registration produced when user handler code registers an
598
589
  // event (`onEvent`) or a dynamic contract registers. References its definition
599
590
  // by value as `.eventConfig` and adds the handler binding plus the
600
- // registration/`where`-derived fetch state. Not `private`: Fuel/SVM
601
- // registrations add no ecosystem-specific fields (so they're bare aliases that
602
- // must stay directly constructable), and the evm→base cast in sources is sound
603
- // by ecosystem homogeneity — an EVM chain only ever holds `evmOnEventRegistration`s.
591
+ // registration/`where`-derived fetch state. Not `private`: Fuel registrations
592
+ // add no ecosystem-specific fields (so the alias must stay directly
593
+ // constructable), and the ecosystem→base casts in sources are sound by
594
+ // ecosystem homogeneity — an EVM chain only ever holds
595
+ // `evmOnEventRegistration`s.
604
596
  type onEventRegistration = {
605
597
  // Chain-scoped sequential index — the registration's position in the
606
598
  // chain's onEventRegistrations array, assigned when registration finishes
@@ -639,11 +631,18 @@ type evmOnEventRegistration = {
639
631
  resolvedWhere: resolvedWhere,
640
632
  }
641
633
 
642
- // Fuel and SVM registrations add no ecosystem-specific fetch state (their
643
- // filters are config-derived and live on the definition), so they're bare
644
- // aliases of the base registration.
634
+ // Fuel registrations add no ecosystem-specific fetch state, so it's a bare
635
+ // alias of the base registration.
645
636
  type fuelOnEventRegistration = onEventRegistration
646
- type svmOnEventRegistration = onEventRegistration
637
+
638
+ type svmOnEventRegistration = {
639
+ ...onEventRegistration,
640
+ /** Disjunctive normal form: outer array is OR of AND-groups, inner array is
641
+ AND across positions. Empty outer array means "no account filter". */
642
+ accountFilters: array<svmAccountFilterGroup>,
643
+ /** `None` matches both outer and inner (CPI-invoked) instructions. */
644
+ isInner: option<bool>,
645
+ }
647
646
 
648
647
  type svmProgramConfig = {
649
648
  name: string,
package/src/Metrics.res CHANGED
@@ -98,6 +98,13 @@ type sourceHeightMetrics = {
98
98
  height: int,
99
99
  }
100
100
 
101
+ type sourceHeightStreamMetrics = {
102
+ source: string,
103
+ chainId: ChainId.t,
104
+ connectCount: int,
105
+ disconnectsByReason: array<(string, int)>,
106
+ }
107
+
101
108
  type t = {
102
109
  startTime: Date.t,
103
110
  // Wall clock when this snapshot was built, so a scrape can be dated.
@@ -124,6 +131,7 @@ type t = {
124
131
  historyPrunes: array<historyPruneMetrics>,
125
132
  sourceRequests: array<sourceRequestMetrics>,
126
133
  sourceHeights: array<sourceHeightMetrics>,
134
+ sourceHeightStreams: array<sourceHeightStreamMetrics>,
127
135
  }
128
136
 
129
137
  // Prometheus floats keep at most 3 decimals; integral values render without a
@@ -220,13 +228,23 @@ let renderMetrics = (b: builder, metrics: t) => {
220
228
  metrics.storageWrites->Array.map(s => (`{storage="${s.storage->escapeLabelValue}"}`, s))
221
229
  let historyPrunes =
222
230
  metrics.historyPrunes->Array.map(s => (`{entity="${s.entity->escapeLabelValue}"}`, s))
231
+ // Every per-source series shares this label set: a scrape whose labels
232
+ // disagree between two of them is one nothing can join on.
233
+ let sourceLabels = (~source, ~chainId, ~extra: option<(string, string)>=?) => {
234
+ let pair = `{source="${source->escapeLabelValue}",chainId="${chainId->ChainId.toString}"`
235
+ switch extra {
236
+ | Some((name, value)) => `${pair},${name}="${value->escapeLabelValue}"}`
237
+ | None => pair ++ "}"
238
+ }
239
+ }
240
+
223
241
  // Two sources can share a name (e.g. primary and fallback RPC urls on the
224
242
  // same host), so aggregate by label set — duplicate samples would make
225
243
  // Prometheus reject the scrape.
226
244
  let sourceRequests = {
227
245
  let byLabels: dict<sourceRequestMetrics> = Dict.make()
228
246
  metrics.sourceRequests->Array.forEach(s => {
229
- let labels = `{source="${s.source->escapeLabelValue}",chainId="${s.chainId->ChainId.toString}",method="${s.method->escapeLabelValue}"}`
247
+ let labels = sourceLabels(~source=s.source, ~chainId=s.chainId, ~extra=("method", s.method))
230
248
  switch byLabels->Utils.Dict.dangerouslyGetNonOption(labels) {
231
249
  | Some(existing) =>
232
250
  byLabels->Dict.set(
@@ -241,7 +259,7 @@ let renderMetrics = (b: builder, metrics: t) => {
241
259
  let sources = {
242
260
  let byLabels: dict<int> = Dict.make()
243
261
  metrics.sourceHeights->Array.forEach(s => {
244
- let labels = `{source="${s.source->escapeLabelValue}",chainId="${s.chainId->ChainId.toString}"}`
262
+ let labels = sourceLabels(~source=s.source, ~chainId=s.chainId)
245
263
  switch byLabels->Utils.Dict.dangerouslyGetNonOption(labels) {
246
264
  | Some(existing) if existing >= s.height => ()
247
265
  | _ => byLabels->Dict.set(labels, s.height)
@@ -250,6 +268,34 @@ let renderMetrics = (b: builder, metrics: t) => {
250
268
  byLabels->Dict.toArray
251
269
  }
252
270
 
271
+ // Zero connects is the one count worth rendering flat: it is a stream that
272
+ // has never come up, which the disconnects say nothing about — retries at a
273
+ // connection that never opened are not disconnections — and which is
274
+ // otherwise indistinguishable from a chain that was never configured to
275
+ // stream at all.
276
+ let heightStreamConnects = {
277
+ let byLabels: dict<int> = Dict.make()
278
+ metrics.sourceHeightStreams->Array.forEach(s =>
279
+ byLabels->Utils.Dict.incrementBy(
280
+ sourceLabels(~source=s.source, ~chainId=s.chainId),
281
+ s.connectCount,
282
+ )
283
+ )
284
+ byLabels->Dict.toArray
285
+ }
286
+ let heightStreamDisconnects = {
287
+ let byLabels: dict<int> = Dict.make()
288
+ metrics.sourceHeightStreams->Array.forEach(s =>
289
+ s.disconnectsByReason->Array.forEach(((reason, count)) =>
290
+ byLabels->Utils.Dict.incrementBy(
291
+ sourceLabels(~source=s.source, ~chainId=s.chainId, ~extra=("reason", reason)),
292
+ count,
293
+ )
294
+ )
295
+ )
296
+ byLabels->Dict.toArray
297
+ }
298
+
253
299
  b->single(
254
300
  ~name="envio_process_start_time_seconds",
255
301
  ~help="Start time of the process since unix epoch in seconds.",
@@ -463,6 +509,24 @@ let renderMetrics = (b: builder, metrics: t) => {
463
509
  ~entries=sourceRequests,
464
510
  ~value=s => s.seconds !== 0. ? Some(s.seconds) : None,
465
511
  )
512
+ if heightStreamConnects->Array.length > 0 {
513
+ b->series(
514
+ ~name="envio_source_height_stream_connects_total",
515
+ ~help="The number of times a source's height subscription connected. Compare against the disconnects total, which is absent until the first disconnect and counts as zero while it is: one more connect than disconnects means the stream is up, and equal counts mean it is down and the indexer is polling instead. Zero connects means the stream has not come up, which is the normal reading for a chain that is still backfilling: subscriptions are only opened once a chain reaches the head.",
516
+ ~kind="counter",
517
+ ~entries=heightStreamConnects,
518
+ ~value=count => count->Int.toFloat,
519
+ )
520
+ }
521
+ if heightStreamDisconnects->Array.length > 0 {
522
+ b->series(
523
+ ~name="envio_source_height_stream_disconnects_total",
524
+ ~help="The number of times a source's height subscription lost a connection, by reason. Failed retries are not counted, so this is outages rather than their length. A rotated disconnect is a connection that served its time; unsubscribed is the source being benched; every other reason ended a connection early.",
525
+ ~kind="counter",
526
+ ~entries=heightStreamDisconnects,
527
+ ~value=count => count->Int.toFloat,
528
+ )
529
+ }
466
530
  b->series(
467
531
  ~name="envio_source_known_height",
468
532
  ~help="The latest known block number reported by the source. This value may lag behind the actual chain height, as it is updated only when queried.",
@@ -706,10 +770,12 @@ let renderMetrics = (b: builder, metrics: t) => {
706
770
  ~help="The number of address registrations per contract on chain, static and dynamic. An address shared by N contracts counts N times.",
707
771
  ~kind="gauge",
708
772
  ~entries=metrics.chains->Array.flatMap(m =>
709
- m.addressesByContract->Array.map(((contract, count)) => (
710
- `{chainId="${m.chainId->ChainId.toString}",contract="${contract->escapeLabelValue}"}`,
711
- count,
712
- ))
773
+ m.addressesByContract->Array.map(
774
+ ((contract, count)) => (
775
+ `{chainId="${m.chainId->ChainId.toString}",contract="${contract->escapeLabelValue}"}`,
776
+ count,
777
+ ),
778
+ )
713
779
  ),
714
780
  ~value=count => count->Int.toFloat,
715
781
  )