envio 3.3.0-alpha.7 → 3.3.0-alpha.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (95) hide show
  1. package/index.d.ts +23 -5
  2. package/package.json +6 -6
  3. package/src/Api.res +1 -1
  4. package/src/Api.res.mjs +1 -1
  5. package/src/ChainFetching.res +21 -10
  6. package/src/ChainFetching.res.mjs +11 -10
  7. package/src/ChainState.res +401 -171
  8. package/src/ChainState.res.mjs +270 -104
  9. package/src/ChainState.resi +19 -7
  10. package/src/Config.res +11 -27
  11. package/src/Config.res.mjs +8 -7
  12. package/src/Core.res +7 -0
  13. package/src/CrossChainState.res +82 -82
  14. package/src/CrossChainState.res.mjs +52 -59
  15. package/src/CrossChainState.resi +1 -1
  16. package/src/Ecosystem.res +3 -3
  17. package/src/Ecosystem.res.mjs +3 -3
  18. package/src/Envio.res +14 -9
  19. package/src/EnvioGlobal.res +53 -0
  20. package/src/EnvioGlobal.res.mjs +31 -0
  21. package/src/EventConfigBuilder.res +117 -69
  22. package/src/EventConfigBuilder.res.mjs +71 -34
  23. package/src/EventProcessing.res +19 -15
  24. package/src/EventProcessing.res.mjs +13 -12
  25. package/src/FetchState.res +410 -185
  26. package/src/FetchState.res.mjs +268 -253
  27. package/src/HandlerLoader.res +8 -113
  28. package/src/HandlerLoader.res.mjs +2 -88
  29. package/src/HandlerRegister.res +518 -144
  30. package/src/HandlerRegister.res.mjs +285 -133
  31. package/src/HandlerRegister.resi +24 -8
  32. package/src/IndexerState.res +6 -3
  33. package/src/IndexerState.res.mjs +3 -3
  34. package/src/IndexerState.resi +1 -1
  35. package/src/IndexingAddresses.res +10 -7
  36. package/src/IndexingAddresses.res.mjs +8 -7
  37. package/src/IndexingAddresses.resi +3 -1
  38. package/src/Internal.res +104 -39
  39. package/src/Internal.res.mjs +11 -1
  40. package/src/LogSelection.res +102 -165
  41. package/src/LogSelection.res.mjs +101 -116
  42. package/src/Main.res +49 -164
  43. package/src/Main.res.mjs +39 -104
  44. package/src/Metrics.res +43 -2
  45. package/src/Metrics.res.mjs +39 -3
  46. package/src/Prometheus.res +0 -35
  47. package/src/Prometheus.res.mjs +33 -68
  48. package/src/RawEvent.res +7 -2
  49. package/src/RawEvent.res.mjs +4 -4
  50. package/src/RollbackCommit.res +4 -1
  51. package/src/RollbackCommit.res.mjs +3 -2
  52. package/src/SimulateItems.res +20 -7
  53. package/src/SimulateItems.res.mjs +7 -11
  54. package/src/TestIndexer.res +1 -1
  55. package/src/TestIndexer.res.mjs +1 -1
  56. package/src/sources/BlockStore.res +46 -0
  57. package/src/sources/BlockStore.res.mjs +24 -0
  58. package/src/sources/EventRouter.res +19 -12
  59. package/src/sources/EventRouter.res.mjs +7 -5
  60. package/src/sources/Evm.res +58 -5
  61. package/src/sources/Evm.res.mjs +44 -5
  62. package/src/sources/EvmChain.res +14 -18
  63. package/src/sources/EvmChain.res.mjs +12 -13
  64. package/src/sources/FieldMask.res +39 -0
  65. package/src/sources/FieldMask.res.mjs +42 -0
  66. package/src/sources/Fuel.res +8 -3
  67. package/src/sources/Fuel.res.mjs +5 -4
  68. package/src/sources/HyperFuelSource.res +31 -40
  69. package/src/sources/HyperFuelSource.res.mjs +52 -44
  70. package/src/sources/HyperSync.res +31 -9
  71. package/src/sources/HyperSync.res.mjs +47 -31
  72. package/src/sources/HyperSync.resi +10 -3
  73. package/src/sources/HyperSyncClient.res +15 -4
  74. package/src/sources/HyperSyncHeightStream.res +1 -8
  75. package/src/sources/HyperSyncHeightStream.res.mjs +0 -2
  76. package/src/sources/HyperSyncSource.res +58 -90
  77. package/src/sources/HyperSyncSource.res.mjs +57 -66
  78. package/src/sources/RpcSource.res +118 -120
  79. package/src/sources/RpcSource.res.mjs +92 -74
  80. package/src/sources/RpcWebSocketHeightStream.res +0 -5
  81. package/src/sources/RpcWebSocketHeightStream.res.mjs +0 -2
  82. package/src/sources/SimulateSource.res +5 -3
  83. package/src/sources/SimulateSource.res.mjs +12 -4
  84. package/src/sources/Source.res +19 -5
  85. package/src/sources/SourceManager.res +60 -10
  86. package/src/sources/SourceManager.res.mjs +54 -8
  87. package/src/sources/SourceManager.resi +10 -0
  88. package/src/sources/Svm.res +14 -10
  89. package/src/sources/Svm.res.mjs +23 -6
  90. package/src/sources/SvmHyperSyncClient.res +5 -6
  91. package/src/sources/SvmHyperSyncSource.res +82 -41
  92. package/src/sources/SvmHyperSyncSource.res.mjs +86 -41
  93. package/src/sources/TransactionStore.res +6 -107
  94. package/src/sources/TransactionStore.res.mjs +5 -86
  95. package/svm.schema.json +19 -0
package/src/Config.res CHANGED
@@ -198,6 +198,7 @@ let svmEventDescriptorSchema = S.schema(s =>
198
198
  "discriminator": s.matches(S.option(S.string)),
199
199
  "discriminatorByteLen": s.matches(S.int),
200
200
  "transactionFields": s.matches(S.array(Internal.svmTransactionFieldSchema)),
201
+ "blockFields": s.matches(S.option(S.array(Internal.svmBlockFieldSchema))),
201
202
  "includeLogs": s.matches(S.bool),
202
203
  "accountFilters": s.matches(
203
204
  S.option(
@@ -654,7 +655,6 @@ let fromPublic = (publicConfigJson: JSON.t) => {
654
655
  ~events: option<array<_>>,
655
656
  ~abi,
656
657
  ~chainId: int,
657
- ~startBlock: option<int>,
658
658
  ~addresses: array<string>,
659
659
  ~svmDefinedTypes: JSON.t=JSON.Null,
660
660
  ) => {
@@ -677,10 +677,6 @@ let fromPublic = (publicConfigJson: JSON.t) => {
677
677
  ~kind=fuelKind,
678
678
  ~sighash,
679
679
  ~rawAbi=abi->(Utils.magic: EvmTypes.Abi.t => JSON.t),
680
- ~isWildcard=false,
681
- ~handler=None,
682
- ~contractRegister=None,
683
- ~startBlock?,
684
680
  ) :> Internal.eventConfig)
685
681
  | None =>
686
682
  JsError.throwWithMessage(
@@ -706,6 +702,7 @@ let fromPublic = (publicConfigJson: JSON.t) => {
706
702
  "discriminator": option<string>,
707
703
  "discriminatorByteLen": int,
708
704
  "transactionFields": array<Internal.svmTransactionField>,
705
+ "blockFields": option<array<Internal.svmBlockField>>,
709
706
  "includeLogs": bool,
710
707
  "accountFilters": option<
711
708
  array<array<{"position": int, "values": array<string>}>>,
@@ -741,16 +738,13 @@ let fromPublic = (publicConfigJson: JSON.t) => {
741
738
  ~discriminator=svm["discriminator"],
742
739
  ~discriminatorByteLen=svm["discriminatorByteLen"],
743
740
  ~transactionFields=svm["transactionFields"],
741
+ ~blockFields=?svm["blockFields"],
744
742
  ~includeLogs=svm["includeLogs"],
745
743
  ~accountFilters,
746
744
  ~isInner=svm["isInner"],
747
- ~isWildcard=false,
748
- ~handler=None,
749
- ~contractRegister=None,
750
745
  ~accounts=svm["accounts"]->Option.getOr([]),
751
746
  ~args=svm["args"]->Option.getOr(JSON.Null),
752
747
  ~definedTypes=svmDefinedTypes,
753
- ~startBlock?,
754
748
  ) :> Internal.eventConfig)
755
749
  | _ =>
756
750
  (EventConfigBuilder.buildEvmEventConfig(
@@ -758,15 +752,8 @@ let fromPublic = (publicConfigJson: JSON.t) => {
758
752
  ~eventName,
759
753
  ~sighash,
760
754
  ~params,
761
- ~isWildcard=false,
762
- ~handler=None,
763
- ~contractRegister=None,
764
- ~eventFilters=None,
765
- ~probeChainId=chainId,
766
- ~onEventBlockFilterSchema=ecosystem.onEventBlockFilterSchema,
767
755
  ~blockFields=?eventItem["blockFields"],
768
756
  ~transactionFields=?eventItem["transactionFields"],
769
- ~startBlock?,
770
757
  ~globalBlockFieldsSet,
771
758
  ~globalTransactionFieldsSet,
772
759
  ) :> Internal.eventConfig)
@@ -818,17 +805,14 @@ let fromPublic = (publicConfigJson: JSON.t) => {
818
805
  let addresses = rawAddresses->Array.map(parseAddress)
819
806
  let startBlock = chainContract->Option.flatMap(cc => cc["startBlock"])
820
807
 
821
- // Build event configs from JSON (field selections resolved inline)
822
- // chainId is threaded in so the where-callback detection probe
823
- // exercises the callback with this chain's real id — handlers
824
- // that branch on `chain.id` are taken through the same path
825
- // they will follow at runtime.
808
+ // Build event definitions from JSON (field selections resolved
809
+ // inline). Handlers and per-chain `where` filters are layered on
810
+ // later as `onEventRegistration`s in `ChainState.makeInternal`.
826
811
  let events = buildContractEvents(
827
812
  ~contractName=capitalizedName,
828
813
  ~events=contractData["events"],
829
814
  ~abi=contractData["abi"],
830
815
  ~chainId,
831
- ~startBlock,
832
816
  ~addresses=rawAddresses,
833
817
  ~svmDefinedTypes=contractData["svmAbi"]
834
818
  ->Option.map(a => a["definedTypes"])
@@ -1303,11 +1287,11 @@ let throwIfIncompatible = (
1303
1287
  }
1304
1288
  }
1305
1289
 
1306
- // The returned value is a pure function of the JSON no handler
1307
- // registrations are applied here. Post-registration configs come from
1308
- // `HandlerLoader.applyRegistrations`. That purity is what lets this
1309
- // memoize without invalidation.
1310
- let loadWithoutRegistrations = () =>
1290
+ // The returned value is a pure function of the JSON: it holds only event
1291
+ // definitions, never handler/contractRegister/where state (that's layered on
1292
+ // separately as `HandlerRegister.registrationsByChainId`). That purity is
1293
+ // what lets this memoize without invalidation.
1294
+ let load = () =>
1311
1295
  switch cached.contents {
1312
1296
  | Some(c) => c
1313
1297
  | None => {
@@ -113,6 +113,7 @@ let svmEventDescriptorSchema = S$RescriptSchema.schema(s => ({
113
113
  discriminator: s.m(S$RescriptSchema.option(S$RescriptSchema.string)),
114
114
  discriminatorByteLen: s.m(S$RescriptSchema.int),
115
115
  transactionFields: s.m(S$RescriptSchema.array(Internal.svmTransactionFieldSchema)),
116
+ blockFields: s.m(S$RescriptSchema.option(S$RescriptSchema.array(Internal.svmBlockFieldSchema))),
116
117
  includeLogs: s.m(S$RescriptSchema.bool),
117
118
  accountFilters: s.m(S$RescriptSchema.option(S$RescriptSchema.array(S$RescriptSchema.schema(s => ({
118
119
  position: s.m(S$RescriptSchema.int),
@@ -490,7 +491,7 @@ function fromPublic(publicConfigJson) {
490
491
  };
491
492
  });
492
493
  }
493
- let buildContractEvents = (contractName, events, abi, chainId, startBlock, addresses, svmDefinedTypesOpt) => {
494
+ let buildContractEvents = (contractName, events, abi, chainId, addresses, svmDefinedTypesOpt) => {
494
495
  let svmDefinedTypes = svmDefinedTypesOpt !== undefined ? svmDefinedTypesOpt : null;
495
496
  if (events !== undefined) {
496
497
  return events.map(eventItem => {
@@ -500,10 +501,10 @@ function fromPublic(publicConfigJson) {
500
501
  let kind = eventItem.kind;
501
502
  switch (ecosystemName) {
502
503
  case "evm" :
503
- return EventConfigBuilder.buildEvmEventConfig(contractName, eventName, sighash, params, false, undefined, undefined, undefined, chainId, ecosystem.onEventBlockFilterSchema, eventItem.blockFields, eventItem.transactionFields, startBlock, Primitive_option.some(globalBlockFieldsSet), Primitive_option.some(globalTransactionFieldsSet));
504
+ return EventConfigBuilder.buildEvmEventConfig(contractName, eventName, sighash, params, eventItem.blockFields, eventItem.transactionFields, Primitive_option.some(globalBlockFieldsSet), Primitive_option.some(globalTransactionFieldsSet));
504
505
  case "fuel" :
505
506
  if (kind !== undefined) {
506
- return EventConfigBuilder.buildFuelEventConfig(contractName, eventName, kind, sighash, abi, false, undefined, undefined, startBlock);
507
+ return EventConfigBuilder.buildFuelEventConfig(contractName, eventName, kind, sighash, abi);
507
508
  } else {
508
509
  return Stdlib_JsError.throwWithMessage(`Fuel event ` + contractName + `.` + eventName + ` is missing "kind" in internal config`);
509
510
  }
@@ -518,7 +519,7 @@ function fromPublic(publicConfigJson) {
518
519
  position: af.position,
519
520
  values: af.values
520
521
  })));
521
- return EventConfigBuilder.buildSvmInstructionEventConfig(contractName, eventName, programId, svm.discriminator, svm.discriminatorByteLen, svm.includeLogs, svm.transactionFields, accountFilters, svm.isInner, false, undefined, undefined, Stdlib_Option.getOr(svm.accounts, []), Stdlib_Option.getOr(svm.args, null), svmDefinedTypes, startBlock);
522
+ return EventConfigBuilder.buildSvmInstructionEventConfig(contractName, eventName, programId, svm.discriminator, svm.discriminatorByteLen, svm.includeLogs, svm.transactionFields, svm.blockFields, accountFilters, svm.isInner, Stdlib_Option.getOr(svm.accounts, []), Stdlib_Option.getOr(svm.args, null), svmDefinedTypes);
522
523
  }
523
524
  });
524
525
  } else {
@@ -559,7 +560,7 @@ function fromPublic(publicConfigJson) {
559
560
  let rawAddresses = Stdlib_Option.getOr(Stdlib_Option.flatMap(chainContract, cc => cc.addresses), []);
560
561
  let addresses = rawAddresses.map(parseAddress);
561
562
  let startBlock = Stdlib_Option.flatMap(chainContract, cc => cc.startBlock);
562
- let events = buildContractEvents(capitalizedName, contractData.events, contractData.abi, chainId, startBlock, rawAddresses, Stdlib_Option.getOr(Stdlib_Option.map(contractData.svmAbi, a => a.definedTypes), null));
563
+ let events = buildContractEvents(capitalizedName, contractData.events, contractData.abi, chainId, rawAddresses, Stdlib_Option.getOr(Stdlib_Option.map(contractData.svmAbi, a => a.definedTypes), null));
563
564
  return {
564
565
  name: capitalizedName,
565
566
  abi: contractData.abi,
@@ -1011,7 +1012,7 @@ function throwIfIncompatible(changedPaths, resetCommand, runCommand, hasClickhou
1011
1012
  Stdlib_JsError.throwWithMessage(`The following config changes are incompatible with the existing indexer data:\n\n` + bullets + `\n\nPick one:\n 1. ` + padTo(option1, col) + `# resume indexing where it left off\n 2. ` + padTo(resetCommand, col) + `# delete all indexed data and start over` + option3);
1012
1013
  }
1013
1014
 
1014
- function loadWithoutRegistrations() {
1015
+ function load() {
1015
1016
  let c = cached.contents;
1016
1017
  if (c !== undefined) {
1017
1018
  return c;
@@ -1060,7 +1061,7 @@ export {
1060
1061
  canonicalJson,
1061
1062
  diffPaths,
1062
1063
  throwIfIncompatible,
1063
- loadWithoutRegistrations,
1064
+ load,
1064
1065
  getPgUserEntities,
1065
1066
  }
1066
1067
  /* schema Not a pure module */
package/src/Core.res CHANGED
@@ -9,6 +9,7 @@ type evmRpcClientCtor
9
9
  type svmHypersyncClientCtor
10
10
  type hyperfuelClientCtor
11
11
  type transactionStoreCtor
12
+ type blockStoreCtor
12
13
 
13
14
  type addon = {
14
15
  getConfigJson: (~configPath: Null.t<string>, ~directory: Null.t<string>) => string,
@@ -23,10 +24,16 @@ type addon = {
23
24
  hyperfuelClient: hyperfuelClientCtor,
24
25
  @as("TransactionStore")
25
26
  transactionStore: transactionStoreCtor,
27
+ @as("BlockStore")
28
+ blockStore: blockStoreCtor,
26
29
  // Ordered transaction-field names exposed for the field-code contract test
27
30
  // (the ReScript `transactionFields` arrays must match the Rust ordinals).
28
31
  evmTransactionFieldNames: unit => array<string>,
29
32
  svmTransactionFieldNames: unit => array<string>,
33
+ // Ordered block-field names for the same contract test (`blockFields` arrays
34
+ // must match the Rust ordinals).
35
+ evmBlockFieldNames: unit => array<string>,
36
+ svmBlockFieldNames: unit => array<string>,
30
37
  }
31
38
 
32
39
  @module("node:module") external createRequire: string => {..} = "createRequire"
@@ -135,13 +135,13 @@ let enterReorgThreshold = (crossChainState: t) => {
135
135
  // whole indexer. A chain is marked caught up only once EVERY chain is caught up
136
136
  // (reached endblock or fetched/processed to head) with no processable events
137
137
  // left — so no chain flips to ready while another is still backfilling.
138
- let applyBatchProgress = (crossChainState: t, ~batch: Batch.t) => {
138
+ let applyBatchProgress = (crossChainState: t, ~batch: Batch.t, ~blockTimestampName: string) => {
139
139
  let chainIds = crossChainState.chainIds
140
140
 
141
141
  let everyChainCaughtUp = ref(true)
142
142
  for i in 0 to chainIds->Array.length - 1 {
143
143
  let cs = crossChainState->getChainState(chainIds->Array.getUnsafe(i))
144
- cs->ChainState.applyBatchProgress(~batch)
144
+ cs->ChainState.applyBatchProgress(~batch, ~blockTimestampName)
145
145
  if !(cs->ChainState.hasProcessedToEndblock || cs->ChainState.isProgressAtHead) {
146
146
  everyChainCaughtUp := false
147
147
  }
@@ -190,98 +190,98 @@ let totalReservedSize = (crossChainState: t) => {
190
190
  total.contents
191
191
  }
192
192
 
193
- // Furthest-behind first: order candidate queries by the chain progress % at
194
- // their fromBlock, so the most behind ranges across all chains are admitted
195
- // before the rest.
196
- let compareByProgress = (a: FetchState.query, b: FetchState.query) =>
197
- Float.compare(a.progress, b.progress)
198
-
199
193
  // Dispatch a fetch tick across the whole indexer from one shared pool of
200
- // ~targetBufferSize ready events. Every chain proposes its candidate queries
201
- // (each carrying an estimated response size) up to its own natural ceiling
202
- // (head/endBlock/mergeBlock), unconstrained by the shared budget; the
203
- // candidates are then pooled, ordered by chain progress (furthest-behind first),
204
- // and admitted until the budget is consumed. So the budget is split per query
205
- // across chains rather than per chain a chain that can only use a little
206
- // leaves the rest for the others automatically.
194
+ // ~targetBufferSize ready events, as a waterfall: visit chains furthest-behind
195
+ // first, hand each the budget remaining at that point (plus its own
196
+ // already-reserved share, since a chain's pending queries aren't "spent" —
197
+ // they're this chain's), let it turn that into queries sized against its own
198
+ // chain-density-derived target block, then subtract what it actually used
199
+ // before moving to the next chain. So a chain that can only use a little
200
+ // (density too low, or already caught up) leaves the rest for the others
201
+ // automatically. A chain visited after the budget ran out simply doesn't
202
+ // query this round — the leader's reservations release as its responses land,
203
+ // so the next tick redistributes. Chains that do get budget are additionally
204
+ // capped at the most-behind chain's target progress mapped onto their own
205
+ // range, so no chain runs further ahead than the chain the pool is
206
+ // prioritizing.
207
207
  let checkAndFetch = async (
208
208
  crossChainState: t,
209
209
  ~dispatchChain: (~chain: ChainMap.Chain.t, ~action: FetchState.nextQuery) => promise<unit>,
210
210
  ) => {
211
- let remaining = Pervasives.max(
212
- 0,
213
- crossChainState.targetBufferSize -
214
- crossChainState->totalReadyCount -
215
- crossChainState->totalReservedSize->Float.toInt,
211
+ let remaining = ref(
212
+ Pervasives.max(
213
+ 0.,
214
+ crossChainState.targetBufferSize->Int.toFloat -.
215
+ crossChainState->totalReadyCount->Int.toFloat -.
216
+ crossChainState->totalReservedSize,
217
+ ),
216
218
  )
217
219
 
218
- let chainIds = crossChainState.chainIds
219
220
  let actionByChain = Dict.make()
220
- // Candidate queries from every chain. Each query carries its chain id and the
221
- // chain progress % at its fromBlock (the admission sort key), set here so the
222
- // pool can be ordered without a side tuple per query.
223
- let candidates = []
224
- for i in 0 to chainIds->Array.length - 1 {
225
- let chainId = chainIds->Array.getUnsafe(i)
226
- let cs = crossChainState->getChainState(chainId)
227
- switch cs->ChainState.getNextQuery {
228
- | (WaitingForNewBlock | NothingToQuery) as action =>
229
- actionByChain->Utils.Dict.setByInt(chainId, action)
230
- | Ready(queries) =>
231
- // Default to NothingToQuery; replaced below if any candidate is admitted.
232
- actionByChain->Utils.Dict.setByInt(chainId, FetchState.NothingToQuery)
233
- queries->Array.forEach(query => {
234
- query.chainId = chainId
235
- query.progress = cs->ChainState.getProgressPercentageAt(~blockNumber=query.fromBlock)
236
- candidates->Array.push(query)
237
- })
221
+ let maxProgress = ref(None)
222
+ crossChainState
223
+ ->priorityOrder
224
+ ->Array.forEach(cs => {
225
+ let chainId = (cs->ChainState.chainConfig).id
226
+ if cs->ChainState.knownHeight == 0 {
227
+ // No height yet — nothing to size a budget or an alignment line
228
+ // against. Skip without consuming budget or claiming leadership, so a
229
+ // chain whose source hasn't reported doesn't unconstrain everyone else.
230
+ actionByChain->Utils.Dict.setByInt(chainId, FetchState.WaitingForNewBlock)
231
+ } else {
232
+ let chainTargetItems = remaining.contents +. cs->ChainState.pendingBudget
233
+ let maxTargetBlock = switch maxProgress.contents {
234
+ | None =>
235
+ // The most-behind chain sets the alignment line for everyone after it.
236
+ maxProgress :=
237
+ Some(
238
+ cs->ChainState.progressAtBlock(
239
+ ~blockNumber=cs->ChainState.targetBlock(~chainTargetItems),
240
+ ),
241
+ )
242
+ None
243
+ | Some(progress) => Some(cs->ChainState.blockAtProgress(~progress))
244
+ }
245
+ switch cs->ChainState.getNextQuery(~chainTargetItems, ~maxTargetBlock?) {
246
+ | (WaitingForNewBlock | NothingToQuery) as action =>
247
+ actionByChain->Utils.Dict.setByInt(chainId, action)
248
+ | Ready(queries) => {
249
+ let consumed =
250
+ queries->Array.reduce(0., (acc, query: FetchState.query) =>
251
+ acc +. query.itemsTarget->Int.toFloat
252
+ )
253
+
254
+ let partitions = Dict.make()
255
+ queries->Array.forEach((query: FetchState.query) =>
256
+ partitions->Dict.set(
257
+ query.partitionId,
258
+ {
259
+ "fromBlock": query.fromBlock,
260
+ "targetBlock": query.toBlock,
261
+ "targetEvents": query.itemsTarget,
262
+ },
263
+ )
264
+ )
265
+ Logging.trace({
266
+ "msg": "Started querying",
267
+ "chainId": chainId,
268
+ "partitions": partitions,
269
+ })
270
+
271
+ actionByChain->Utils.Dict.setByInt(chainId, FetchState.Ready(queries))
272
+ // Mark the queries in flight and reserve their size against the
273
+ // shared budget; released as each response lands in
274
+ // handleQueryResult.
275
+ cs->ChainState.startFetchingQueries(~queries)
276
+ remaining := Pervasives.max(0., remaining.contents -. consumed)
277
+ }
278
+ }
238
279
  }
239
- }
240
-
241
- candidates->Array.sort(compareByProgress)
242
-
243
- // Admit furthest-behind first until the budget runs out. The condition is
244
- // checked before each query, so as long as there's any budget left we admit
245
- // the next one even when its estimate alone exceeds the remainder — otherwise a
246
- // chain whose only query is bigger than the budget would never make progress.
247
- let admittedByChain = Dict.make()
248
- let running = ref(0.)
249
- let remainingF = remaining->Int.toFloat
250
- let idx = ref(0)
251
- while running.contents < remainingF && idx.contents < candidates->Array.length {
252
- let query = candidates->Array.getUnsafe(idx.contents)
253
- admittedByChain->Utils.Dict.push(query.chainId->Int.toString, query)
254
- running := running.contents +. query.estResponseSize
255
- idx := idx.contents + 1
256
- }
257
- admittedByChain->Dict.forEachWithKey((queries, chainId) => {
258
- let partitions = Dict.make()
259
- queries->Array.forEach((query: FetchState.query) =>
260
- partitions->Dict.set(
261
- query.partitionId,
262
- {
263
- "fromBlock": query.fromBlock,
264
- "targetBlock": query.toBlock,
265
- "targetEvents": query.estResponseSize->Math.round->Float.toInt,
266
- },
267
- )
268
- )
269
- Logging.trace({
270
- "msg": "Started querying",
271
- "chainId": chainId->Int.fromString->Option.getUnsafe,
272
- "partitions": partitions,
273
- })
274
- actionByChain->Dict.set(chainId, FetchState.Ready(queries))
275
- // Mark the admitted queries in flight and reserve their size against the
276
- // shared budget; released as each response lands in handleQueryResult.
277
- crossChainState
278
- ->getChainState(chainId->Int.fromString->Option.getUnsafe)
279
- ->ChainState.startFetchingQueries(~queries)
280
280
  })
281
281
 
282
282
  let promises = []
283
- for i in 0 to chainIds->Array.length - 1 {
284
- let chainId = chainIds->Array.getUnsafe(i)
283
+ for i in 0 to crossChainState.chainIds->Array.length - 1 {
284
+ let chainId = crossChainState.chainIds->Array.getUnsafe(i)
285
285
  switch actionByChain->Utils.Dict.dangerouslyGetByIntNonOption(chainId) {
286
286
  | Some(NothingToQuery)
287
287
  | None => ()
@@ -7,9 +7,7 @@ import * as Logging from "./Logging.res.mjs";
7
7
  import * as ChainMap from "./ChainMap.res.mjs";
8
8
  import * as ChainState from "./ChainState.res.mjs";
9
9
  import * as Prometheus from "./Prometheus.res.mjs";
10
- import * as Stdlib_Int from "@rescript/runtime/lib/es6/Stdlib_Int.js";
11
- import * as Stdlib_Dict from "@rescript/runtime/lib/es6/Stdlib_Dict.js";
12
- import * as Primitive_int from "@rescript/runtime/lib/es6/Primitive_int.js";
10
+ import * as Stdlib_Array from "@rescript/runtime/lib/es6/Stdlib_Array.js";
13
11
  import * as Primitive_float from "@rescript/runtime/lib/es6/Primitive_float.js";
14
12
  import * as SafeCheckpointTracking from "./SafeCheckpointTracking.res.mjs";
15
13
 
@@ -95,12 +93,12 @@ function enterReorgThreshold(crossChainState) {
95
93
  crossChainState.isInReorgThreshold = true;
96
94
  }
97
95
 
98
- function applyBatchProgress(crossChainState, batch) {
96
+ function applyBatchProgress(crossChainState, batch, blockTimestampName) {
99
97
  let chainIds = crossChainState.chainIds;
100
98
  let everyChainCaughtUp = true;
101
99
  for (let i = 0, i_finish = chainIds.length; i < i_finish; ++i) {
102
100
  let cs = crossChainState.chainStates[chainIds[i]];
103
- ChainState.applyBatchProgress(cs, batch);
101
+ ChainState.applyBatchProgress(cs, batch, blockTimestampName);
104
102
  if (!(ChainState.hasProcessedToEndblock(cs) || ChainState.isProgressAtHead(cs))) {
105
103
  everyChainCaughtUp = false;
106
104
  }
@@ -135,73 +133,68 @@ function totalReservedSize(crossChainState) {
135
133
  return total;
136
134
  }
137
135
 
138
- function compareByProgress(a, b) {
139
- return Primitive_float.compare(a.progress, b.progress);
140
- }
141
-
142
136
  async function checkAndFetch(crossChainState, dispatchChain) {
143
- let remaining = Primitive_int.max(0, (crossChainState.targetBufferSize - totalReadyCount(crossChainState) | 0) - (totalReservedSize(crossChainState) | 0) | 0);
144
- let chainIds = crossChainState.chainIds;
137
+ let remaining = {
138
+ contents: Primitive_float.max(0, crossChainState.targetBufferSize - totalReadyCount(crossChainState) - totalReservedSize(crossChainState))
139
+ };
145
140
  let actionByChain = {};
146
- let candidates = [];
147
- for (let i = 0, i_finish = chainIds.length; i < i_finish; ++i) {
148
- let chainId = chainIds[i];
149
- let cs = crossChainState.chainStates[chainId];
150
- let action = ChainState.getNextQuery(cs);
141
+ let maxProgress = {
142
+ contents: undefined
143
+ };
144
+ priorityOrder(crossChainState).forEach(cs => {
145
+ let chainId = ChainState.chainConfig(cs).id;
146
+ if (ChainState.knownHeight(cs) === 0) {
147
+ actionByChain[chainId] = "WaitingForNewBlock";
148
+ return;
149
+ }
150
+ let chainTargetItems = remaining.contents + ChainState.pendingBudget(cs);
151
+ let progress = maxProgress.contents;
152
+ let maxTargetBlock = progress !== undefined ? ChainState.blockAtProgress(cs, progress) : (maxProgress.contents = ChainState.progressAtBlock(cs, ChainState.targetBlock(cs, chainTargetItems)), undefined);
153
+ let action = ChainState.getNextQuery(cs, chainTargetItems, maxTargetBlock);
151
154
  if (typeof action !== "object") {
155
+ if (action === "WaitingForNewBlock") {
156
+ actionByChain[chainId] = action;
157
+ return;
158
+ }
152
159
  actionByChain[chainId] = action;
160
+ return;
153
161
  } else {
154
- actionByChain[chainId] = "NothingToQuery";
155
- action._0.forEach(query => {
156
- query.chainId = chainId;
157
- query.progress = ChainState.getProgressPercentageAt(cs, query.fromBlock);
158
- candidates.push(query);
162
+ let queries = action._0;
163
+ let consumed = Stdlib_Array.reduce(queries, 0, (acc, query) => acc + query.itemsTarget);
164
+ let partitions = {};
165
+ queries.forEach(query => {
166
+ partitions[query.partitionId] = {
167
+ fromBlock: query.fromBlock,
168
+ targetBlock: query.toBlock,
169
+ targetEvents: query.itemsTarget
170
+ };
159
171
  });
160
- }
161
- }
162
- candidates.sort(compareByProgress);
163
- let admittedByChain = {};
164
- let running = 0;
165
- let idx = 0;
166
- while (running < remaining && idx < candidates.length) {
167
- let query = candidates[idx];
168
- Utils.Dict.push(admittedByChain, query.chainId.toString(), query);
169
- running = running + query.estResponseSize;
170
- idx = idx + 1 | 0;
171
- };
172
- Stdlib_Dict.forEachWithKey(admittedByChain, (queries, chainId) => {
173
- let partitions = {};
174
- queries.forEach(query => {
175
- partitions[query.partitionId] = {
176
- fromBlock: query.fromBlock,
177
- targetBlock: query.toBlock,
178
- targetEvents: Math.round(query.estResponseSize) | 0
172
+ Logging.trace({
173
+ msg: "Started querying",
174
+ chainId: chainId,
175
+ partitions: partitions
176
+ });
177
+ actionByChain[chainId] = {
178
+ TAG: "Ready",
179
+ _0: queries
179
180
  };
180
- });
181
- Logging.trace({
182
- msg: "Started querying",
183
- chainId: Stdlib_Int.fromString(chainId, undefined),
184
- partitions: partitions
185
- });
186
- actionByChain[chainId] = {
187
- TAG: "Ready",
188
- _0: queries
189
- };
190
- let chainId$1 = Stdlib_Int.fromString(chainId, undefined);
191
- ChainState.startFetchingQueries(crossChainState.chainStates[chainId$1], queries);
181
+ ChainState.startFetchingQueries(cs, queries);
182
+ remaining.contents = Primitive_float.max(0, remaining.contents - consumed);
183
+ return;
184
+ }
192
185
  });
193
186
  let promises = [];
194
- for (let i$1 = 0, i_finish$1 = chainIds.length; i$1 < i_finish$1; ++i$1) {
195
- let chainId$1 = chainIds[i$1];
196
- let action$1 = actionByChain[chainId$1];
197
- if (action$1 !== undefined) {
187
+ for (let i = 0, i_finish = crossChainState.chainIds.length; i < i_finish; ++i) {
188
+ let chainId = crossChainState.chainIds[i];
189
+ let action = actionByChain[chainId];
190
+ if (action !== undefined) {
198
191
  let exit = 0;
199
- if (typeof action$1 === "object" || action$1 === "WaitingForNewBlock") {
192
+ if (typeof action === "object" || action === "WaitingForNewBlock") {
200
193
  exit = 1;
201
194
  }
202
195
  if (exit === 1) {
203
- let chain = ChainMap.Chain.makeUnsafe(chainId$1);
204
- promises.push(dispatchChain(chain, action$1));
196
+ let chain = ChainMap.Chain.makeUnsafe(chainId);
197
+ promises.push(dispatchChain(chain, action));
205
198
  }
206
199
  }
207
200
  }
@@ -29,7 +29,7 @@ let createBatch: (
29
29
  ~isRollback: bool,
30
30
  ) => Batch.t
31
31
  let enterReorgThreshold: t => unit
32
- let applyBatchProgress: (t, ~batch: Batch.t) => unit
32
+ let applyBatchProgress: (t, ~batch: Batch.t, ~blockTimestampName: string) => unit
33
33
 
34
34
  // Fetch control.
35
35
  let priorityOrder: t => array<ChainState.t>
package/src/Ecosystem.res CHANGED
@@ -69,12 +69,12 @@ let getItemLogger = {
69
69
  | None =>
70
70
  let logger = switch item {
71
71
  | Internal.Event(_) => ecosystem.toEventLogger(item->Internal.castUnsafeEventItem)
72
- | Block({blockNumber, onBlockConfig}) =>
72
+ | Block({blockNumber, onBlockRegistration}) =>
73
73
  Logging.createChildFrom(
74
74
  ~logger=ecosystem.logger,
75
75
  ~params={
76
- "onBlock": onBlockConfig.name,
77
- "chainId": onBlockConfig.chainId,
76
+ "onBlock": onBlockRegistration.name,
77
+ "chainId": onBlockRegistration.chainId,
78
78
  "block": blockNumber,
79
79
  },
80
80
  )
@@ -26,10 +26,10 @@ function getItemLogger(item, ecosystem) {
26
26
  if (item.kind === 0) {
27
27
  logger$1 = ecosystem.toEventLogger(item);
28
28
  } else {
29
- let onBlockConfig = item.onBlockConfig;
29
+ let onBlockRegistration = item.onBlockRegistration;
30
30
  logger$1 = Logging.createChildFrom(ecosystem.logger, {
31
- onBlock: onBlockConfig.name,
32
- chainId: onBlockConfig.chainId,
31
+ onBlock: onBlockRegistration.name,
32
+ chainId: onBlockRegistration.chainId,
33
33
  block: item.blockNumber
34
34
  });
35
35
  }
package/src/Envio.res CHANGED
@@ -66,18 +66,21 @@ type svmLog = {
66
66
  message: string,
67
67
  }
68
68
 
69
- /** Block context for a matched instruction. `time`/`hash` follow the
70
- EVM/Fuel field names so the shared `Ecosystem.t` getters in `Svm.res` read
71
- them uniformly. */
69
+ /** Block context for a matched instruction. `slot`/`hash` are always present;
70
+ `time` can still be absent HyperSync/Solana may not report a block time for
71
+ every slot. Every other field is opt-in via `field_selection.block_fields`,
72
+ materialised from the per-chain block store at batch prep. */
72
73
  type svmInstructionBlock = {
73
74
  /** Slot this instruction's block was matched in. */
74
75
  slot: int,
75
- /** Unix block time (seconds). `0` when HyperSync didn't return a block
76
- for this instruction's slot. */
77
- time: int,
78
- /** Block hash. Currently always empty — populated by the future
79
- reorg-guard `queryBlockHash(slot)` route. */
76
+ /** Unix block time (seconds). */
77
+ time?: int,
80
78
  hash: string,
79
+ /** Block height (distinct from slot). Absent when not selected via
80
+ `field_selection.block_fields`. */
81
+ height?: int,
82
+ parentSlot?: int,
83
+ parentHash?: string,
81
84
  }
82
85
 
83
86
  /** The per-instruction payload handlers receive as their `instruction`
@@ -111,7 +114,9 @@ type svmInstruction = {
111
114
  /** Program log entries scoped to this instruction. Absent when the
112
115
  per-instruction `include_logs` flag is `false`. */
113
116
  logs?: array<svmLog>,
114
- block: svmInstructionBlock,
117
+ // Omitted on construction; materialised onto the payload at batch prep,
118
+ // before a handler ever reads it.
119
+ block?: svmInstructionBlock,
115
120
  }
116
121
 
117
122
  /** Arguments passed to handlers registered via `indexer.onInstruction`. */