envio 3.3.0-alpha.1 → 3.3.0-alpha.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (84) hide show
  1. package/index.d.ts +23 -21
  2. package/package.json +6 -6
  3. package/src/Batch.res +0 -6
  4. package/src/Batch.res.mjs +0 -11
  5. package/src/BatchProcessing.res +2 -12
  6. package/src/BatchProcessing.res.mjs +2 -7
  7. package/src/ChainFetching.res +42 -6
  8. package/src/ChainFetching.res.mjs +30 -11
  9. package/src/ChainMetadata.res +1 -11
  10. package/src/ChainMetadata.res.mjs +1 -10
  11. package/src/ChainState.res +150 -1
  12. package/src/ChainState.res.mjs +138 -9
  13. package/src/ChainState.resi +51 -1
  14. package/src/Config.res +3 -6
  15. package/src/Config.res.mjs +2 -3
  16. package/src/Core.res +7 -0
  17. package/src/CrossChainState.res +24 -26
  18. package/src/CrossChainState.res.mjs +19 -16
  19. package/src/Ecosystem.res +6 -3
  20. package/src/Envio.res +3 -2
  21. package/src/EventConfigBuilder.res +61 -23
  22. package/src/EventConfigBuilder.res.mjs +27 -13
  23. package/src/EventProcessing.res +63 -21
  24. package/src/EventProcessing.res.mjs +47 -22
  25. package/src/FetchState.res +53 -43
  26. package/src/FetchState.res.mjs +71 -54
  27. package/src/HandlerLoader.res +10 -1
  28. package/src/HandlerLoader.res.mjs +15 -8
  29. package/src/Internal.res +46 -7
  30. package/src/Internal.res.mjs +19 -1
  31. package/src/LoadLayer.res +5 -5
  32. package/src/LoadLayer.res.mjs +6 -6
  33. package/src/Main.res +3 -50
  34. package/src/Main.res.mjs +2 -23
  35. package/src/PgStorage.res +4 -4
  36. package/src/PgStorage.res.mjs +5 -5
  37. package/src/Prometheus.res +10 -10
  38. package/src/Prometheus.res.mjs +9 -9
  39. package/src/PruneStaleHistory.res +2 -2
  40. package/src/PruneStaleHistory.res.mjs +3 -3
  41. package/src/Rollback.res +3 -3
  42. package/src/Rollback.res.mjs +4 -4
  43. package/src/SimulateItems.res +80 -15
  44. package/src/SimulateItems.res.mjs +65 -21
  45. package/src/TestIndexer.res +48 -36
  46. package/src/TestIndexer.res.mjs +35 -28
  47. package/src/bindings/Performance.res +7 -0
  48. package/src/bindings/Performance.res.mjs +21 -0
  49. package/src/bindings/Performance.resi +7 -0
  50. package/src/sources/EventRouter.res +17 -21
  51. package/src/sources/EventRouter.res.mjs +6 -9
  52. package/src/sources/Evm.res +47 -67
  53. package/src/sources/Evm.res.mjs +42 -65
  54. package/src/sources/Fuel.res +3 -3
  55. package/src/sources/Fuel.res.mjs +1 -6
  56. package/src/sources/HyperFuelSource.res +15 -11
  57. package/src/sources/HyperFuelSource.res.mjs +13 -11
  58. package/src/sources/HyperSync.res +7 -1
  59. package/src/sources/HyperSync.res.mjs +6 -3
  60. package/src/sources/HyperSync.resi +2 -0
  61. package/src/sources/HyperSyncClient.res +14 -79
  62. package/src/sources/HyperSyncClient.res.mjs +1 -22
  63. package/src/sources/HyperSyncSource.res +44 -27
  64. package/src/sources/HyperSyncSource.res.mjs +32 -27
  65. package/src/sources/RpcSource.res +14 -10
  66. package/src/sources/RpcSource.res.mjs +11 -9
  67. package/src/sources/SimulateSource.res +4 -2
  68. package/src/sources/SimulateSource.res.mjs +1 -0
  69. package/src/sources/Source.res +5 -1
  70. package/src/sources/SourceManager.res +17 -16
  71. package/src/sources/SourceManager.res.mjs +10 -15
  72. package/src/sources/Svm.res +23 -6
  73. package/src/sources/Svm.res.mjs +24 -6
  74. package/src/sources/SvmHyperSyncClient.res +3 -29
  75. package/src/sources/SvmHyperSyncSource.res +78 -96
  76. package/src/sources/SvmHyperSyncSource.res.mjs +79 -91
  77. package/src/sources/TransactionStore.res +128 -0
  78. package/src/sources/TransactionStore.res.mjs +97 -0
  79. package/src/tui/Tui.res +13 -16
  80. package/src/tui/Tui.res.mjs +12 -14
  81. package/svm.schema.json +30 -37
  82. package/src/bindings/Hrtime.res +0 -58
  83. package/src/bindings/Hrtime.res.mjs +0 -90
  84. package/src/bindings/Hrtime.resi +0 -30
package/index.d.ts CHANGED
@@ -997,24 +997,6 @@ export type SvmTokenBalance = {
997
997
  readonly postAmount?: string;
998
998
  };
999
999
 
1000
- /** Parent transaction surfaced when an instruction's
1001
- * `include_transaction` flag is `true`. */
1002
- export type SvmTransaction = {
1003
- readonly signatures: readonly string[];
1004
- readonly feePayer?: string;
1005
- readonly success?: boolean;
1006
- readonly err?: string;
1007
- /** Lamports. */
1008
- readonly fee?: bigint;
1009
- readonly computeUnitsConsumed?: bigint;
1010
- readonly accountKeys: readonly string[];
1011
- readonly recentBlockhash?: string;
1012
- readonly version?: string;
1013
- /** SPL Token / Token-2022 balance snapshots for this transaction.
1014
- * Present when `include_token_balances` is `true`. */
1015
- readonly tokenBalances?: readonly SvmTokenBalance[];
1016
- };
1017
-
1018
1000
  export type SvmLog = {
1019
1001
  readonly kind: string;
1020
1002
  readonly message: string;
@@ -1033,6 +1015,7 @@ export type SvmLog = {
1033
1015
  * are base58 strings. */
1034
1016
  export type SvmInstruction<
1035
1017
  Params extends SvmInstructionParams = SvmInstructionParams,
1018
+ Tx = SvmTransaction,
1036
1019
  > = {
1037
1020
  /** Program name as declared under `programs[].name` in `config.yaml`. */
1038
1021
  readonly programName: string;
@@ -1050,8 +1033,11 @@ export type SvmInstruction<
1050
1033
  readonly d8?: string;
1051
1034
  /** Borsh-decoded params. Present when a schema is configured and matched. */
1052
1035
  readonly params?: Params;
1053
- /** Present when the instruction's `include_transaction` is `true`. */
1054
- readonly transaction?: SvmTransaction;
1036
+ /** Parent transaction. Carries only the fields selected via this
1037
+ * instruction's `field_selection`; unselected fields are typed as
1038
+ * `FieldNotSelected<...>` so reading them is a compile error. Always present
1039
+ * (`{}` when no fields are selected). */
1040
+ readonly transaction: Tx;
1055
1041
  /** Present when the instruction's `include_logs` is `true`; only logs
1056
1042
  * scoped to this exact instruction (matching `instruction_address`). */
1057
1043
  readonly logs?: readonly SvmLog[];
@@ -1327,7 +1313,10 @@ type SvmEcosystem<Config extends IndexerConfigTypes = GlobalConfig> =
1327
1313
  handler: (
1328
1314
  args: SvmOnInstructionHandlerArgs<
1329
1315
  Config,
1330
- SvmInstruction<SvmParamsFromProgramTable<Programs[P][I]>>
1316
+ SvmInstruction<
1317
+ SvmParamsFromProgramTable<Programs[P][I]>,
1318
+ Programs[P][I]["transaction"]
1319
+ >
1331
1320
  >,
1332
1321
  ) => Promise<void>,
1333
1322
  ) => void;
@@ -1695,6 +1684,7 @@ type EvmContractsT = GlobalConfig extends { evm: { contracts: infer X extends
1695
1684
  type FuelChainsT = GlobalConfig extends { fuel: { chains: infer X extends Record<string, { id: number }> } } ? X : {};
1696
1685
  type FuelContractsT = GlobalConfig extends { fuel: { contracts: infer X extends Record<string, Record<string, any>> } } ? X : {};
1697
1686
  type SvmChainsT = GlobalConfig extends { svm: { chains: infer X extends Record<string, { id: number }> } } ? X : {};
1687
+ type SvmProgramsT = GlobalConfig extends { svm: { programs: infer X extends Record<string, Record<string, any>> } } ? X : {};
1698
1688
  type EntitiesT = GlobalConfig extends { entities: infer X extends Record<string, object> } ? X : {};
1699
1689
  type EnumsT = GlobalConfig extends { enums: infer X extends Record<string, any> } ? X : {};
1700
1690
 
@@ -1716,6 +1706,18 @@ export type FuelChainId = IsEmptyObject<FuelChainsT> extends true ? NotConfigure
1716
1706
  /** Union of all configured SVM chain IDs. */
1717
1707
  export type SvmChainId = IsEmptyObject<SvmChainsT> extends true ? NotConfigured<"SvmChainId", "Configure SVM chains"> : SvmChainsT [keyof SvmChainsT ]["id"];
1718
1708
 
1709
+ /** The SVM parent-transaction type generated from this project's
1710
+ * `field_selection`: the union of every instruction's `transaction` shape,
1711
+ * with unselected fields typed as `FieldNotSelected<...>`. Resolves to a
1712
+ * `NotConfigured` hint until `envio codegen` augments {@link Global}. */
1713
+ export type SvmTransaction = IsEmptyObject<SvmProgramsT> extends true
1714
+ ? NotConfigured<"SvmTransaction", "Configure SVM programs">
1715
+ : {
1716
+ [P in keyof SvmProgramsT]: {
1717
+ [I in keyof SvmProgramsT[P]]: SvmProgramsT[P][I]["transaction"];
1718
+ }[keyof SvmProgramsT[P]];
1719
+ }[keyof SvmProgramsT];
1720
+
1719
1721
  /** Lookup an EVM event type by contract and event name. Without generics,
1720
1722
  * resolves to the discriminated union of every EVM event in the project. */
1721
1723
  export type EvmEvent<
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "envio",
3
- "version": "3.3.0-alpha.1",
3
+ "version": "3.3.0-alpha.3",
4
4
  "type": "module",
5
5
  "description": "A latency and sync speed optimized, developer friendly blockchain data indexer.",
6
6
  "bin": "./bin.mjs",
@@ -69,10 +69,10 @@
69
69
  "tsx": "4.21.0"
70
70
  },
71
71
  "optionalDependencies": {
72
- "envio-linux-x64": "3.3.0-alpha.1",
73
- "envio-linux-x64-musl": "3.3.0-alpha.1",
74
- "envio-linux-arm64": "3.3.0-alpha.1",
75
- "envio-darwin-x64": "3.3.0-alpha.1",
76
- "envio-darwin-arm64": "3.3.0-alpha.1"
72
+ "envio-linux-x64": "3.3.0-alpha.3",
73
+ "envio-linux-x64-musl": "3.3.0-alpha.3",
74
+ "envio-linux-arm64": "3.3.0-alpha.3",
75
+ "envio-darwin-x64": "3.3.0-alpha.3",
76
+ "envio-darwin-arm64": "3.3.0-alpha.3"
77
77
  }
78
78
  }
package/src/Batch.res CHANGED
@@ -34,12 +34,6 @@ type t = {
34
34
  checkpointEventsProcessed: array<int>,
35
35
  }
36
36
 
37
- let hasReadyItem = (fetchStates: array<FetchState.t>) => {
38
- fetchStates->Array.some(fetchState => {
39
- fetchState->FetchState.isActivelyIndexing && fetchState->FetchState.hasReadyItem
40
- })
41
- }
42
-
43
37
  let getProgressedChainsById = {
44
38
  let getChainAfterBatchIfProgressed = (
45
39
  ~chainBeforeBatch: chainBeforeBatch,
package/src/Batch.res.mjs CHANGED
@@ -4,16 +4,6 @@ import * as Utils from "./Utils.res.mjs";
4
4
  import * as FetchState from "./FetchState.res.mjs";
5
5
  import * as ReorgDetection from "./ReorgDetection.res.mjs";
6
6
 
7
- function hasReadyItem(fetchStates) {
8
- return fetchStates.some(fetchState => {
9
- if (FetchState.isActivelyIndexing(fetchState)) {
10
- return FetchState.hasReadyItem(fetchState);
11
- } else {
12
- return false;
13
- }
14
- });
15
- }
16
-
17
7
  function getChainAfterBatchIfProgressed(chainBeforeBatch, progressBlockNumberAfterBatch, fetchStateAfterBatch, batchSize) {
18
8
  if (chainBeforeBatch.progressBlockNumber < progressBlockNumberAfterBatch) {
19
9
  return {
@@ -208,7 +198,6 @@ function findLastEventItem(batch, chainId) {
208
198
  }
209
199
 
210
200
  export {
211
- hasReadyItem,
212
201
  getProgressedChainsById,
213
202
  addReorgCheckpoints,
214
203
  prepareBatch,
@@ -77,15 +77,7 @@ and processNextBatch = async (state: IndexerState.t, ~scheduleFetch): unit => {
77
77
  state
78
78
  ->IndexerState.chainStates
79
79
  ->Dict.valuesToArray
80
- ->Array.every(cs => {
81
- let fetchState = switch progressedChainsById->Utils.Dict.dangerouslyGetByIntNonOption(
82
- (cs->ChainState.fetchState).chainId,
83
- ) {
84
- | Some(chainAfterBatch) => chainAfterBatch.fetchState
85
- | None => cs->ChainState.fetchState
86
- }
87
- fetchState->FetchState.isReadyToEnterReorgThreshold
88
- })
80
+ ->Array.every(cs => cs->ChainState.isReadyToEnterReorgThresholdAfterBatch(~batch))
89
81
 
90
82
  if shouldEnterReorgThreshold {
91
83
  IndexerState.enterReorgThreshold(state)
@@ -171,9 +163,7 @@ and processNextBatch = async (state: IndexerState.t, ~scheduleFetch): unit => {
171
163
  state
172
164
  ->IndexerState.chainStates
173
165
  ->Dict.valuesToArray
174
- ->Array.every(cs =>
175
- cs->ChainState.isProgressAtHead && (cs->ChainState.fetchState).endBlock->Option.isNone
176
- )
166
+ ->Array.every(ChainState.isAtHeadWithoutEndBlock)
177
167
  ) {
178
168
  IndexerState.errorExit(
179
169
  state,
@@ -4,12 +4,10 @@ import * as Utils from "./Utils.res.mjs";
4
4
  import * as Config from "./Config.res.mjs";
5
5
  import * as Logging from "./Logging.res.mjs";
6
6
  import * as ChainState from "./ChainState.res.mjs";
7
- import * as FetchState from "./FetchState.res.mjs";
8
7
  import * as IndexerState from "./IndexerState.res.mjs";
9
8
  import * as ChainMetadata from "./ChainMetadata.res.mjs";
10
9
  import * as ErrorHandling from "./ErrorHandling.res.mjs";
11
10
  import * as InMemoryStore from "./InMemoryStore.res.mjs";
12
- import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js";
13
11
  import * as ExitOnCaughtUp from "./ExitOnCaughtUp.res.mjs";
14
12
  import * as EventProcessing from "./EventProcessing.res.mjs";
15
13
  import * as PruneStaleHistory from "./PruneStaleHistory.res.mjs";
@@ -30,10 +28,7 @@ async function processNextBatch(state, scheduleFetch) {
30
28
  let batch = IndexerState.createBatch(state, IndexerState.processedCheckpointId(state), IndexerState.config(state).batchSize, isRollbackBatch);
31
29
  let progressedChainsById = batch.progressedChainsById;
32
30
  let isBelowReorgThreshold = !isInReorgThresholdBeforeUpdate && IndexerState.config(state).shouldRollbackOnReorg;
33
- let shouldEnterReorgThreshold = isBelowReorgThreshold && Object.values(IndexerState.chainStates(state)).every(cs => {
34
- let chainAfterBatch = progressedChainsById[ChainState.fetchState(cs).chainId];
35
- return FetchState.isReadyToEnterReorgThreshold(chainAfterBatch !== undefined ? chainAfterBatch.fetchState : ChainState.fetchState(cs));
36
- });
31
+ let shouldEnterReorgThreshold = isBelowReorgThreshold && Object.values(IndexerState.chainStates(state)).every(cs => ChainState.isReadyToEnterReorgThresholdAfterBatch(cs, batch));
37
32
  if (shouldEnterReorgThreshold) {
38
33
  IndexerState.enterReorgThreshold(state);
39
34
  }
@@ -75,7 +70,7 @@ async function processNextBatch(state, scheduleFetch) {
75
70
  }
76
71
  if (allCaughtUp && !IndexerState.keepProcessAlive(state)) {
77
72
  return await ExitOnCaughtUp.run(state);
78
- } else if (!allCaughtUp && IndexerState.exitAfterFirstEventBlock(state) && Object.values(IndexerState.chainStates(state)).every(cs => ChainState.isProgressAtHead(cs) ? Stdlib_Option.isNone(ChainState.fetchState(cs).endBlock) : false)) {
73
+ } else if (!allCaughtUp && IndexerState.exitAfterFirstEventBlock(state) && Object.values(IndexerState.chainStates(state)).every(ChainState.isAtHeadWithoutEndBlock)) {
79
74
  return IndexerState.errorExit(state, ErrorHandling.make(new Error("No events found between startBlock and chain head. Cannot auto-detect endBlock."), undefined, undefined));
80
75
  } else {
81
76
  ChainMetadata.stage(state);
@@ -11,7 +11,14 @@ type partitionQueryResponse = {
11
11
  let runContractRegistersOrThrow = async (
12
12
  ~itemsWithContractRegister: array<Internal.item>,
13
13
  ~config: Config.t,
14
+ ~chainState: ChainState.t,
15
+ ~page: option<TransactionStore.t>,
14
16
  ) => {
17
+ // contractRegister handlers can read event.transaction, so materialise the
18
+ // selected fields onto the payloads before running them. All items belong to
19
+ // the chain being fetched, hence its single page store and mask.
20
+ await chainState->ChainState.materializePageItems(~items=itemsWithContractRegister, ~page)
21
+
15
22
  let itemsWithDcs = []
16
23
 
17
24
  let onRegister = (~item: Internal.item, ~contractAddress, ~contractName) => {
@@ -106,6 +113,7 @@ let rec onQueryResponse = async (
106
113
  let chainState = state->IndexerState.getChainState(~chain)
107
114
  let {
108
115
  parsedQueueItems,
116
+ transactionStore,
109
117
  latestFetchedBlockNumber,
110
118
  latestFetchedBlockTimestamp,
111
119
  stats,
@@ -114,7 +122,7 @@ let rec onQueryResponse = async (
114
122
  fromBlockQueried,
115
123
  } = response
116
124
 
117
- if knownHeight > (chainState->ChainState.fetchState).knownHeight {
125
+ if knownHeight > chainState->ChainState.knownHeight {
118
126
  Prometheus.SourceHeight.set(
119
127
  ~blockNumber=knownHeight,
120
128
  ~chainId=(chainState->ChainState.chainConfig).id,
@@ -212,6 +220,7 @@ let rec onQueryResponse = async (
212
220
  blockTimestamp: latestFetchedBlockTimestamp,
213
221
  },
214
222
  ~query,
223
+ ~transactionStore,
215
224
  )
216
225
  ChainMetadata.stage(state)
217
226
  scheduleFetch()
@@ -224,6 +233,8 @@ let rec onQueryResponse = async (
224
233
  switch await runContractRegistersOrThrow(
225
234
  ~itemsWithContractRegister,
226
235
  ~config=state->IndexerState.config,
236
+ ~chainState,
237
+ ~page=transactionStore,
227
238
  ) {
228
239
  | exception exn => IndexerState.errorExit(state, exn->ErrorHandling.make)
229
240
  | newItemsWithDcs => proceed(~newItemsWithDcs)
@@ -240,6 +251,7 @@ and applyQueryResponse = (
240
251
  ~knownHeight,
241
252
  ~latestFetchedBlock,
242
253
  ~query,
254
+ ~transactionStore,
243
255
  ) => {
244
256
  let chainState = state->IndexerState.getChainState(~chain)
245
257
  let wasFetchingAtHead = chainState->ChainState.isFetchingAtHead
@@ -250,6 +262,7 @@ and applyQueryResponse = (
250
262
  ~newItems,
251
263
  ~newItemsWithDcs,
252
264
  ~knownHeight,
265
+ ~transactionStore,
253
266
  )
254
267
 
255
268
  // In auto-exit mode, set endBlock to the first event's block when events arrive.
@@ -294,7 +307,7 @@ let finishWaitingForNewBlock = (
294
307
  ->IndexerState.chainStates
295
308
  ->Dict.valuesToArray
296
309
  ->Array.every(cs => {
297
- cs->ChainState.fetchState->FetchState.isReadyToEnterReorgThreshold
310
+ cs->ChainState.isReadyToEnterReorgThreshold
298
311
  })
299
312
 
300
313
  // Kick processing in case there are block handlers to run.
@@ -316,7 +329,6 @@ let fetchChain = async (
316
329
  ) => {
317
330
  let chainState = state->IndexerState.getChainState(~chain)
318
331
  if !(state->IndexerState.isResolvingReorg) && !(state->IndexerState.isStopped) {
319
- let fetchState = chainState->ChainState.fetchState
320
332
  let isRealtime = state->IndexerState.isRealtime
321
333
  let sourceManager = chainState->ChainState.sourceManager
322
334
 
@@ -324,11 +336,19 @@ let fetchChain = async (
324
336
  // there's nothing to fetch. During backfill any such chain is idle.
325
337
  let reducedPolling = !isRealtime
326
338
 
339
+ // Accumulated across all queries dispatched in this tick so their per-query
340
+ // results collapse into a single completion log instead of one per query.
341
+ let dispatchedCount = switch action {
342
+ | FetchState.Ready(queries) => queries->Array.length
343
+ | WaitingForNewBlock | NothingToQuery => 0
344
+ }
345
+ let fetchedByPartition = Dict.make()
346
+ let fetchedCount = ref(0)
347
+
327
348
  // Owns its error boundary: launch doesn't catch, so any failure here (the
328
349
  // query, response handling, or dispatch itself) must stop the indexer.
329
350
  try {
330
- await sourceManager->SourceManager.dispatch(
331
- ~fetchState,
351
+ await chainState->ChainState.dispatch(
332
352
  ~waitForNewBlock=(~knownHeight) =>
333
353
  sourceManager->SourceManager.waitForNewBlock(~knownHeight, ~isRealtime, ~reducedPolling),
334
354
  ~onNewBlock=(~knownHeight) =>
@@ -347,9 +367,25 @@ let fetchChain = async (
347
367
  try {
348
368
  let response = await sourceManager->SourceManager.executeQuery(
349
369
  ~query,
350
- ~knownHeight=fetchState.knownHeight,
370
+ ~knownHeight=chainState->ChainState.knownHeight,
351
371
  ~isRealtime,
352
372
  )
373
+ fetchedCount := fetchedCount.contents + 1
374
+ fetchedByPartition->Dict.set(
375
+ query.partitionId,
376
+ {
377
+ "fromBlock": response.fromBlockQueried,
378
+ "toBlock": response.latestFetchedBlockNumber,
379
+ "numEvents": response.parsedQueueItems->Array.length,
380
+ },
381
+ )
382
+ if dispatchedCount > 0 && fetchedCount.contents === dispatchedCount {
383
+ Logging.trace({
384
+ "msg": "Finished querying",
385
+ "chainId": chain->ChainMap.Chain.toChainId,
386
+ "partitions": fetchedByPartition,
387
+ })
388
+ }
353
389
  await onQueryResponse(
354
390
  state,
355
391
  {chain, response, query},
@@ -4,7 +4,6 @@ import * as Utils from "./Utils.res.mjs";
4
4
  import * as Logging from "./Logging.res.mjs";
5
5
  import * as Ecosystem from "./Ecosystem.res.mjs";
6
6
  import * as ChainState from "./ChainState.res.mjs";
7
- import * as FetchState from "./FetchState.res.mjs";
8
7
  import * as Prometheus from "./Prometheus.res.mjs";
9
8
  import * as IndexerState from "./IndexerState.res.mjs";
10
9
  import * as ChainMetadata from "./ChainMetadata.res.mjs";
@@ -17,7 +16,8 @@ import * as Stdlib_Promise from "@rescript/runtime/lib/es6/Stdlib_Promise.js";
17
16
  import * as Primitive_exceptions from "@rescript/runtime/lib/es6/Primitive_exceptions.js";
18
17
  import * as ContractRegisterContext from "./ContractRegisterContext.res.mjs";
19
18
 
20
- async function runContractRegistersOrThrow(itemsWithContractRegister, config) {
19
+ async function runContractRegistersOrThrow(itemsWithContractRegister, config, chainState, page) {
20
+ await ChainState.materializePageItems(chainState, itemsWithContractRegister, page);
21
21
  let itemsWithDcs = [];
22
22
  let onRegister = (item, contractAddress, contractName) => {
23
23
  let dc_registrationBlock = item.blockNumber;
@@ -71,10 +71,10 @@ async function runContractRegistersOrThrow(itemsWithContractRegister, config) {
71
71
  return itemsWithDcs;
72
72
  }
73
73
 
74
- function applyQueryResponse(state, chain, newItems, newItemsWithDcs, knownHeight, latestFetchedBlock, query) {
74
+ function applyQueryResponse(state, chain, newItems, newItemsWithDcs, knownHeight, latestFetchedBlock, query, transactionStore) {
75
75
  let chainState = IndexerState.getChainState(state, chain);
76
76
  let wasFetchingAtHead = ChainState.isFetchingAtHead(chainState);
77
- ChainState.handleQueryResult(chainState, query, newItems, newItemsWithDcs, latestFetchedBlock, knownHeight);
77
+ ChainState.handleQueryResult(chainState, query, newItems, newItemsWithDcs, latestFetchedBlock, knownHeight, transactionStore);
78
78
  if (IndexerState.exitAfterFirstEventBlock(state) && newItems.length !== 0) {
79
79
  ChainState.setEndBlockToFirstEvent(chainState, newItems[0].blockNumber);
80
80
  }
@@ -94,9 +94,10 @@ async function onQueryResponse(state, param, stateId, scheduleFetch, schedulePro
94
94
  let stats = response.stats;
95
95
  let latestFetchedBlockTimestamp = response.latestFetchedBlockTimestamp;
96
96
  let latestFetchedBlockNumber = response.latestFetchedBlockNumber;
97
+ let transactionStore = response.transactionStore;
97
98
  let parsedQueueItems = response.parsedQueueItems;
98
99
  let knownHeight = response.knownHeight;
99
- if (knownHeight > ChainState.fetchState(chainState).knownHeight) {
100
+ if (knownHeight > ChainState.knownHeight(chainState)) {
100
101
  Prometheus.SourceHeight.set(SourceManager.getActiveSource(ChainState.sourceManager(chainState)).name, ChainState.chainConfig(chainState).id, knownHeight);
101
102
  }
102
103
  Prometheus.FetchingBlockRange.increment(chain, stats["total time elapsed (s)"], Stdlib_Option.getOr(stats["parsing time (s)"], 0), parsedQueueItems.length, (latestFetchedBlockNumber - response.fromBlockQueried | 0) + 1 | 0);
@@ -133,7 +134,7 @@ async function onQueryResponse(state, param, stateId, scheduleFetch, schedulePro
133
134
  applyQueryResponse(state, chain, newItems, newItemsWithDcs, knownHeight, {
134
135
  blockNumber: latestFetchedBlockNumber,
135
136
  blockTimestamp: latestFetchedBlockTimestamp
136
- }, query);
137
+ }, query, transactionStore);
137
138
  ChainMetadata.stage(state);
138
139
  scheduleFetch();
139
140
  return scheduleProcessing();
@@ -144,7 +145,7 @@ async function onQueryResponse(state, param, stateId, scheduleFetch, schedulePro
144
145
  }
145
146
  let newItemsWithDcs;
146
147
  try {
147
- newItemsWithDcs = await runContractRegistersOrThrow(itemsWithContractRegister, IndexerState.config(state));
148
+ newItemsWithDcs = await runContractRegistersOrThrow(itemsWithContractRegister, IndexerState.config(state), chainState, transactionStore);
148
149
  } catch (raw_exn) {
149
150
  let exn = Primitive_exceptions.internalToException(raw_exn);
150
151
  return IndexerState.errorExit(state, ErrorHandling.make(exn, undefined, undefined));
@@ -159,7 +160,7 @@ function finishWaitingForNewBlock(state, chain, knownHeight, stateId, scheduleFe
159
160
  let chainState = IndexerState.getChainState(state, chain);
160
161
  ChainState.updateKnownHeight(chainState, knownHeight);
161
162
  let isBelowReorgThreshold = !IndexerState.isInReorgThreshold(state) && IndexerState.config(state).shouldRollbackOnReorg;
162
- let shouldEnterReorgThreshold = isBelowReorgThreshold && Object.values(IndexerState.chainStates(state)).every(cs => FetchState.isReadyToEnterReorgThreshold(ChainState.fetchState(cs)));
163
+ let shouldEnterReorgThreshold = isBelowReorgThreshold && Object.values(IndexerState.chainStates(state)).every(ChainState.isReadyToEnterReorgThreshold);
163
164
  if (shouldEnterReorgThreshold) {
164
165
  IndexerState.enterReorgThreshold(state);
165
166
  }
@@ -172,14 +173,32 @@ async function fetchChain(state, chain, action, stateId, scheduleFetch, schedule
172
173
  if (IndexerState.isResolvingReorg(state) || IndexerState.isStopped(state)) {
173
174
  return;
174
175
  }
175
- let fetchState = ChainState.fetchState(chainState);
176
176
  let isRealtime = IndexerState.isRealtime(state);
177
177
  let sourceManager = ChainState.sourceManager(chainState);
178
178
  let reducedPolling = !isRealtime;
179
+ let dispatchedCount;
180
+ dispatchedCount = typeof action !== "object" ? 0 : action._0.length;
181
+ let fetchedByPartition = {};
182
+ let fetchedCount = {
183
+ contents: 0
184
+ };
179
185
  try {
180
- return await SourceManager.dispatch(sourceManager, fetchState, async query => {
186
+ return await ChainState.dispatch(chainState, async query => {
181
187
  try {
182
- let response = await SourceManager.executeQuery(sourceManager, query, fetchState.knownHeight, isRealtime);
188
+ let response = await SourceManager.executeQuery(sourceManager, query, ChainState.knownHeight(chainState), isRealtime);
189
+ fetchedCount.contents = fetchedCount.contents + 1 | 0;
190
+ fetchedByPartition[query.partitionId] = {
191
+ fromBlock: response.fromBlockQueried,
192
+ toBlock: response.latestFetchedBlockNumber,
193
+ numEvents: response.parsedQueueItems.length
194
+ };
195
+ if (dispatchedCount > 0 && fetchedCount.contents === dispatchedCount) {
196
+ Logging.trace({
197
+ msg: "Finished querying",
198
+ chainId: chain,
199
+ partitions: fetchedByPartition
200
+ });
201
+ }
183
202
  return await onQueryResponse(state, {
184
203
  chain: chain,
185
204
  response: response,
@@ -7,19 +7,9 @@ let stage = (state: IndexerState.t) => {
7
7
  ->IndexerState.chainStates
8
8
  ->Dict.valuesToArray
9
9
  ->Array.forEach(cs => {
10
- let fetchState = cs->ChainState.fetchState
11
10
  chainsData->Dict.set(
12
11
  (cs->ChainState.chainConfig).id->Int.toString,
13
- {
14
- firstEventBlockNumber: fetchState.firstEventBlock->Null.fromOption,
15
- isHyperSync: (
16
- cs->ChainState.sourceManager->SourceManager.getActiveSource
17
- ).poweredByHyperSync,
18
- latestFetchedBlockNumber: fetchState->FetchState.bufferBlockNumber,
19
- timestampCaughtUpToHeadOrEndblock: cs
20
- ->ChainState.timestampCaughtUpToHeadOrEndblock
21
- ->Null.fromOption,
22
- },
12
+ cs->ChainState.toChainMetadata,
23
13
  )
24
14
  })
25
15
 
@@ -2,21 +2,12 @@
2
2
 
3
3
  import * as Writing from "./Writing.res.mjs";
4
4
  import * as ChainState from "./ChainState.res.mjs";
5
- import * as FetchState from "./FetchState.res.mjs";
6
- import * as Stdlib_Null from "@rescript/runtime/lib/es6/Stdlib_Null.js";
7
5
  import * as IndexerState from "./IndexerState.res.mjs";
8
- import * as SourceManager from "./sources/SourceManager.res.mjs";
9
6
 
10
7
  function stage(state) {
11
8
  let chainsData = {};
12
9
  Object.values(IndexerState.chainStates(state)).forEach(cs => {
13
- let fetchState = ChainState.fetchState(cs);
14
- chainsData[ChainState.chainConfig(cs).id.toString()] = {
15
- first_event_block: Stdlib_Null.fromOption(fetchState.firstEventBlock),
16
- buffer_block: FetchState.bufferBlockNumber(fetchState),
17
- ready_at: Stdlib_Null.fromOption(ChainState.timestampCaughtUpToHeadOrEndblock(cs)),
18
- _is_hyper_sync: SourceManager.getActiveSource(ChainState.sourceManager(cs)).poweredByHyperSync
19
- };
10
+ chainsData[ChainState.chainConfig(cs).id.toString()] = ChainState.toChainMetadata(cs);
20
11
  });
21
12
  Writing.setChainMeta(state, chainsData);
22
13
  }