envio 3.2.0 → 3.3.0-alpha.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (113) hide show
  1. package/package.json +6 -7
  2. package/src/Batch.res +12 -15
  3. package/src/Batch.res.mjs +4 -5
  4. package/src/BatchProcessing.res +199 -0
  5. package/src/BatchProcessing.res.mjs +125 -0
  6. package/src/ChainFetching.res +375 -0
  7. package/src/ChainFetching.res.mjs +206 -0
  8. package/src/ChainMetadata.res +27 -0
  9. package/src/ChainMetadata.res.mjs +27 -0
  10. package/src/ChainState.res +618 -0
  11. package/src/ChainState.res.mjs +461 -0
  12. package/src/ChainState.resi +85 -0
  13. package/src/Config.res +13 -4
  14. package/src/Config.res.mjs +8 -3
  15. package/src/ContractRegisterContext.res +106 -0
  16. package/src/ContractRegisterContext.res.mjs +76 -0
  17. package/src/Core.res +3 -0
  18. package/src/CrossChainState.res +269 -0
  19. package/src/CrossChainState.res.mjs +197 -0
  20. package/src/CrossChainState.resi +47 -0
  21. package/src/Ecosystem.res +58 -0
  22. package/src/Ecosystem.res.mjs +43 -1
  23. package/src/Env.res +7 -2
  24. package/src/Env.res.mjs +5 -2
  25. package/src/EventConfigBuilder.res +35 -0
  26. package/src/EventConfigBuilder.res.mjs +21 -0
  27. package/src/EventProcessing.res +60 -46
  28. package/src/EventProcessing.res.mjs +33 -32
  29. package/src/EventUtils.res +0 -4
  30. package/src/EventUtils.res.mjs +0 -4
  31. package/src/ExitOnCaughtUp.res +10 -0
  32. package/src/ExitOnCaughtUp.res.mjs +22 -0
  33. package/src/FetchState.res +162 -113
  34. package/src/FetchState.res.mjs +181 -119
  35. package/src/HandlerLoader.res +6 -1
  36. package/src/HandlerLoader.res.mjs +13 -36
  37. package/src/InMemoryStore.res +36 -451
  38. package/src/InMemoryStore.res.mjs +20 -369
  39. package/src/IndexerLoop.res +44 -0
  40. package/src/IndexerLoop.res.mjs +46 -0
  41. package/src/IndexerState.res +544 -0
  42. package/src/IndexerState.res.mjs +547 -0
  43. package/src/IndexerState.resi +132 -0
  44. package/src/Internal.res +44 -7
  45. package/src/LoadLayer.res +20 -18
  46. package/src/LoadLayer.res.mjs +14 -12
  47. package/src/LoadLayer.resi +6 -3
  48. package/src/LogSelection.res +86 -17
  49. package/src/LogSelection.res.mjs +67 -14
  50. package/src/Logging.res +11 -44
  51. package/src/Logging.res.mjs +10 -42
  52. package/src/Main.res +54 -79
  53. package/src/Main.res.mjs +44 -56
  54. package/src/PgStorage.res +8 -71
  55. package/src/PgStorage.res.mjs +3 -47
  56. package/src/Prometheus.res +7 -8
  57. package/src/Prometheus.res.mjs +7 -4
  58. package/src/PruneStaleHistory.res +45 -0
  59. package/src/PruneStaleHistory.res.mjs +46 -0
  60. package/src/RawEvent.res +73 -0
  61. package/src/RawEvent.res.mjs +51 -0
  62. package/src/Rollback.res +204 -0
  63. package/src/Rollback.res.mjs +118 -0
  64. package/src/SimulateItems.res +12 -10
  65. package/src/SimulateItems.res.mjs +1 -1
  66. package/src/TestIndexer.res +34 -27
  67. package/src/TestIndexer.res.mjs +4 -1
  68. package/src/UserContext.res +26 -114
  69. package/src/UserContext.res.mjs +15 -78
  70. package/src/Writing.res +242 -0
  71. package/src/Writing.res.mjs +215 -0
  72. package/src/db/InternalTable.res +14 -27
  73. package/src/sources/Evm.res +40 -1
  74. package/src/sources/Evm.res.mjs +94 -84
  75. package/src/sources/Fuel.res +40 -1
  76. package/src/sources/Fuel.res.mjs +36 -26
  77. package/src/sources/HyperFuel.res +41 -120
  78. package/src/sources/HyperFuel.res.mjs +42 -80
  79. package/src/sources/HyperFuel.resi +1 -24
  80. package/src/sources/HyperFuelClient.res +16 -297
  81. package/src/sources/HyperFuelClient.res.mjs +5 -4
  82. package/src/sources/HyperFuelSource.res +33 -8
  83. package/src/sources/HyperFuelSource.res.mjs +56 -10
  84. package/src/sources/HyperSyncSource.res +7 -5
  85. package/src/sources/HyperSyncSource.res.mjs +3 -3
  86. package/src/sources/RpcSource.res +2 -2
  87. package/src/sources/RpcSource.res.mjs +1 -1
  88. package/src/sources/SourceManager.res +8 -18
  89. package/src/sources/SourceManager.res.mjs +8 -5
  90. package/src/sources/SourceManager.resi +4 -1
  91. package/src/sources/Svm.res +18 -1
  92. package/src/sources/Svm.res.mjs +30 -22
  93. package/src/sources/SvmHyperSyncSource.res +1 -1
  94. package/src/sources/SvmHyperSyncSource.res.mjs +1 -1
  95. package/src/tui/Tui.res +38 -36
  96. package/src/tui/Tui.res.mjs +51 -51
  97. package/src/ChainFetcher.res +0 -519
  98. package/src/ChainFetcher.res.mjs +0 -314
  99. package/src/ChainManager.res +0 -358
  100. package/src/ChainManager.res.mjs +0 -291
  101. package/src/Ctx.res +0 -6
  102. package/src/Ctx.res.mjs +0 -2
  103. package/src/GlobalState.res +0 -1056
  104. package/src/GlobalState.res.mjs +0 -1086
  105. package/src/GlobalStateManager.res +0 -57
  106. package/src/GlobalStateManager.res.mjs +0 -68
  107. package/src/GlobalStateManager.resi +0 -7
  108. package/src/Ports.res +0 -5
  109. package/src/Ports.res.mjs +0 -9
  110. package/src/adapters/MarkBatchProcessedAdapter.res +0 -5
  111. package/src/adapters/MarkBatchProcessedAdapter.res.mjs +0 -14
  112. package/src/sources/EnvioApiClient.res +0 -15
  113. package/src/sources/EnvioApiClient.res.mjs +0 -24
@@ -1,21 +1,24 @@
1
1
  // Generated by ReScript, PLEASE EDIT WITH CARE
2
2
 
3
- import * as Rest from "../vendored/Rest.res.mjs";
4
3
  import * as Hrtime from "../bindings/Hrtime.res.mjs";
5
4
  import * as Source from "./Source.res.mjs";
6
5
  import * as Logging from "../Logging.res.mjs";
7
6
  import * as HyperFuel from "./HyperFuel.res.mjs";
8
- import * as HyperSync from "./HyperSync.res.mjs";
9
7
  import * as Prometheus from "../Prometheus.res.mjs";
10
8
  import * as EventRouter from "./EventRouter.res.mjs";
9
+ import * as Stdlib_JsExn from "@rescript/runtime/lib/es6/Stdlib_JsExn.js";
11
10
  import * as ErrorHandling from "../ErrorHandling.res.mjs";
12
11
  import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js";
13
- import * as EnvioApiClient from "./EnvioApiClient.res.mjs";
14
12
  import * as Stdlib_JsError from "@rescript/runtime/lib/es6/Stdlib_JsError.js";
13
+ import * as HyperFuelClient from "./HyperFuelClient.res.mjs";
15
14
  import * as Primitive_exceptions from "@rescript/runtime/lib/es6/Primitive_exceptions.js";
16
15
 
17
16
  let EventRoutingFailed = /* @__PURE__ */Primitive_exceptions.create("HyperFuelSource.EventRoutingFailed");
18
17
 
18
+ function isUnauthorizedError(message) {
19
+ return message.includes("401 Unauthorized");
20
+ }
21
+
19
22
  let mintEventTag = "mint";
20
23
 
21
24
  let burnEventTag = "burn";
@@ -176,9 +179,22 @@ function memoGetSelectionConfig(chain) {
176
179
  }
177
180
 
178
181
  function make(param) {
179
- let endpointUrl = param.endpointUrl;
182
+ let apiToken = param.apiToken;
180
183
  let chain = param.chain;
181
184
  let name = "HyperFuel";
185
+ let apiToken$1 = apiToken !== undefined ? apiToken : Stdlib_JsError.throwWithMessage(`An Envio API token is required for using HyperFuel as a data-source.
186
+ Set the ENVIO_API_TOKEN environment variable in your .env file.
187
+ Learn more or get a free Envio API token at: https://envio.dev/app/api-tokens`);
188
+ let client;
189
+ try {
190
+ client = HyperFuelClient.make({
191
+ url: param.endpointUrl,
192
+ apiToken: apiToken$1
193
+ });
194
+ } catch (raw_exn) {
195
+ let exn = Primitive_exceptions.internalToException(raw_exn);
196
+ client = ErrorHandling.mkLogAndRaise(undefined, "Failed to instantiate the HyperFuel client", exn);
197
+ }
182
198
  let getSelectionConfig = memoGetSelectionConfig(chain);
183
199
  let getItemsOrThrow = async (fromBlock, toBlock, addressesByContractName, indexingAddresses, knownHeight, param, selection, retry, logger) => {
184
200
  let totalTimeRef = Hrtime.makeTimer();
@@ -188,10 +204,10 @@ function make(param) {
188
204
  Prometheus.SourceRequestCount.increment(name, chain, "getLogs");
189
205
  let pageUnsafe;
190
206
  try {
191
- pageUnsafe = await HyperFuel.GetLogs.query(endpointUrl, fromBlock, toBlock, recieptsSelection);
207
+ pageUnsafe = await HyperFuel.GetLogs.query(client, fromBlock, toBlock, recieptsSelection);
192
208
  } catch (raw_error) {
193
209
  let error = Primitive_exceptions.internalToException(raw_error);
194
- if (error.RE_EXN_ID === HyperSync.GetLogs.$$Error) {
210
+ if (error.RE_EXN_ID === HyperFuel.GetLogs.$$Error) {
195
211
  let error$1 = error._1;
196
212
  let tmp;
197
213
  if (typeof error$1 !== "object") {
@@ -339,7 +355,7 @@ function make(param) {
339
355
  blockNumber: block.height,
340
356
  blockHash: block.id,
341
357
  logIndex: receiptIndex,
342
- event: {
358
+ payload: {
343
359
  contractName: eventConfig$1.contractName,
344
360
  eventName: eventConfig$1.name,
345
361
  params: params,
@@ -388,7 +404,6 @@ function make(param) {
388
404
  };
389
405
  };
390
406
  let getBlockHashes = (param, param$1) => Stdlib_JsError.throwWithMessage("HyperFuel does not support getting block hashes");
391
- let jsonApiClient = EnvioApiClient.make(endpointUrl);
392
407
  return {
393
408
  name: name,
394
409
  sourceFor: "Sync",
@@ -398,7 +413,37 @@ function make(param) {
398
413
  getBlockHashes: getBlockHashes,
399
414
  getHeightOrThrow: async () => {
400
415
  let timerRef = Hrtime.makeTimer();
401
- let height = await Rest.fetch(HyperFuel.heightRoute, undefined, jsonApiClient);
416
+ let height;
417
+ try {
418
+ height = await client.getHeight();
419
+ } catch (raw_e) {
420
+ let e = Primitive_exceptions.internalToException(raw_e);
421
+ if (e.RE_EXN_ID === "JsExn") {
422
+ let e$1 = e._1;
423
+ let message = Stdlib_JsExn.message(e$1);
424
+ if (message !== undefined) {
425
+ if (message.includes("401 Unauthorized")) {
426
+ Logging.error(`Your ENVIO_API_TOKEN was rejected by HyperFuel (401 Unauthorized). The indexer will not be able to fetch events. Update the token and try again using 'envio start' or 'envio dev'. For more info: https://docs.envio.dev/docs/HyperSync/api-tokens`);
427
+ await new Promise((param, param$1) => {});
428
+ height = 0;
429
+ } else {
430
+ throw {
431
+ RE_EXN_ID: "JsExn",
432
+ _1: e$1,
433
+ Error: new Error()
434
+ };
435
+ }
436
+ } else {
437
+ throw {
438
+ RE_EXN_ID: "JsExn",
439
+ _1: e$1,
440
+ Error: new Error()
441
+ };
442
+ }
443
+ } else {
444
+ throw e;
445
+ }
446
+ }
402
447
  let seconds = Hrtime.toSecondsFloat(Hrtime.timeSince(timerRef));
403
448
  Prometheus.SourceRequestCount.increment(name, chain, "getHeight");
404
449
  Prometheus.SourceRequestCount.addSeconds(name, chain, "getHeight", seconds);
@@ -410,6 +455,7 @@ function make(param) {
410
455
 
411
456
  export {
412
457
  EventRoutingFailed,
458
+ isUnauthorizedError,
413
459
  mintEventTag,
414
460
  burnEventTag,
415
461
  transferEventTag,
@@ -422,4 +468,4 @@ export {
422
468
  memoGetSelectionConfig,
423
469
  make,
424
470
  }
425
- /* Rest Not a pure module */
471
+ /* Logging Not a pure module */
@@ -167,9 +167,9 @@ let make = (
167
167
  let apiToken = switch apiToken {
168
168
  | Some(token) => token
169
169
  | None =>
170
- JsError.throwWithMessage(`An API token is required for using HyperSync as a data-source.
170
+ JsError.throwWithMessage(`An Envio API token is required for using HyperSync as a data-source.
171
171
  Set the ENVIO_API_TOKEN environment variable in your .env file.
172
- Learn more or get a free API token at: https://envio.dev/app/api-tokens`)
172
+ Learn more or get a free Envio API token at: https://envio.dev/app/api-tokens`)
173
173
  }
174
174
 
175
175
  let client = switch HyperSyncClient.make(
@@ -206,16 +206,18 @@ Learn more or get a free API token at: https://envio.dev/app/api-tokens`)
206
206
  blockNumber: block.number->Option.getUnsafe,
207
207
  blockHash: block.hash->Option.getUnsafe,
208
208
  logIndex,
209
- event: {
209
+ payload: {
210
210
  contractName: eventConfig.contractName,
211
211
  eventName: eventConfig.name,
212
212
  chainId,
213
213
  params,
214
- transaction,
214
+ transaction: transaction->(
215
+ Utils.magic: HyperSyncClient.ResponseTypes.transaction => Internal.eventTransaction
216
+ ),
215
217
  block: block->(Utils.magic: HyperSyncClient.ResponseTypes.block => Internal.eventBlock),
216
218
  srcAddress,
217
219
  logIndex,
218
- }->Internal.fromGenericEvent,
220
+ }->Evm.fromPayload,
219
221
  })
220
222
  }
221
223
 
@@ -113,9 +113,9 @@ function make(param) {
113
113
  let chain = param.chain;
114
114
  let name = "HyperSync";
115
115
  let getSelectionConfig = memoGetSelectionConfig(chain);
116
- let apiToken$1 = apiToken !== undefined ? apiToken : Stdlib_JsError.throwWithMessage(`An API token is required for using HyperSync as a data-source.
116
+ let apiToken$1 = apiToken !== undefined ? apiToken : Stdlib_JsError.throwWithMessage(`An Envio API token is required for using HyperSync as a data-source.
117
117
  Set the ENVIO_API_TOKEN environment variable in your .env file.
118
- Learn more or get a free API token at: https://envio.dev/app/api-tokens`);
118
+ Learn more or get a free Envio API token at: https://envio.dev/app/api-tokens`);
119
119
  let client;
120
120
  try {
121
121
  client = HyperSyncClient.make(endpointUrl, apiToken$1, param.clientTimeoutMillis, param.allEventParams, !param.lowercaseAddresses, param.serializationFormat, param.enableQueryCaching, undefined, undefined, undefined, param.logLevel);
@@ -135,7 +135,7 @@ Learn more or get a free API token at: https://envio.dev/app/api-tokens`);
135
135
  blockNumber: block.number,
136
136
  blockHash: block.hash,
137
137
  logIndex: logIndex,
138
- event: {
138
+ payload: {
139
139
  contractName: eventConfig.contractName,
140
140
  eventName: eventConfig.name,
141
141
  params: params,
@@ -1200,7 +1200,7 @@ let make = (
1200
1200
  blockHash: block->getBlockHash,
1201
1201
  chain,
1202
1202
  logIndex: log.logIndex,
1203
- event: {
1203
+ payload: {
1204
1204
  contractName: eventConfig.contractName,
1205
1205
  eventName: eventConfig.name,
1206
1206
  chainId: chain->ChainMap.Chain.toChainId,
@@ -1209,7 +1209,7 @@ let make = (
1209
1209
  block,
1210
1210
  srcAddress: routedAddress,
1211
1211
  logIndex: log.logIndex,
1212
- }->Internal.fromGenericEvent,
1212
+ }->Evm.fromPayload,
1213
1213
  })
1214
1214
  }
1215
1215
  )(),
@@ -1131,7 +1131,7 @@ function make(param) {
1131
1131
  blockNumber: block.number,
1132
1132
  blockHash: block.hash,
1133
1133
  logIndex: log.logIndex,
1134
- event: {
1134
+ payload: {
1135
1135
  contractName: eventConfig.contractName,
1136
1136
  eventName: eventConfig.name,
1137
1137
  params: decoded$1,
@@ -11,16 +11,12 @@ type sourceState = {
11
11
  mutable lastFailedAt: option<float>,
12
12
  }
13
13
 
14
- // Ideally the ChainFetcher name suits this better
15
- // But currently the ChainFetcher module is immutable
16
- // and handles both processing and fetching.
17
- // So this module is to encapsulate the fetching logic only
14
+ // Encapsulates the fetching logic for a chain's sources.
18
15
  // with a mutable state for easier reasoning and testing.
19
16
  type t = {
20
17
  sourcesState: array<sourceState>,
21
18
  mutable statusStart: Hrtime.timeRef,
22
19
  mutable status: sourceManagerStatus,
23
- maxPartitionConcurrency: int,
24
20
  newBlockStallTimeout: int,
25
21
  newBlockStallTimeoutRealtime: int,
26
22
  stalledPollingInterval: int,
@@ -46,6 +42,10 @@ type t = {
46
42
 
47
43
  let getActiveSource = sourceManager => sourceManager.activeSource
48
44
 
45
+ // Partition queries currently in flight on this chain's sources. Summed across
46
+ // chains by CrossChainState to enforce the indexer-wide concurrency budget.
47
+ let inFlightCount = sourceManager => sourceManager.fetchingPartitionsCount
48
+
49
49
  let getRateLimitTimeMs = sourceManager =>
50
50
  sourceManager.committedRateLimitTimeMs +.
51
51
  switch sourceManager.activeRateLimitStartMs {
@@ -151,7 +151,6 @@ let makeGetHeightRetryInterval = (
151
151
 
152
152
  let make = (
153
153
  ~sources: array<Source.t>,
154
- ~maxPartitionConcurrency,
155
154
  ~isRealtime,
156
155
  ~newBlockStallTimeout=60_000,
157
156
  ~newBlockStallTimeoutRealtime=20_000,
@@ -172,16 +171,11 @@ let make = (
172
171
  | None =>
173
172
  JsError.throwWithMessage("Invalid configuration, no data-source for historical sync provided")
174
173
  }
175
- Prometheus.IndexingMaxConcurrency.set(
176
- ~maxConcurrency=maxPartitionConcurrency,
177
- ~chainId=initialActiveSource.chain->ChainMap.Chain.toChainId,
178
- )
179
174
  Prometheus.IndexingConcurrency.set(
180
175
  ~concurrency=0,
181
176
  ~chainId=initialActiveSource.chain->ChainMap.Chain.toChainId,
182
177
  )
183
178
  {
184
- maxPartitionConcurrency,
185
179
  sourcesState: sources->Array.map(source => {
186
180
  source,
187
181
  knownHeight: 0,
@@ -229,15 +223,11 @@ let fetchNext = async (
229
223
  ~executeQuery,
230
224
  ~waitForNewBlock,
231
225
  ~onNewBlock,
226
+ ~concurrencyLimit,
227
+ ~bufferLimit,
232
228
  ~stateId,
233
229
  ) => {
234
- let {maxPartitionConcurrency} = sourceManager
235
-
236
- let nextQuery = fetchState->FetchState.getNextQuery(
237
- ~concurrencyLimit={
238
- maxPartitionConcurrency - sourceManager.fetchingPartitionsCount
239
- },
240
- )
230
+ let nextQuery = fetchState->FetchState.getNextQuery(~concurrencyLimit, ~bufferLimit)
241
231
 
242
232
  switch nextQuery {
243
233
  | ReachedMaxConcurrency
@@ -18,6 +18,10 @@ function getActiveSource(sourceManager) {
18
18
  return sourceManager.activeSource;
19
19
  }
20
20
 
21
+ function inFlightCount(sourceManager) {
22
+ return sourceManager.fetchingPartitionsCount;
23
+ }
24
+
21
25
  function getRateLimitTimeMs(sourceManager) {
22
26
  let startMs = sourceManager.activeRateLimitStartMs;
23
27
  return sourceManager.committedRateLimitTimeMs + (
@@ -115,7 +119,7 @@ function makeGetHeightRetryInterval(initialRetryInterval, backoffMultiplicative,
115
119
  };
116
120
  }
117
121
 
118
- function make(sources, maxPartitionConcurrency, isRealtime, newBlockStallTimeoutOpt, newBlockStallTimeoutRealtimeOpt, stalledPollingIntervalOpt, reducedPollingIntervalOpt, recoveryTimeoutOpt, getHeightRetryIntervalOpt) {
122
+ function make(sources, isRealtime, newBlockStallTimeoutOpt, newBlockStallTimeoutRealtimeOpt, stalledPollingIntervalOpt, reducedPollingIntervalOpt, recoveryTimeoutOpt, getHeightRetryIntervalOpt) {
119
123
  let newBlockStallTimeout = newBlockStallTimeoutOpt !== undefined ? newBlockStallTimeoutOpt : 60000;
120
124
  let newBlockStallTimeoutRealtime = newBlockStallTimeoutRealtimeOpt !== undefined ? newBlockStallTimeoutRealtimeOpt : 20000;
121
125
  let stalledPollingInterval = stalledPollingIntervalOpt !== undefined ? stalledPollingIntervalOpt : 5000;
@@ -125,7 +129,6 @@ function make(sources, maxPartitionConcurrency, isRealtime, newBlockStallTimeout
125
129
  let hasRealtime = sources.some(s => s.sourceFor === "Realtime");
126
130
  let source = sources.find(source => getSourceRole(source.sourceFor, isRealtime, hasRealtime) === "Primary");
127
131
  let initialActiveSource = source !== undefined ? source : Stdlib_JsError.throwWithMessage("Invalid configuration, no data-source for historical sync provided");
128
- Prometheus.IndexingMaxConcurrency.set(maxPartitionConcurrency, initialActiveSource.chain);
129
132
  Prometheus.IndexingConcurrency.set(0, initialActiveSource.chain);
130
133
  return {
131
134
  sourcesState: sources.map(source => ({
@@ -138,7 +141,6 @@ function make(sources, maxPartitionConcurrency, isRealtime, newBlockStallTimeout
138
141
  })),
139
142
  statusStart: Hrtime.makeTimer(),
140
143
  status: "Idle",
141
- maxPartitionConcurrency: maxPartitionConcurrency,
142
144
  newBlockStallTimeout: newBlockStallTimeout,
143
145
  newBlockStallTimeoutRealtime: newBlockStallTimeoutRealtime,
144
146
  stalledPollingInterval: stalledPollingInterval,
@@ -175,8 +177,8 @@ function trackNewStatus(sourceManager, newStatus) {
175
177
  sourceManager.status = newStatus;
176
178
  }
177
179
 
178
- async function fetchNext(sourceManager, fetchState, executeQuery, waitForNewBlock, onNewBlock, stateId) {
179
- let nextQuery = FetchState.getNextQuery(fetchState, sourceManager.maxPartitionConcurrency - sourceManager.fetchingPartitionsCount | 0);
180
+ async function fetchNext(sourceManager, fetchState, executeQuery, waitForNewBlock, onNewBlock, concurrencyLimit, bufferLimit, stateId) {
181
+ let nextQuery = FetchState.getNextQuery(fetchState, concurrencyLimit, bufferLimit);
180
182
  if (typeof nextQuery !== "object") {
181
183
  switch (nextQuery) {
182
184
  case "WaitingForNewBlock" :
@@ -680,6 +682,7 @@ export {
680
682
  getSourceRole,
681
683
  make,
682
684
  getActiveSource,
685
+ inFlightCount,
683
686
  onReorg,
684
687
  fetchNext,
685
688
  waitForNewBlock,
@@ -10,7 +10,6 @@ let getSourceRole: (
10
10
 
11
11
  let make: (
12
12
  ~sources: array<Source.t>,
13
- ~maxPartitionConcurrency: int,
14
13
  ~isRealtime: bool,
15
14
  ~newBlockStallTimeout: int=?,
16
15
  ~newBlockStallTimeoutRealtime: int=?,
@@ -22,6 +21,8 @@ let make: (
22
21
 
23
22
  let getActiveSource: t => Source.t
24
23
 
24
+ let inFlightCount: t => int
25
+
25
26
  let onReorg: (t, ~rollbackTargetBlock: int) => unit
26
27
 
27
28
  let fetchNext: (
@@ -30,6 +31,8 @@ let fetchNext: (
30
31
  ~executeQuery: FetchState.query => promise<unit>,
31
32
  ~waitForNewBlock: (~knownHeight: int) => promise<int>,
32
33
  ~onNewBlock: (~knownHeight: int) => unit,
34
+ ~concurrencyLimit: int,
35
+ ~bufferLimit: int,
33
36
  ~stateId: int,
34
37
  ) => promise<unit>
35
38
 
@@ -4,7 +4,7 @@ let cleanUpRawEventFieldsInPlace: JSON.t => unit = %raw(`fields => {
4
4
  delete fields.time
5
5
  }`)
6
6
 
7
- let ecosystem: Ecosystem.t = {
7
+ let make = (~logger: Pino.t): Ecosystem.t => {
8
8
  name: Svm,
9
9
  blockFields: ["slot"],
10
10
  transactionFields: [],
@@ -19,6 +19,23 @@ let ecosystem: Ecosystem.t = {
19
19
  // SVM has no event handlers, so there is no `onEvent` `where` value to
20
20
  // parse. The schema is a no-op object that always surfaces `None`.
21
21
  onEventBlockFilterSchema: S.object(_ => None),
22
+ logger,
23
+ toEvent: eventItem => eventItem.payload->Internal.payloadToEvent,
24
+ toEventLogger: eventItem => {
25
+ let instruction =
26
+ eventItem.payload->(Utils.magic: Internal.eventPayload => Envio.svmInstruction)
27
+ Logging.createChildFrom(
28
+ ~logger,
29
+ ~params={
30
+ "program": eventItem.eventConfig.contractName,
31
+ "instruction": eventItem.eventConfig.name,
32
+ "chainId": eventItem.chain->ChainMap.Chain.toChainId,
33
+ "slot": eventItem.blockNumber,
34
+ "programId": instruction.programId,
35
+ },
36
+ )
37
+ },
38
+ toRawEvent: _ => JsError.throwWithMessage("Raw events are not supported for SVM"),
22
39
  }
23
40
 
24
41
  module GetFinalizedSlot = {
@@ -4,6 +4,7 @@ import * as Rpc from "./Rpc.res.mjs";
4
4
  import * as Rest from "../vendored/Rest.res.mjs";
5
5
  import * as Utils from "../Utils.res.mjs";
6
6
  import * as Hrtime from "../bindings/Hrtime.res.mjs";
7
+ import * as Logging from "../Logging.res.mjs";
7
8
  import * as Prometheus from "../Prometheus.res.mjs";
8
9
  import * as Stdlib_JsError from "@rescript/runtime/lib/es6/Stdlib_JsError.js";
9
10
  import * as S$RescriptSchema from "rescript-schema/src/S.res.mjs";
@@ -14,26 +15,33 @@ let cleanUpRawEventFieldsInPlace = (fields => {
14
15
  delete fields.time
15
16
  });
16
17
 
17
- let ecosystem_blockFields = ["slot"];
18
-
19
- let ecosystem_transactionFields = [];
20
-
21
- let ecosystem_onBlockFilterSchema = S$RescriptSchema.object(s => s.f("slot", S$RescriptSchema.option(S$RescriptSchema.unknown)));
22
-
23
- let ecosystem_onEventBlockFilterSchema = S$RescriptSchema.object(param => {});
24
-
25
- let ecosystem = {
26
- name: "svm",
27
- blockFields: ecosystem_blockFields,
28
- transactionFields: ecosystem_transactionFields,
29
- blockNumberName: "height",
30
- blockTimestampName: "time",
31
- blockHashName: "hash",
32
- cleanUpRawEventFieldsInPlace: cleanUpRawEventFieldsInPlace,
33
- onBlockMethodName: "onSlot",
34
- onBlockFilterSchema: ecosystem_onBlockFilterSchema,
35
- onEventBlockFilterSchema: ecosystem_onEventBlockFilterSchema
36
- };
18
+ function make(logger) {
19
+ return {
20
+ name: "svm",
21
+ blockFields: ["slot"],
22
+ transactionFields: [],
23
+ blockNumberName: "height",
24
+ blockTimestampName: "time",
25
+ blockHashName: "hash",
26
+ cleanUpRawEventFieldsInPlace: cleanUpRawEventFieldsInPlace,
27
+ onBlockMethodName: "onSlot",
28
+ onBlockFilterSchema: S$RescriptSchema.object(s => s.f("slot", S$RescriptSchema.option(S$RescriptSchema.unknown))),
29
+ onEventBlockFilterSchema: S$RescriptSchema.object(param => {}),
30
+ logger: logger,
31
+ toEvent: eventItem => eventItem.payload,
32
+ toEventLogger: eventItem => {
33
+ let instruction = eventItem.payload;
34
+ return Logging.createChildFrom(logger, {
35
+ program: eventItem.eventConfig.contractName,
36
+ instruction: eventItem.eventConfig.name,
37
+ chainId: eventItem.chain,
38
+ slot: eventItem.blockNumber,
39
+ programId: instruction.programId
40
+ });
41
+ },
42
+ toRawEvent: param => Stdlib_JsError.throwWithMessage("Raw events are not supported for SVM")
43
+ };
44
+ }
37
45
 
38
46
  let route = Rpc.makeRpcRoute("getSlot", S$RescriptSchema.tuple(s => {
39
47
  s.tag(0, {
@@ -72,8 +80,8 @@ function makeRPCSource(chain, rpc, sourceForOpt) {
72
80
 
73
81
  export {
74
82
  cleanUpRawEventFieldsInPlace,
75
- ecosystem,
83
+ make,
76
84
  GetFinalizedSlot,
77
85
  makeRPCSource,
78
86
  }
79
- /* ecosystem Not a pure module */
87
+ /* route Not a pure module */
@@ -510,7 +510,7 @@ let make = ({chain, endpointUrl, apiToken, eventConfigs, clientTimeoutMillis}: o
510
510
  blockNumber: instr.slot,
511
511
  blockHash: "",
512
512
  logIndex: synthLogIndex(instr),
513
- event: payload->(Utils.magic: Envio.svmInstruction => Internal.event),
513
+ payload: payload->(Utils.magic: Envio.svmInstruction => Internal.eventPayload),
514
514
  }),
515
515
  )
516
516
  ->ignore
@@ -420,7 +420,7 @@ function make(param) {
420
420
  blockNumber: instr.slot,
421
421
  blockHash: "",
422
422
  logIndex: synthLogIndex(instr),
423
- event: payload
423
+ payload: payload
424
424
  });
425
425
  }
426
426
  });
package/src/tui/Tui.res CHANGED
@@ -134,11 +134,11 @@ module App = {
134
134
  @react.component
135
135
  let make = (~getState) => {
136
136
  let stdoutColumns = Hooks.useStdoutColumns()
137
- // GlobalState is mutated in place — passing the same ref to useState
137
+ // IndexerState is mutated in place — passing the same ref to useState
138
138
  // would bail out via Object.is and skip the re-render. Tick a counter
139
139
  // instead and read state freshly from getState() on every render.
140
140
  let (tick, setTick) = React.useState(() => 0)
141
- let state: GlobalState.t = getState()
141
+ let state: IndexerState.t = getState()
142
142
 
143
143
  React.useEffect(() => {
144
144
  let intervalId = setInterval(() => {
@@ -153,18 +153,22 @@ module App = {
153
153
  }, [getState])
154
154
 
155
155
  let chains =
156
- state.chainManager.chainFetchers
157
- ->ChainMap.values
158
- ->Array.map(cf => {
159
- let {numEventsProcessed, fetchState} = cf
156
+ state
157
+ ->IndexerState.chainStates
158
+ ->Dict.valuesToArray
159
+ ->Array.map(cs => {
160
+ let numEventsProcessed = cs->ChainState.numEventsProcessed
161
+ let fetchState = cs->ChainState.fetchState
162
+ let committedProgressBlockNumber = cs->ChainState.committedProgressBlockNumber
163
+ let timestampCaughtUpToHeadOrEndblock = cs->ChainState.timestampCaughtUpToHeadOrEndblock
164
+ let sourceManager = cs->ChainState.sourceManager
160
165
  let latestFetchedBlockNumber = Pervasives.max(fetchState->FetchState.bufferBlockNumber, 0)
161
- let hasProcessedToEndblock = cf->ChainFetcher.hasProcessedToEndblock
162
- let knownHeight =
163
- cf->ChainFetcher.hasProcessedToEndblock
164
- ? cf.fetchState.endBlock->Option.getOr(cf.fetchState.knownHeight)
165
- : cf.fetchState.knownHeight
166
+ let hasProcessedToEndblock = cs->ChainState.hasProcessedToEndblock
167
+ let knownHeight = hasProcessedToEndblock
168
+ ? fetchState.endBlock->Option.getOr(fetchState.knownHeight)
169
+ : fetchState.knownHeight
166
170
 
167
- let firstEventBlock = cf.fetchState.firstEventBlock
171
+ let firstEventBlock = fetchState.firstEventBlock
168
172
  let progress: TuiData.progress = if hasProcessedToEndblock {
169
173
  // If the endblock has been reached then set the progress to synced.
170
174
  // if there's chains that have no events in the block range start->end,
@@ -172,25 +176,25 @@ module App = {
172
176
  // This ensures TUI still displays synced in this case
173
177
  Synced({
174
178
  firstEventBlockNumber: firstEventBlock->Option.getOr(0),
175
- latestProcessedBlock: cf.committedProgressBlockNumber,
176
- timestampCaughtUpToHeadOrEndblock: cf.timestampCaughtUpToHeadOrEndblock->Option.getOr(
179
+ latestProcessedBlock: committedProgressBlockNumber,
180
+ timestampCaughtUpToHeadOrEndblock: timestampCaughtUpToHeadOrEndblock->Option.getOr(
177
181
  Date.now()->Date.fromTime,
178
182
  ),
179
183
  numEventsProcessed,
180
184
  })
181
185
  } else {
182
- switch (firstEventBlock, cf.timestampCaughtUpToHeadOrEndblock) {
186
+ switch (firstEventBlock, timestampCaughtUpToHeadOrEndblock) {
183
187
  | (Some(firstEventBlockNumber), Some(timestampCaughtUpToHeadOrEndblock)) =>
184
188
  Synced({
185
189
  firstEventBlockNumber,
186
- latestProcessedBlock: cf.committedProgressBlockNumber,
190
+ latestProcessedBlock: committedProgressBlockNumber,
187
191
  timestampCaughtUpToHeadOrEndblock,
188
192
  numEventsProcessed,
189
193
  })
190
194
  | (Some(firstEventBlockNumber), None) =>
191
195
  Syncing({
192
196
  firstEventBlockNumber,
193
- latestProcessedBlock: cf.committedProgressBlockNumber,
197
+ latestProcessedBlock: committedProgressBlockNumber,
194
198
  numEventsProcessed,
195
199
  })
196
200
  | (None, _) => SearchingForEvents
@@ -203,25 +207,23 @@ module App = {
203
207
  knownHeight,
204
208
  latestFetchedBlockNumber,
205
209
  eventsProcessed: numEventsProcessed,
206
- chainId: cf.chainConfig.id->Int.toString,
207
- progressBlock: cf.committedProgressBlockNumber < cf.fetchState.startBlock
208
- ? Some(cf.fetchState.startBlock)
209
- : Some(cf.committedProgressBlockNumber),
210
+ chainId: (cs->ChainState.chainConfig).id->Int.toString,
211
+ progressBlock: committedProgressBlockNumber < fetchState.startBlock
212
+ ? Some(fetchState.startBlock)
213
+ : Some(committedProgressBlockNumber),
210
214
  bufferBlock: Some(latestFetchedBlockNumber),
211
- sourceBlock: Some(cf.fetchState.knownHeight),
212
- firstEventBlockNumber: cf.fetchState.firstEventBlock,
213
- startBlock: cf.fetchState.startBlock,
214
- endBlock: cf.fetchState.endBlock,
215
- poweredByHyperSync: (
216
- cf.sourceManager->SourceManager.getActiveSource
217
- ).poweredByHyperSync,
218
- blockUnit: switch state.ctx.config.ecosystem.name {
215
+ sourceBlock: Some(fetchState.knownHeight),
216
+ firstEventBlockNumber: fetchState.firstEventBlock,
217
+ startBlock: fetchState.startBlock,
218
+ endBlock: fetchState.endBlock,
219
+ poweredByHyperSync: (sourceManager->SourceManager.getActiveSource).poweredByHyperSync,
220
+ blockUnit: switch (state->IndexerState.config).ecosystem.name {
219
221
  | Svm => "Slot"
220
222
  | Evm | Fuel => "Block"
221
223
  },
222
- rateLimitTimeMs: cf.sourceManager->SourceManager.getRateLimitTimeMs,
223
- isRateLimited: cf.sourceManager->SourceManager.isRateLimited,
224
- rateLimitResetInMs: cf.sourceManager->SourceManager.getRateLimitResetInMs,
224
+ rateLimitTimeMs: sourceManager->SourceManager.getRateLimitTimeMs,
225
+ isRateLimited: sourceManager->SourceManager.isRateLimited,
226
+ rateLimitResetInMs: sourceManager->SourceManager.getRateLimitResetInMs,
225
227
  }: TuiData.chain
226
228
  )
227
229
  })
@@ -269,7 +271,7 @@ module App = {
269
271
  totalEventsProcessed
270
272
  eventsPerSecond={SyncETA.isIndexerFullySynced(chains) ? None : eventsPerSecond}
271
273
  />
272
- <SyncETA chains indexerStartTime=state.indexerStartTime />
274
+ <SyncETA chains indexerStartTime={state->IndexerState.indexerStartTime} />
273
275
  {
274
276
  let maxRateLimitTimeMs =
275
277
  chains->Array.reduce(0., (acc, chain) => Pervasives.max(acc, chain.rateLimitTimeMs))
@@ -315,7 +317,7 @@ module App = {
315
317
  }
316
318
  }
317
319
  </Box>
318
- {if state.ctx.config.isDev {
320
+ {if (state->IndexerState.config).isDev {
319
321
  <Box flexDirection={Row}>
320
322
  <Text> {"Dev Console: "->React.string} </Text>
321
323
  <Text color={Info} underline=true> {`${Env.envioAppUrl}/console`->React.string} </Text>
@@ -323,7 +325,7 @@ module App = {
323
325
  } else {
324
326
  React.null
325
327
  }}
326
- {switch (state.ctx.config.storage.clickhouse, Env.ClickHouse.host()) {
328
+ {switch ((state->IndexerState.config).storage.clickhouse, Env.ClickHouse.host()) {
327
329
  | (true, Some(host)) =>
328
330
  <Box flexDirection={Row}>
329
331
  <Text> {"ClickHouse: "->React.string} </Text>
@@ -331,7 +333,7 @@ module App = {
331
333
  </Box>
332
334
  | _ => React.null
333
335
  }}
334
- <Messages config=state.ctx.config />
336
+ <Messages config={state->IndexerState.config} />
335
337
  </Box>
336
338
  }
337
339
  }