envio 3.2.1 → 3.3.0-alpha.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (106) hide show
  1. package/package.json +6 -7
  2. package/src/Batch.res +12 -15
  3. package/src/Batch.res.mjs +4 -5
  4. package/src/BatchProcessing.res +199 -0
  5. package/src/BatchProcessing.res.mjs +125 -0
  6. package/src/ChainFetching.res +373 -0
  7. package/src/ChainFetching.res.mjs +206 -0
  8. package/src/ChainMetadata.res +27 -0
  9. package/src/ChainMetadata.res.mjs +27 -0
  10. package/src/ChainState.res +643 -0
  11. package/src/ChainState.res.mjs +476 -0
  12. package/src/ChainState.resi +87 -0
  13. package/src/Config.res +13 -4
  14. package/src/Config.res.mjs +8 -3
  15. package/src/ContractRegisterContext.res +106 -0
  16. package/src/ContractRegisterContext.res.mjs +76 -0
  17. package/src/Core.res +3 -0
  18. package/src/CrossChainState.res +294 -0
  19. package/src/CrossChainState.res.mjs +221 -0
  20. package/src/CrossChainState.resi +39 -0
  21. package/src/Ecosystem.res +58 -0
  22. package/src/Ecosystem.res.mjs +43 -1
  23. package/src/Env.res +0 -2
  24. package/src/Env.res.mjs +0 -3
  25. package/src/EventConfigBuilder.res +2 -2
  26. package/src/EventProcessing.res +60 -46
  27. package/src/EventProcessing.res.mjs +33 -32
  28. package/src/EventUtils.res +0 -4
  29. package/src/EventUtils.res.mjs +0 -4
  30. package/src/ExitOnCaughtUp.res +10 -0
  31. package/src/ExitOnCaughtUp.res.mjs +22 -0
  32. package/src/FetchState.res +205 -73
  33. package/src/FetchState.res.mjs +242 -103
  34. package/src/InMemoryStore.res +36 -451
  35. package/src/InMemoryStore.res.mjs +20 -369
  36. package/src/IndexerLoop.res +43 -0
  37. package/src/IndexerLoop.res.mjs +46 -0
  38. package/src/IndexerState.res +536 -0
  39. package/src/IndexerState.res.mjs +543 -0
  40. package/src/IndexerState.resi +128 -0
  41. package/src/Internal.res +29 -8
  42. package/src/LoadLayer.res +20 -18
  43. package/src/LoadLayer.res.mjs +14 -12
  44. package/src/LoadLayer.resi +6 -3
  45. package/src/Logging.res +11 -44
  46. package/src/Logging.res.mjs +10 -42
  47. package/src/Main.res +54 -79
  48. package/src/Main.res.mjs +44 -56
  49. package/src/PgStorage.res +8 -71
  50. package/src/PgStorage.res.mjs +3 -47
  51. package/src/Prometheus.res +1 -13
  52. package/src/Prometheus.res.mjs +84 -96
  53. package/src/PruneStaleHistory.res +45 -0
  54. package/src/PruneStaleHistory.res.mjs +46 -0
  55. package/src/RawEvent.res +73 -0
  56. package/src/RawEvent.res.mjs +51 -0
  57. package/src/Rollback.res +204 -0
  58. package/src/Rollback.res.mjs +118 -0
  59. package/src/SimulateItems.res +12 -10
  60. package/src/SimulateItems.res.mjs +1 -1
  61. package/src/UserContext.res +26 -114
  62. package/src/UserContext.res.mjs +15 -78
  63. package/src/Writing.res +242 -0
  64. package/src/Writing.res.mjs +215 -0
  65. package/src/db/InternalTable.res +14 -27
  66. package/src/sources/Evm.res +40 -1
  67. package/src/sources/Evm.res.mjs +94 -84
  68. package/src/sources/Fuel.res +40 -1
  69. package/src/sources/Fuel.res.mjs +36 -26
  70. package/src/sources/HyperFuel.res +41 -120
  71. package/src/sources/HyperFuel.res.mjs +42 -80
  72. package/src/sources/HyperFuel.resi +1 -24
  73. package/src/sources/HyperFuelClient.res +16 -297
  74. package/src/sources/HyperFuelClient.res.mjs +5 -4
  75. package/src/sources/HyperFuelSource.res +33 -8
  76. package/src/sources/HyperFuelSource.res.mjs +56 -10
  77. package/src/sources/HyperSyncSource.res +5 -3
  78. package/src/sources/HyperSyncSource.res.mjs +1 -1
  79. package/src/sources/RpcSource.res +2 -2
  80. package/src/sources/RpcSource.res.mjs +1 -1
  81. package/src/sources/SourceManager.res +14 -23
  82. package/src/sources/SourceManager.res.mjs +27 -29
  83. package/src/sources/SourceManager.resi +4 -2
  84. package/src/sources/Svm.res +18 -1
  85. package/src/sources/Svm.res.mjs +30 -22
  86. package/src/sources/SvmHyperSyncSource.res +1 -1
  87. package/src/sources/SvmHyperSyncSource.res.mjs +1 -1
  88. package/src/tui/Tui.res +38 -36
  89. package/src/tui/Tui.res.mjs +51 -51
  90. package/src/ChainFetcher.res +0 -519
  91. package/src/ChainFetcher.res.mjs +0 -314
  92. package/src/ChainManager.res +0 -358
  93. package/src/ChainManager.res.mjs +0 -291
  94. package/src/Ctx.res +0 -6
  95. package/src/Ctx.res.mjs +0 -2
  96. package/src/GlobalState.res +0 -1056
  97. package/src/GlobalState.res.mjs +0 -1086
  98. package/src/GlobalStateManager.res +0 -57
  99. package/src/GlobalStateManager.res.mjs +0 -68
  100. package/src/GlobalStateManager.resi +0 -7
  101. package/src/Ports.res +0 -5
  102. package/src/Ports.res.mjs +0 -9
  103. package/src/adapters/MarkBatchProcessedAdapter.res +0 -5
  104. package/src/adapters/MarkBatchProcessedAdapter.res.mjs +0 -14
  105. package/src/sources/EnvioApiClient.res +0 -15
  106. package/src/sources/EnvioApiClient.res.mjs +0 -24
@@ -2,6 +2,8 @@ open Source
2
2
 
3
3
  exception EventRoutingFailed
4
4
 
5
+ let isUnauthorizedError = (message: string) => message->String.includes("401 Unauthorized")
6
+
5
7
  let mintEventTag = "mint"
6
8
  let burnEventTag = "burn"
7
9
  let transferEventTag = "transfer"
@@ -207,11 +209,26 @@ let memoGetSelectionConfig = (~chain) => {
207
209
  type options = {
208
210
  chain: ChainMap.Chain.t,
209
211
  endpointUrl: string,
212
+ apiToken: option<string>,
210
213
  }
211
214
 
212
- let make = ({chain, endpointUrl}: options): t => {
215
+ let make = ({chain, endpointUrl, apiToken}: options): t => {
213
216
  let name = "HyperFuel"
214
217
 
218
+ let apiToken = switch apiToken {
219
+ | Some(token) => token
220
+ | None =>
221
+ JsError.throwWithMessage(`An Envio API token is required for using HyperFuel as a data-source.
222
+ Set the ENVIO_API_TOKEN environment variable in your .env file.
223
+ Learn more or get a free Envio API token at: https://envio.dev/app/api-tokens`)
224
+ }
225
+
226
+ let client = switch HyperFuelClient.make({url: endpointUrl, apiToken}) {
227
+ | client => client
228
+ | exception exn =>
229
+ exn->ErrorHandling.mkLogAndRaise(~msg="Failed to instantiate the HyperFuel client")
230
+ }
231
+
215
232
  let getSelectionConfig = memoGetSelectionConfig(~chain)
216
233
 
217
234
  let getItemsOrThrow = async (
@@ -239,12 +256,12 @@ let make = ({chain, endpointUrl}: options): t => {
239
256
  ~method="getLogs",
240
257
  )
241
258
  let pageUnsafe = try await HyperFuel.GetLogs.query(
242
- ~serverUrl=endpointUrl,
259
+ ~client,
243
260
  ~fromBlock,
244
261
  ~toBlock,
245
262
  ~recieptsSelection,
246
263
  ) catch {
247
- | HyperSync.GetLogs.Error(error) =>
264
+ | HyperFuel.GetLogs.Error(error) =>
248
265
  throw(
249
266
  Source.GetItemsError(
250
267
  Source.FailedGettingItems({
@@ -397,7 +414,7 @@ let make = ({chain, endpointUrl}: options): t => {
397
414
  blockNumber: block.height,
398
415
  blockHash: block.id,
399
416
  logIndex: receiptIndex,
400
- event: {
417
+ payload: {
401
418
  contractName: eventConfig.contractName,
402
419
  eventName: eventConfig.name,
403
420
  chainId,
@@ -408,7 +425,7 @@ let make = ({chain, endpointUrl}: options): t => {
408
425
  block: block->Obj.magic,
409
426
  srcAddress: contractAddress,
410
427
  logIndex: receiptIndex,
411
- }->Internal.fromGenericEvent,
428
+ }->Fuel.fromPayload,
412
429
  })
413
430
  })
414
431
 
@@ -450,8 +467,6 @@ let make = ({chain, endpointUrl}: options): t => {
450
467
  let getBlockHashes = (~blockNumbers as _, ~logger as _) =>
451
468
  JsError.throwWithMessage("HyperFuel does not support getting block hashes")
452
469
 
453
- let jsonApiClient = EnvioApiClient.make(endpointUrl)
454
-
455
470
  {
456
471
  name,
457
472
  sourceFor: Sync,
@@ -461,7 +476,17 @@ let make = ({chain, endpointUrl}: options): t => {
461
476
  poweredByHyperSync: true,
462
477
  getHeightOrThrow: async () => {
463
478
  let timerRef = Hrtime.makeTimer()
464
- let height = await HyperFuel.heightRoute->Rest.fetch((), ~client=jsonApiClient)
479
+ let height = try await client->HyperFuelClient.getHeight catch {
480
+ | JsExn(e) =>
481
+ switch e->JsExn.message {
482
+ | Some(message) if message->isUnauthorizedError =>
483
+ Logging.error(`Your ENVIO_API_TOKEN was rejected by HyperFuel (401 Unauthorized). The indexer will not be able to fetch events. Update the token and try again using 'envio start' or 'envio dev'. For more info: https://docs.envio.dev/docs/HyperSync/api-tokens`)
484
+ // Retrying an unauthorized request can never succeed, so block forever
485
+ let _ = await Promise.make((_, _) => ())
486
+ 0
487
+ | _ => throw(JsExn(e))
488
+ }
489
+ }
465
490
  let seconds = timerRef->Hrtime.timeSince->Hrtime.toSecondsFloat
466
491
  Prometheus.SourceRequestCount.increment(
467
492
  ~sourceName=name,
@@ -1,21 +1,24 @@
1
1
  // Generated by ReScript, PLEASE EDIT WITH CARE
2
2
 
3
- import * as Rest from "../vendored/Rest.res.mjs";
4
3
  import * as Hrtime from "../bindings/Hrtime.res.mjs";
5
4
  import * as Source from "./Source.res.mjs";
6
5
  import * as Logging from "../Logging.res.mjs";
7
6
  import * as HyperFuel from "./HyperFuel.res.mjs";
8
- import * as HyperSync from "./HyperSync.res.mjs";
9
7
  import * as Prometheus from "../Prometheus.res.mjs";
10
8
  import * as EventRouter from "./EventRouter.res.mjs";
9
+ import * as Stdlib_JsExn from "@rescript/runtime/lib/es6/Stdlib_JsExn.js";
11
10
  import * as ErrorHandling from "../ErrorHandling.res.mjs";
12
11
  import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js";
13
- import * as EnvioApiClient from "./EnvioApiClient.res.mjs";
14
12
  import * as Stdlib_JsError from "@rescript/runtime/lib/es6/Stdlib_JsError.js";
13
+ import * as HyperFuelClient from "./HyperFuelClient.res.mjs";
15
14
  import * as Primitive_exceptions from "@rescript/runtime/lib/es6/Primitive_exceptions.js";
16
15
 
17
16
  let EventRoutingFailed = /* @__PURE__ */Primitive_exceptions.create("HyperFuelSource.EventRoutingFailed");
18
17
 
18
+ function isUnauthorizedError(message) {
19
+ return message.includes("401 Unauthorized");
20
+ }
21
+
19
22
  let mintEventTag = "mint";
20
23
 
21
24
  let burnEventTag = "burn";
@@ -176,9 +179,22 @@ function memoGetSelectionConfig(chain) {
176
179
  }
177
180
 
178
181
  function make(param) {
179
- let endpointUrl = param.endpointUrl;
182
+ let apiToken = param.apiToken;
180
183
  let chain = param.chain;
181
184
  let name = "HyperFuel";
185
+ let apiToken$1 = apiToken !== undefined ? apiToken : Stdlib_JsError.throwWithMessage(`An Envio API token is required for using HyperFuel as a data-source.
186
+ Set the ENVIO_API_TOKEN environment variable in your .env file.
187
+ Learn more or get a free Envio API token at: https://envio.dev/app/api-tokens`);
188
+ let client;
189
+ try {
190
+ client = HyperFuelClient.make({
191
+ url: param.endpointUrl,
192
+ apiToken: apiToken$1
193
+ });
194
+ } catch (raw_exn) {
195
+ let exn = Primitive_exceptions.internalToException(raw_exn);
196
+ client = ErrorHandling.mkLogAndRaise(undefined, "Failed to instantiate the HyperFuel client", exn);
197
+ }
182
198
  let getSelectionConfig = memoGetSelectionConfig(chain);
183
199
  let getItemsOrThrow = async (fromBlock, toBlock, addressesByContractName, indexingAddresses, knownHeight, param, selection, retry, logger) => {
184
200
  let totalTimeRef = Hrtime.makeTimer();
@@ -188,10 +204,10 @@ function make(param) {
188
204
  Prometheus.SourceRequestCount.increment(name, chain, "getLogs");
189
205
  let pageUnsafe;
190
206
  try {
191
- pageUnsafe = await HyperFuel.GetLogs.query(endpointUrl, fromBlock, toBlock, recieptsSelection);
207
+ pageUnsafe = await HyperFuel.GetLogs.query(client, fromBlock, toBlock, recieptsSelection);
192
208
  } catch (raw_error) {
193
209
  let error = Primitive_exceptions.internalToException(raw_error);
194
- if (error.RE_EXN_ID === HyperSync.GetLogs.$$Error) {
210
+ if (error.RE_EXN_ID === HyperFuel.GetLogs.$$Error) {
195
211
  let error$1 = error._1;
196
212
  let tmp;
197
213
  if (typeof error$1 !== "object") {
@@ -339,7 +355,7 @@ function make(param) {
339
355
  blockNumber: block.height,
340
356
  blockHash: block.id,
341
357
  logIndex: receiptIndex,
342
- event: {
358
+ payload: {
343
359
  contractName: eventConfig$1.contractName,
344
360
  eventName: eventConfig$1.name,
345
361
  params: params,
@@ -388,7 +404,6 @@ function make(param) {
388
404
  };
389
405
  };
390
406
  let getBlockHashes = (param, param$1) => Stdlib_JsError.throwWithMessage("HyperFuel does not support getting block hashes");
391
- let jsonApiClient = EnvioApiClient.make(endpointUrl);
392
407
  return {
393
408
  name: name,
394
409
  sourceFor: "Sync",
@@ -398,7 +413,37 @@ function make(param) {
398
413
  getBlockHashes: getBlockHashes,
399
414
  getHeightOrThrow: async () => {
400
415
  let timerRef = Hrtime.makeTimer();
401
- let height = await Rest.fetch(HyperFuel.heightRoute, undefined, jsonApiClient);
416
+ let height;
417
+ try {
418
+ height = await client.getHeight();
419
+ } catch (raw_e) {
420
+ let e = Primitive_exceptions.internalToException(raw_e);
421
+ if (e.RE_EXN_ID === "JsExn") {
422
+ let e$1 = e._1;
423
+ let message = Stdlib_JsExn.message(e$1);
424
+ if (message !== undefined) {
425
+ if (message.includes("401 Unauthorized")) {
426
+ Logging.error(`Your ENVIO_API_TOKEN was rejected by HyperFuel (401 Unauthorized). The indexer will not be able to fetch events. Update the token and try again using 'envio start' or 'envio dev'. For more info: https://docs.envio.dev/docs/HyperSync/api-tokens`);
427
+ await new Promise((param, param$1) => {});
428
+ height = 0;
429
+ } else {
430
+ throw {
431
+ RE_EXN_ID: "JsExn",
432
+ _1: e$1,
433
+ Error: new Error()
434
+ };
435
+ }
436
+ } else {
437
+ throw {
438
+ RE_EXN_ID: "JsExn",
439
+ _1: e$1,
440
+ Error: new Error()
441
+ };
442
+ }
443
+ } else {
444
+ throw e;
445
+ }
446
+ }
402
447
  let seconds = Hrtime.toSecondsFloat(Hrtime.timeSince(timerRef));
403
448
  Prometheus.SourceRequestCount.increment(name, chain, "getHeight");
404
449
  Prometheus.SourceRequestCount.addSeconds(name, chain, "getHeight", seconds);
@@ -410,6 +455,7 @@ function make(param) {
410
455
 
411
456
  export {
412
457
  EventRoutingFailed,
458
+ isUnauthorizedError,
413
459
  mintEventTag,
414
460
  burnEventTag,
415
461
  transferEventTag,
@@ -422,4 +468,4 @@ export {
422
468
  memoGetSelectionConfig,
423
469
  make,
424
470
  }
425
- /* Rest Not a pure module */
471
+ /* Logging Not a pure module */
@@ -206,16 +206,18 @@ Learn more or get a free Envio API token at: https://envio.dev/app/api-tokens`)
206
206
  blockNumber: block.number->Option.getUnsafe,
207
207
  blockHash: block.hash->Option.getUnsafe,
208
208
  logIndex,
209
- event: {
209
+ payload: {
210
210
  contractName: eventConfig.contractName,
211
211
  eventName: eventConfig.name,
212
212
  chainId,
213
213
  params,
214
- transaction,
214
+ transaction: transaction->(
215
+ Utils.magic: HyperSyncClient.ResponseTypes.transaction => Internal.eventTransaction
216
+ ),
215
217
  block: block->(Utils.magic: HyperSyncClient.ResponseTypes.block => Internal.eventBlock),
216
218
  srcAddress,
217
219
  logIndex,
218
- }->Internal.fromGenericEvent,
220
+ }->Evm.fromPayload,
219
221
  })
220
222
  }
221
223
 
@@ -135,7 +135,7 @@ Learn more or get a free Envio API token at: https://envio.dev/app/api-tokens`);
135
135
  blockNumber: block.number,
136
136
  blockHash: block.hash,
137
137
  logIndex: logIndex,
138
- event: {
138
+ payload: {
139
139
  contractName: eventConfig.contractName,
140
140
  eventName: eventConfig.name,
141
141
  params: params,
@@ -1200,7 +1200,7 @@ let make = (
1200
1200
  blockHash: block->getBlockHash,
1201
1201
  chain,
1202
1202
  logIndex: log.logIndex,
1203
- event: {
1203
+ payload: {
1204
1204
  contractName: eventConfig.contractName,
1205
1205
  eventName: eventConfig.name,
1206
1206
  chainId: chain->ChainMap.Chain.toChainId,
@@ -1209,7 +1209,7 @@ let make = (
1209
1209
  block,
1210
1210
  srcAddress: routedAddress,
1211
1211
  logIndex: log.logIndex,
1212
- }->Internal.fromGenericEvent,
1212
+ }->Evm.fromPayload,
1213
1213
  })
1214
1214
  }
1215
1215
  )(),
@@ -1131,7 +1131,7 @@ function make(param) {
1131
1131
  blockNumber: block.number,
1132
1132
  blockHash: block.hash,
1133
1133
  logIndex: log.logIndex,
1134
- event: {
1134
+ payload: {
1135
1135
  contractName: eventConfig.contractName,
1136
1136
  eventName: eventConfig.name,
1137
1137
  params: decoded$1,
@@ -11,16 +11,12 @@ type sourceState = {
11
11
  mutable lastFailedAt: option<float>,
12
12
  }
13
13
 
14
- // Ideally the ChainFetcher name suits this better
15
- // But currently the ChainFetcher module is immutable
16
- // and handles both processing and fetching.
17
- // So this module is to encapsulate the fetching logic only
14
+ // Encapsulates the fetching logic for a chain's sources.
18
15
  // with a mutable state for easier reasoning and testing.
19
16
  type t = {
20
17
  sourcesState: array<sourceState>,
21
18
  mutable statusStart: Hrtime.timeRef,
22
19
  mutable status: sourceManagerStatus,
23
- maxPartitionConcurrency: int,
24
20
  newBlockStallTimeout: int,
25
21
  newBlockStallTimeoutRealtime: int,
26
22
  stalledPollingInterval: int,
@@ -46,6 +42,10 @@ type t = {
46
42
 
47
43
  let getActiveSource = sourceManager => sourceManager.activeSource
48
44
 
45
+ // Partition queries currently in flight on this chain's sources. Summed across
46
+ // chains by CrossChainState to enforce the indexer-wide concurrency budget.
47
+ let inFlightCount = sourceManager => sourceManager.fetchingPartitionsCount
48
+
49
49
  let getRateLimitTimeMs = sourceManager =>
50
50
  sourceManager.committedRateLimitTimeMs +.
51
51
  switch sourceManager.activeRateLimitStartMs {
@@ -151,7 +151,6 @@ let makeGetHeightRetryInterval = (
151
151
 
152
152
  let make = (
153
153
  ~sources: array<Source.t>,
154
- ~maxPartitionConcurrency,
155
154
  ~isRealtime,
156
155
  ~newBlockStallTimeout=60_000,
157
156
  ~newBlockStallTimeoutRealtime=20_000,
@@ -172,16 +171,11 @@ let make = (
172
171
  | None =>
173
172
  JsError.throwWithMessage("Invalid configuration, no data-source for historical sync provided")
174
173
  }
175
- Prometheus.IndexingMaxConcurrency.set(
176
- ~maxConcurrency=maxPartitionConcurrency,
177
- ~chainId=initialActiveSource.chain->ChainMap.Chain.toChainId,
178
- )
179
174
  Prometheus.IndexingConcurrency.set(
180
175
  ~concurrency=0,
181
176
  ~chainId=initialActiveSource.chain->ChainMap.Chain.toChainId,
182
177
  )
183
178
  {
184
- maxPartitionConcurrency,
185
179
  sourcesState: sources->Array.map(source => {
186
180
  source,
187
181
  knownHeight: 0,
@@ -223,24 +217,20 @@ let trackNewStatus = (sourceManager: t, ~newStatus) => {
223
217
  sourceManager.status = newStatus
224
218
  }
225
219
 
226
- let fetchNext = async (
220
+ // Carry out the fetch decision made by CrossChainState.checkAndFetch: either
221
+ // dispatch the admitted queries or start waiting for a new block. Selection
222
+ // (getNextQuery + cross-chain admission) happens upstream so the budget is split
223
+ // per query across all chains.
224
+ let dispatch = async (
227
225
  sourceManager: t,
228
226
  ~fetchState: FetchState.t,
229
227
  ~executeQuery,
230
228
  ~waitForNewBlock,
231
229
  ~onNewBlock,
230
+ ~action: FetchState.nextQuery,
232
231
  ~stateId,
233
232
  ) => {
234
- let {maxPartitionConcurrency} = sourceManager
235
-
236
- let nextQuery = fetchState->FetchState.getNextQuery(
237
- ~concurrencyLimit={
238
- maxPartitionConcurrency - sourceManager.fetchingPartitionsCount
239
- },
240
- )
241
-
242
- switch nextQuery {
243
- | ReachedMaxConcurrency
233
+ switch action {
244
234
  | NothingToQuery => ()
245
235
  | WaitingForNewBlock =>
246
236
  switch sourceManager.waitingForNewBlockStateId {
@@ -261,7 +251,8 @@ let fetchNext = async (
261
251
  }
262
252
  }
263
253
  | Ready(queries) => {
264
- fetchState->FetchState.startFetchingQueries(~queries)
254
+ // Queries are already marked in flight by ChainState.startFetchingQueries
255
+ // when they were admitted; here we just execute them.
265
256
  sourceManager.fetchingPartitionsCount =
266
257
  sourceManager.fetchingPartitionsCount + queries->Array.length
267
258
  Prometheus.IndexingConcurrency.set(
@@ -18,6 +18,10 @@ function getActiveSource(sourceManager) {
18
18
  return sourceManager.activeSource;
19
19
  }
20
20
 
21
+ function inFlightCount(sourceManager) {
22
+ return sourceManager.fetchingPartitionsCount;
23
+ }
24
+
21
25
  function getRateLimitTimeMs(sourceManager) {
22
26
  let startMs = sourceManager.activeRateLimitStartMs;
23
27
  return sourceManager.committedRateLimitTimeMs + (
@@ -115,7 +119,7 @@ function makeGetHeightRetryInterval(initialRetryInterval, backoffMultiplicative,
115
119
  };
116
120
  }
117
121
 
118
- function make(sources, maxPartitionConcurrency, isRealtime, newBlockStallTimeoutOpt, newBlockStallTimeoutRealtimeOpt, stalledPollingIntervalOpt, reducedPollingIntervalOpt, recoveryTimeoutOpt, getHeightRetryIntervalOpt) {
122
+ function make(sources, isRealtime, newBlockStallTimeoutOpt, newBlockStallTimeoutRealtimeOpt, stalledPollingIntervalOpt, reducedPollingIntervalOpt, recoveryTimeoutOpt, getHeightRetryIntervalOpt) {
119
123
  let newBlockStallTimeout = newBlockStallTimeoutOpt !== undefined ? newBlockStallTimeoutOpt : 60000;
120
124
  let newBlockStallTimeoutRealtime = newBlockStallTimeoutRealtimeOpt !== undefined ? newBlockStallTimeoutRealtimeOpt : 20000;
121
125
  let stalledPollingInterval = stalledPollingIntervalOpt !== undefined ? stalledPollingIntervalOpt : 5000;
@@ -125,7 +129,6 @@ function make(sources, maxPartitionConcurrency, isRealtime, newBlockStallTimeout
125
129
  let hasRealtime = sources.some(s => s.sourceFor === "Realtime");
126
130
  let source = sources.find(source => getSourceRole(source.sourceFor, isRealtime, hasRealtime) === "Primary");
127
131
  let initialActiveSource = source !== undefined ? source : Stdlib_JsError.throwWithMessage("Invalid configuration, no data-source for historical sync provided");
128
- Prometheus.IndexingMaxConcurrency.set(maxPartitionConcurrency, initialActiveSource.chain);
129
132
  Prometheus.IndexingConcurrency.set(0, initialActiveSource.chain);
130
133
  return {
131
134
  sourcesState: sources.map(source => ({
@@ -138,7 +141,6 @@ function make(sources, maxPartitionConcurrency, isRealtime, newBlockStallTimeout
138
141
  })),
139
142
  statusStart: Hrtime.makeTimer(),
140
143
  status: "Idle",
141
- maxPartitionConcurrency: maxPartitionConcurrency,
142
144
  newBlockStallTimeout: newBlockStallTimeout,
143
145
  newBlockStallTimeoutRealtime: newBlockStallTimeoutRealtime,
144
146
  stalledPollingInterval: stalledPollingInterval,
@@ -175,33 +177,28 @@ function trackNewStatus(sourceManager, newStatus) {
175
177
  sourceManager.status = newStatus;
176
178
  }
177
179
 
178
- async function fetchNext(sourceManager, fetchState, executeQuery, waitForNewBlock, onNewBlock, stateId) {
179
- let nextQuery = FetchState.getNextQuery(fetchState, sourceManager.maxPartitionConcurrency - sourceManager.fetchingPartitionsCount | 0);
180
- if (typeof nextQuery !== "object") {
181
- switch (nextQuery) {
182
- case "WaitingForNewBlock" :
183
- let waitingStateId = sourceManager.waitingForNewBlockStateId;
184
- if (waitingStateId !== undefined && waitingStateId >= stateId) {
185
- return;
186
- }
187
- trackNewStatus(sourceManager, "WaitingForNewBlock");
188
- sourceManager.waitingForNewBlockStateId = stateId;
189
- let knownHeight = await waitForNewBlock(fetchState.knownHeight);
190
- let waitingStateId$1 = sourceManager.waitingForNewBlockStateId;
191
- if (waitingStateId$1 !== undefined && waitingStateId$1 === stateId) {
192
- trackNewStatus(sourceManager, "Idle");
193
- sourceManager.waitingForNewBlockStateId = undefined;
194
- return onNewBlock(knownHeight);
195
- } else {
196
- return;
197
- }
198
- case "ReachedMaxConcurrency" :
199
- case "NothingToQuery" :
200
- return;
180
+ async function dispatch(sourceManager, fetchState, executeQuery, waitForNewBlock, onNewBlock, action, stateId) {
181
+ if (typeof action !== "object") {
182
+ if (action !== "WaitingForNewBlock") {
183
+ return;
184
+ }
185
+ let waitingStateId = sourceManager.waitingForNewBlockStateId;
186
+ if (waitingStateId !== undefined && waitingStateId >= stateId) {
187
+ return;
188
+ }
189
+ trackNewStatus(sourceManager, "WaitingForNewBlock");
190
+ sourceManager.waitingForNewBlockStateId = stateId;
191
+ let knownHeight = await waitForNewBlock(fetchState.knownHeight);
192
+ let waitingStateId$1 = sourceManager.waitingForNewBlockStateId;
193
+ if (waitingStateId$1 !== undefined && waitingStateId$1 === stateId) {
194
+ trackNewStatus(sourceManager, "Idle");
195
+ sourceManager.waitingForNewBlockStateId = undefined;
196
+ return onNewBlock(knownHeight);
197
+ } else {
198
+ return;
201
199
  }
202
200
  } else {
203
- let queries = nextQuery._0;
204
- FetchState.startFetchingQueries(fetchState, queries);
201
+ let queries = action._0;
205
202
  sourceManager.fetchingPartitionsCount = sourceManager.fetchingPartitionsCount + queries.length | 0;
206
203
  Prometheus.IndexingConcurrency.set(sourceManager.fetchingPartitionsCount, sourceManager.activeSource.chain);
207
204
  trackNewStatus(sourceManager, "Querieng");
@@ -680,8 +677,9 @@ export {
680
677
  getSourceRole,
681
678
  make,
682
679
  getActiveSource,
680
+ inFlightCount,
683
681
  onReorg,
684
- fetchNext,
682
+ dispatch,
685
683
  waitForNewBlock,
686
684
  executeQuery,
687
685
  makeGetHeightRetryInterval,
@@ -10,7 +10,6 @@ let getSourceRole: (
10
10
 
11
11
  let make: (
12
12
  ~sources: array<Source.t>,
13
- ~maxPartitionConcurrency: int,
14
13
  ~isRealtime: bool,
15
14
  ~newBlockStallTimeout: int=?,
16
15
  ~newBlockStallTimeoutRealtime: int=?,
@@ -22,14 +21,17 @@ let make: (
22
21
 
23
22
  let getActiveSource: t => Source.t
24
23
 
24
+ let inFlightCount: t => int
25
+
25
26
  let onReorg: (t, ~rollbackTargetBlock: int) => unit
26
27
 
27
- let fetchNext: (
28
+ let dispatch: (
28
29
  t,
29
30
  ~fetchState: FetchState.t,
30
31
  ~executeQuery: FetchState.query => promise<unit>,
31
32
  ~waitForNewBlock: (~knownHeight: int) => promise<int>,
32
33
  ~onNewBlock: (~knownHeight: int) => unit,
34
+ ~action: FetchState.nextQuery,
33
35
  ~stateId: int,
34
36
  ) => promise<unit>
35
37
 
@@ -4,7 +4,7 @@ let cleanUpRawEventFieldsInPlace: JSON.t => unit = %raw(`fields => {
4
4
  delete fields.time
5
5
  }`)
6
6
 
7
- let ecosystem: Ecosystem.t = {
7
+ let make = (~logger: Pino.t): Ecosystem.t => {
8
8
  name: Svm,
9
9
  blockFields: ["slot"],
10
10
  transactionFields: [],
@@ -19,6 +19,23 @@ let ecosystem: Ecosystem.t = {
19
19
  // SVM has no event handlers, so there is no `onEvent` `where` value to
20
20
  // parse. The schema is a no-op object that always surfaces `None`.
21
21
  onEventBlockFilterSchema: S.object(_ => None),
22
+ logger,
23
+ toEvent: eventItem => eventItem.payload->Internal.payloadToEvent,
24
+ toEventLogger: eventItem => {
25
+ let instruction =
26
+ eventItem.payload->(Utils.magic: Internal.eventPayload => Envio.svmInstruction)
27
+ Logging.createChildFrom(
28
+ ~logger,
29
+ ~params={
30
+ "program": eventItem.eventConfig.contractName,
31
+ "instruction": eventItem.eventConfig.name,
32
+ "chainId": eventItem.chain->ChainMap.Chain.toChainId,
33
+ "slot": eventItem.blockNumber,
34
+ "programId": instruction.programId,
35
+ },
36
+ )
37
+ },
38
+ toRawEvent: _ => JsError.throwWithMessage("Raw events are not supported for SVM"),
22
39
  }
23
40
 
24
41
  module GetFinalizedSlot = {
@@ -4,6 +4,7 @@ import * as Rpc from "./Rpc.res.mjs";
4
4
  import * as Rest from "../vendored/Rest.res.mjs";
5
5
  import * as Utils from "../Utils.res.mjs";
6
6
  import * as Hrtime from "../bindings/Hrtime.res.mjs";
7
+ import * as Logging from "../Logging.res.mjs";
7
8
  import * as Prometheus from "../Prometheus.res.mjs";
8
9
  import * as Stdlib_JsError from "@rescript/runtime/lib/es6/Stdlib_JsError.js";
9
10
  import * as S$RescriptSchema from "rescript-schema/src/S.res.mjs";
@@ -14,26 +15,33 @@ let cleanUpRawEventFieldsInPlace = (fields => {
14
15
  delete fields.time
15
16
  });
16
17
 
17
- let ecosystem_blockFields = ["slot"];
18
-
19
- let ecosystem_transactionFields = [];
20
-
21
- let ecosystem_onBlockFilterSchema = S$RescriptSchema.object(s => s.f("slot", S$RescriptSchema.option(S$RescriptSchema.unknown)));
22
-
23
- let ecosystem_onEventBlockFilterSchema = S$RescriptSchema.object(param => {});
24
-
25
- let ecosystem = {
26
- name: "svm",
27
- blockFields: ecosystem_blockFields,
28
- transactionFields: ecosystem_transactionFields,
29
- blockNumberName: "height",
30
- blockTimestampName: "time",
31
- blockHashName: "hash",
32
- cleanUpRawEventFieldsInPlace: cleanUpRawEventFieldsInPlace,
33
- onBlockMethodName: "onSlot",
34
- onBlockFilterSchema: ecosystem_onBlockFilterSchema,
35
- onEventBlockFilterSchema: ecosystem_onEventBlockFilterSchema
36
- };
18
+ function make(logger) {
19
+ return {
20
+ name: "svm",
21
+ blockFields: ["slot"],
22
+ transactionFields: [],
23
+ blockNumberName: "height",
24
+ blockTimestampName: "time",
25
+ blockHashName: "hash",
26
+ cleanUpRawEventFieldsInPlace: cleanUpRawEventFieldsInPlace,
27
+ onBlockMethodName: "onSlot",
28
+ onBlockFilterSchema: S$RescriptSchema.object(s => s.f("slot", S$RescriptSchema.option(S$RescriptSchema.unknown))),
29
+ onEventBlockFilterSchema: S$RescriptSchema.object(param => {}),
30
+ logger: logger,
31
+ toEvent: eventItem => eventItem.payload,
32
+ toEventLogger: eventItem => {
33
+ let instruction = eventItem.payload;
34
+ return Logging.createChildFrom(logger, {
35
+ program: eventItem.eventConfig.contractName,
36
+ instruction: eventItem.eventConfig.name,
37
+ chainId: eventItem.chain,
38
+ slot: eventItem.blockNumber,
39
+ programId: instruction.programId
40
+ });
41
+ },
42
+ toRawEvent: param => Stdlib_JsError.throwWithMessage("Raw events are not supported for SVM")
43
+ };
44
+ }
37
45
 
38
46
  let route = Rpc.makeRpcRoute("getSlot", S$RescriptSchema.tuple(s => {
39
47
  s.tag(0, {
@@ -72,8 +80,8 @@ function makeRPCSource(chain, rpc, sourceForOpt) {
72
80
 
73
81
  export {
74
82
  cleanUpRawEventFieldsInPlace,
75
- ecosystem,
83
+ make,
76
84
  GetFinalizedSlot,
77
85
  makeRPCSource,
78
86
  }
79
- /* ecosystem Not a pure module */
87
+ /* route Not a pure module */
@@ -510,7 +510,7 @@ let make = ({chain, endpointUrl, apiToken, eventConfigs, clientTimeoutMillis}: o
510
510
  blockNumber: instr.slot,
511
511
  blockHash: "",
512
512
  logIndex: synthLogIndex(instr),
513
- event: payload->(Utils.magic: Envio.svmInstruction => Internal.event),
513
+ payload: payload->(Utils.magic: Envio.svmInstruction => Internal.eventPayload),
514
514
  }),
515
515
  )
516
516
  ->ignore
@@ -420,7 +420,7 @@ function make(param) {
420
420
  blockNumber: instr.slot,
421
421
  blockHash: "",
422
422
  logIndex: synthLogIndex(instr),
423
- event: payload
423
+ payload: payload
424
424
  });
425
425
  }
426
426
  });