envio 3.3.0-alpha.0 → 3.3.0-alpha.1

package/src/Prometheus.res.mjs

@@ -400,42 +400,28 @@ let IndexingAddresses = {
   set: set$4
 };
 
-let gauge$5 = PromClient.Gauge.makeGauge({
-  name: "envio_indexing_max_concurrency",
-  help: "The maximum number of concurrent data-source queries across the whole indexer."
-});
+let gauge$5 = makeOrThrow$1("envio_indexing_concurrency", "The number of executing concurrent queries to the chain data-source.", chainIdLabelsSchema);
 
-function set$5(maxConcurrency) {
-  gauge$5.set(maxConcurrency);
+function set$5(concurrency, chainId) {
+  handleInt$1(gauge$5, chainId, concurrency);
 }
 
-let IndexingMaxConcurrency = {
+let IndexingConcurrency = {
   gauge: gauge$5,
   set: set$5
 };
 
-let gauge$6 = makeOrThrow$1("envio_indexing_concurrency", "The number of executing concurrent queries to the chain data-source.", chainIdLabelsSchema);
+let gauge$6 = makeOrThrow$1("envio_indexing_partitions", "The number of partitions used to split fetching logic by addresses and block ranges.", chainIdLabelsSchema);
 
-function set$6(concurrency, chainId) {
-  handleInt$1(gauge$6, chainId, concurrency);
+function set$6(partitionsCount, chainId) {
+  handleInt$1(gauge$6, chainId, partitionsCount);
 }
 
-let IndexingConcurrency = {
+let IndexingPartitions = {
   gauge: gauge$6,
   set: set$6
 };
 
-let gauge$7 = makeOrThrow$1("envio_indexing_partitions", "The number of partitions used to split fetching logic by addresses and block ranges.", chainIdLabelsSchema);
-
-function set$7(partitionsCount, chainId) {
-  handleInt$1(gauge$7, chainId, partitionsCount);
-}
-
-let IndexingPartitions = {
-  gauge: gauge$7,
-  set: set$7
-};
-
 let counter = makeOrThrow("envio_indexing_idle_seconds", "The time the indexer source syncing has been idle. A high value may indicate the source sync is a bottleneck.", chainIdLabelsSchema);
 
 let IndexingIdleTime = {
@@ -454,51 +440,51 @@ let IndexingQueryTime = {
   counter: counter$2
 };
 
-let gauge$8 = makeOrThrow$1("envio_indexing_buffer_size", "The current number of items in the indexing buffer.", chainIdLabelsSchema);
+let gauge$7 = makeOrThrow$1("envio_indexing_buffer_size", "The current number of items in the indexing buffer.", chainIdLabelsSchema);
 
-function set$8(bufferSize, chainId) {
-  handleInt$1(gauge$8, chainId, bufferSize);
+function set$7(bufferSize, chainId) {
+  handleInt$1(gauge$7, chainId, bufferSize);
 }
 
 let IndexingBufferSize = {
-  gauge: gauge$8,
-  set: set$8
+  gauge: gauge$7,
+  set: set$7
 };
 
-let gauge$9 = PromClient.Gauge.makeGauge({
+let gauge$8 = PromClient.Gauge.makeGauge({
   name: "envio_indexing_target_buffer_size",
   help: "The indexer-wide target buffer size shared across all chains. The actual number of items in the queue may exceed this value, but the indexer always tries to keep the buffer filled up to this target."
 });
 
-function set$9(targetBufferSize) {
-  gauge$9.set(targetBufferSize);
+function set$8(targetBufferSize) {
+  gauge$8.set(targetBufferSize);
 }
 
 let IndexingTargetBufferSize = {
-  gauge: gauge$9,
-  set: set$9
+  gauge: gauge$8,
+  set: set$8
 };
 
-let gauge$10 = makeOrThrow$1("envio_indexing_buffer_block", "The highest block number that has been fully fetched by the indexer.", chainIdLabelsSchema);
+let gauge$9 = makeOrThrow$1("envio_indexing_buffer_block", "The highest block number that has been fully fetched by the indexer.", chainIdLabelsSchema);
 
-function set$10(blockNumber, chainId) {
-  handleInt$1(gauge$10, chainId, blockNumber);
+function set$9(blockNumber, chainId) {
+  handleInt$1(gauge$9, chainId, blockNumber);
 }
 
 let IndexingBufferBlockNumber = {
-  gauge: gauge$10,
-  set: set$10
+  gauge: gauge$9,
+  set: set$9
 };
 
-let gauge$11 = makeOrThrow$1("envio_indexing_end_block", "The block number to stop indexing at. (inclusive)", chainIdLabelsSchema);
+let gauge$10 = makeOrThrow$1("envio_indexing_end_block", "The block number to stop indexing at. (inclusive)", chainIdLabelsSchema);
 
-function set$11(endBlock, chainId) {
-  handleInt$1(gauge$11, chainId, endBlock);
+function set$10(endBlock, chainId) {
+  handleInt$1(gauge$10, chainId, endBlock);
 }
 
 let IndexingEndBlock = {
-  gauge: gauge$11,
-  set: set$11
+  gauge: gauge$10,
+  set: set$10
 };
 
 let sourceLabelsSchema = S$RescriptSchema.schema(s => ({
@@ -539,18 +525,18 @@ let SourceRequestCount = {
   addSeconds: addSeconds
 };
 
-let gauge$12 = makeOrThrow$1("envio_source_known_height", "The latest known block number reported by the source. This value may lag behind the actual chain height, as it is updated only when queried.", sourceLabelsSchema);
+let gauge$11 = makeOrThrow$1("envio_source_known_height", "The latest known block number reported by the source. This value may lag behind the actual chain height, as it is updated only when queried.", sourceLabelsSchema);
 
-function set$12(sourceName, chainId, blockNumber) {
-  handleInt$1(gauge$12, {
+function set$11(sourceName, chainId, blockNumber) {
+  handleInt$1(gauge$11, {
     source: sourceName,
     chainId: chainId
   }, blockNumber);
 }
 
 let SourceHeight = {
-  gauge: gauge$12,
-  set: set$12
+  gauge: gauge$11,
+  set: set$11
 };
 
 let counter$4 = makeOrThrow("envio_reorg_detected_total", "Total number of reorgs detected", chainIdLabelsSchema);
@@ -564,43 +550,43 @@ let ReorgCount = {
   increment: increment$5
 };
 
-let gauge$13 = makeOrThrow$1("envio_reorg_detected_block", "The block number where reorg was detected the last time. This doesn't mean that the block was reorged, this is simply where we found block hash to be different.", chainIdLabelsSchema);
+let gauge$12 = makeOrThrow$1("envio_reorg_detected_block", "The block number where reorg was detected the last time. This doesn't mean that the block was reorged, this is simply where we found block hash to be different.", chainIdLabelsSchema);
 
-function set$13(blockNumber, chain) {
-  handleInt$1(gauge$13, chain, blockNumber);
+function set$12(blockNumber, chain) {
+  handleInt$1(gauge$12, chain, blockNumber);
 }
 
 let ReorgDetectionBlockNumber = {
-  gauge: gauge$13,
-  set: set$13
+  gauge: gauge$12,
+  set: set$12
 };
 
-let gauge$14 = PromClient.Gauge.makeGauge({
+let gauge$13 = PromClient.Gauge.makeGauge({
   name: "envio_reorg_threshold",
   help: "Whether indexing is currently within the reorg threshold"
 });
 
-function set$14(isInReorgThreshold) {
-  gauge$14.set(isInReorgThreshold ? 1 : 0);
+function set$13(isInReorgThreshold) {
+  gauge$13.set(isInReorgThreshold ? 1 : 0);
 }
 
 let ReorgThreshold = {
-  gauge: gauge$14,
-  set: set$14
+  gauge: gauge$13,
+  set: set$13
 };
 
-let gauge$15 = PromClient.Gauge.makeGauge({
+let gauge$14 = PromClient.Gauge.makeGauge({
   name: "envio_rollback_enabled",
   help: "Whether rollback on reorg is enabled"
 });
 
-function set$15(enabled) {
-  gauge$15.set(enabled ? 1 : 0);
+function set$14(enabled) {
+  gauge$14.set(enabled ? 1 : 0);
 }
 
 let RollbackEnabled = {
-  gauge: gauge$15,
-  set: set$15
+  gauge: gauge$14,
+  set: set$14
 };
 
 let timeCounter$3 = PromClient.Counter.makeCounter({
@@ -649,62 +635,62 @@ let RollbackHistoryPrune = {
   increment: increment$7
 };
 
-let gauge$16 = makeOrThrow$1("envio_rollback_target_block", "The block number reorg was rollbacked to the last time.", chainIdLabelsSchema);
+let gauge$15 = makeOrThrow$1("envio_rollback_target_block", "The block number reorg was rollbacked to the last time.", chainIdLabelsSchema);
 
-function set$16(blockNumber, chain) {
-  handleInt$1(gauge$16, chain, blockNumber);
+function set$15(blockNumber, chain) {
+  handleInt$1(gauge$15, chain, blockNumber);
 }
 
 let RollbackTargetBlockNumber = {
-  gauge: gauge$16,
-  set: set$16
+  gauge: gauge$15,
+  set: set$15
 };
 
-let gauge$17 = PromClient.Gauge.makeGauge({
+let gauge$16 = PromClient.Gauge.makeGauge({
   name: "envio_processing_max_batch_size",
   help: "The maximum number of items to process in a single batch."
 });
 
-function set$17(maxBatchSize) {
-  gauge$17.set(maxBatchSize);
+function set$16(maxBatchSize) {
+  gauge$16.set(maxBatchSize);
 }
 
 let ProcessingMaxBatchSize = {
-  gauge: gauge$17,
-  set: set$17
+  gauge: gauge$16,
+  set: set$16
 };
 
-let gauge$18 = makeOrThrow$1("envio_progress_block", "The block number of the latest block processed and stored in the database.", chainIdLabelsSchema);
+let gauge$17 = makeOrThrow$1("envio_progress_block", "The block number of the latest block processed and stored in the database.", chainIdLabelsSchema);
 
-function set$18(blockNumber, chainId) {
-  handleInt$1(gauge$18, chainId, blockNumber);
+function set$17(blockNumber, chainId) {
+  handleInt$1(gauge$17, chainId, blockNumber);
 }
 
 let ProgressBlockNumber = {
-  gauge: gauge$18,
-  set: set$18
+  gauge: gauge$17,
+  set: set$17
 };
 
-let gauge$19 = makeOrThrow$1("envio_progress_events", "The number of events processed and reflected in the database.", chainIdLabelsSchema);
+let gauge$18 = makeOrThrow$1("envio_progress_events", "The number of events processed and reflected in the database.", chainIdLabelsSchema);
 
-function set$19(processedCount, chainId) {
-  handleFloat$1(gauge$19, chainId, processedCount);
+function set$18(processedCount, chainId) {
+  handleFloat$1(gauge$18, chainId, processedCount);
 }
 
 let ProgressEventsCount = {
-  gauge: gauge$19,
-  set: set$19
+  gauge: gauge$18,
+  set: set$18
 };
 
-let gauge$20 = makeOrThrow$1("envio_progress_latency", "The latency in milliseconds between the latest processed event creation and the time it was written to storage.", chainIdLabelsSchema);
+let gauge$19 = makeOrThrow$1("envio_progress_latency", "The latency in milliseconds between the latest processed event creation and the time it was written to storage.", chainIdLabelsSchema);
 
-function set$20(latencyMs, chainId) {
-  handleInt$1(gauge$20, chainId, latencyMs);
+function set$19(latencyMs, chainId) {
+  handleInt$1(gauge$19, chainId, latencyMs);
 }
 
 let ProgressLatency = {
-  gauge: gauge$20,
-  set: set$20
+  gauge: gauge$19,
+  set: set$19
 };
 
 let effectLabelsSchema = S$RescriptSchema.object(s => s.f("effect", S$RescriptSchema.string));
@@ -724,15 +710,15 @@ let EffectCalls = {
   activeCallsCount: activeCallsCount
 };
 
-let gauge$21 = makeOrThrow$1("envio_effect_cache", "The number of items in the effect cache.", effectLabelsSchema);
+let gauge$20 = makeOrThrow$1("envio_effect_cache", "The number of items in the effect cache.", effectLabelsSchema);
 
-function set$21(count, effectName) {
-  handleInt$1(gauge$21, effectName, count);
+function set$20(count, effectName) {
+  handleInt$1(gauge$20, effectName, count);
 }
 
 let EffectCacheCount = {
-  gauge: gauge$21,
-  set: set$21
+  gauge: gauge$20,
+  set: set$20
 };
 
 let counter$7 = makeOrThrow("envio_effect_cache_invalidations", "The number of effect cache invalidations.", effectLabelsSchema);
@@ -746,18 +732,18 @@ let EffectCacheInvalidationsCount = {
   increment: increment$8
 };
 
-let gauge$22 = makeOrThrow$1("envio_effect_queue", "The number of effect calls waiting in the rate limit queue.", effectLabelsSchema);
+let gauge$21 = makeOrThrow$1("envio_effect_queue", "The number of effect calls waiting in the rate limit queue.", effectLabelsSchema);
 
 let timeCounter$6 = makeOrThrow("envio_effect_queue_wait_seconds", "The time spent waiting in the rate limit queue.", effectLabelsSchema);
 
-function set$22(count, effectName) {
-  handleInt$1(gauge$22, effectName, count);
+function set$21(count, effectName) {
+  handleInt$1(gauge$21, effectName, count);
 }
 
 let EffectQueueCount = {
-  gauge: gauge$22,
+  gauge: gauge$21,
   timeCounter: timeCounter$6,
-  set: set$22
+  set: set$21
 };
 
 let loadLabelsSchema = S$RescriptSchema.schema(s => ({
@@ -861,7 +847,6 @@ export {
   Info,
   ProcessStartTimeSeconds,
   IndexingAddresses,
-  IndexingMaxConcurrency,
   IndexingConcurrency,
   IndexingPartitions,
   IndexingIdleTime,

package/src/sources/SourceManager.res

@@ -217,20 +217,20 @@ let trackNewStatus = (sourceManager: t, ~newStatus) => {
   sourceManager.status = newStatus
 }
 
-let fetchNext = async (
+// Carry out the fetch decision made by CrossChainState.checkAndFetch: either
+// dispatch the admitted queries or start waiting for a new block. Selection
+// (getNextQuery + cross-chain admission) happens upstream so the budget is split
+// per query across all chains.
+let dispatch = async (
   sourceManager: t,
   ~fetchState: FetchState.t,
   ~executeQuery,
   ~waitForNewBlock,
   ~onNewBlock,
-  ~concurrencyLimit,
-  ~bufferLimit,
+  ~action: FetchState.nextQuery,
   ~stateId,
 ) => {
-  let nextQuery = fetchState->FetchState.getNextQuery(~concurrencyLimit, ~bufferLimit)
-
-  switch nextQuery {
-  | ReachedMaxConcurrency
+  switch action {
   | NothingToQuery => ()
   | WaitingForNewBlock =>
     switch sourceManager.waitingForNewBlockStateId {
@@ -251,7 +251,8 @@ let fetchNext = async (
       }
     }
   | Ready(queries) => {
-      fetchState->FetchState.startFetchingQueries(~queries)
+      // Queries are already marked in flight by ChainState.startFetchingQueries
+      // when they were admitted; here we just execute them.
       sourceManager.fetchingPartitionsCount =
         sourceManager.fetchingPartitionsCount + queries->Array.length
       Prometheus.IndexingConcurrency.set(

package/src/sources/SourceManager.res.mjs

@@ -177,33 +177,28 @@ function trackNewStatus(sourceManager, newStatus) {
   sourceManager.status = newStatus;
 }
 
-async function fetchNext(sourceManager, fetchState, executeQuery, waitForNewBlock, onNewBlock, concurrencyLimit, bufferLimit, stateId) {
-  let nextQuery = FetchState.getNextQuery(fetchState, concurrencyLimit, bufferLimit);
-  if (typeof nextQuery !== "object") {
-    switch (nextQuery) {
-      case "WaitingForNewBlock" :
-        let waitingStateId = sourceManager.waitingForNewBlockStateId;
-        if (waitingStateId !== undefined && waitingStateId >= stateId) {
-          return;
-        }
-        trackNewStatus(sourceManager, "WaitingForNewBlock");
-        sourceManager.waitingForNewBlockStateId = stateId;
-        let knownHeight = await waitForNewBlock(fetchState.knownHeight);
-        let waitingStateId$1 = sourceManager.waitingForNewBlockStateId;
-        if (waitingStateId$1 !== undefined && waitingStateId$1 === stateId) {
-          trackNewStatus(sourceManager, "Idle");
-          sourceManager.waitingForNewBlockStateId = undefined;
-          return onNewBlock(knownHeight);
-        } else {
-          return;
-        }
-      case "ReachedMaxConcurrency" :
-      case "NothingToQuery" :
-        return;
+async function dispatch(sourceManager, fetchState, executeQuery, waitForNewBlock, onNewBlock, action, stateId) {
+  if (typeof action !== "object") {
+    if (action !== "WaitingForNewBlock") {
+      return;
+    }
+    let waitingStateId = sourceManager.waitingForNewBlockStateId;
+    if (waitingStateId !== undefined && waitingStateId >= stateId) {
+      return;
+    }
+    trackNewStatus(sourceManager, "WaitingForNewBlock");
+    sourceManager.waitingForNewBlockStateId = stateId;
+    let knownHeight = await waitForNewBlock(fetchState.knownHeight);
+    let waitingStateId$1 = sourceManager.waitingForNewBlockStateId;
+    if (waitingStateId$1 !== undefined && waitingStateId$1 === stateId) {
+      trackNewStatus(sourceManager, "Idle");
+      sourceManager.waitingForNewBlockStateId = undefined;
+      return onNewBlock(knownHeight);
+    } else {
+      return;
     }
   } else {
-    let queries = nextQuery._0;
-    FetchState.startFetchingQueries(fetchState, queries);
+    let queries = action._0;
     sourceManager.fetchingPartitionsCount = sourceManager.fetchingPartitionsCount + queries.length | 0;
     Prometheus.IndexingConcurrency.set(sourceManager.fetchingPartitionsCount, sourceManager.activeSource.chain);
     trackNewStatus(sourceManager, "Querieng");
@@ -684,7 +679,7 @@ export {
   getActiveSource,
   inFlightCount,
   onReorg,
-  fetchNext,
+  dispatch,
   waitForNewBlock,
   executeQuery,
   makeGetHeightRetryInterval,

package/src/sources/SourceManager.resi

@@ -25,14 +25,13 @@ let inFlightCount: t => int
 
 let onReorg: (t, ~rollbackTargetBlock: int) => unit
 
-let fetchNext: (
+let dispatch: (
   t,
   ~fetchState: FetchState.t,
   ~executeQuery: FetchState.query => promise<unit>,
   ~waitForNewBlock: (~knownHeight: int) => promise<int>,
   ~onNewBlock: (~knownHeight: int) => unit,
-  ~concurrencyLimit: int,
-  ~bufferLimit: int,
+  ~action: FetchState.nextQuery,
   ~stateId: int,
 ) => promise<unit>