envio 3.3.0-alpha.11 → 3.3.0-alpha.13

package/src/EffectState.res.mjs

@@ -0,0 +1,74 @@
+// Generated by ReScript, PLEASE EDIT WITH CARE
+
+import * as Utils from "./Utils.res.mjs";
+import * as Internal from "./Internal.res.mjs";
+
+function make() {
+  return {
+    tables: {},
+    rateLimits: {}
+  };
+}
+
+function getRateLimitState(self, tableName, effect) {
+  let match = effect.rateLimit;
+  if (match === undefined) {
+    return;
+  }
+  let callsPerDuration = match.callsPerDuration;
+  let existing = self.rateLimits[tableName];
+  let tmp;
+  if (existing !== undefined) {
+    tmp = existing;
+  } else {
+    let created = {
+      callsPerDuration: callsPerDuration,
+      durationMs: match.durationMs,
+      availableCalls: callsPerDuration,
+      windowStartTime: Date.now(),
+      queueCount: 0,
+      nextWindowPromise: undefined
+    };
+    self.rateLimits[tableName] = created;
+    tmp = created;
+  }
+  return tmp;
+}
+
+function getTable(self, effect, scope) {
+  let tableName = Internal.EffectCache.toTableName(effect.name, scope);
+  let inMemTable = self.tables[tableName];
+  if (inMemTable !== undefined) {
+    return inMemTable;
+  }
+  let inMemTable$1 = {
+    idsToStore: [],
+    invalidationsCount: 0,
+    dict: {},
+    changesCount: 0,
+    effect: effect,
+    scope: scope,
+    table: Internal.makeCacheTable(effect.name, scope),
+    rateLimitState: getRateLimitState(self, tableName, effect),
+    activeCallsCount: 0,
+    prevCallStartTimerRef: null
+  };
+  self.tables[tableName] = inMemTable$1;
+  return inMemTable$1;
+}
+
+function forEach(self, fn) {
+  Utils.Dict.forEach(self.tables, fn);
+}
+
+function resetForRollback(self) {
+  self.tables = {};
+}
+
+export {
+  make,
+  getTable,
+  forEach,
+  resetForRollback,
+}
+/* Utils Not a pure module */

package/src/EffectState.resi

@@ -0,0 +1,32 @@
+type effectRateLimitState = {
+  callsPerDuration: int,
+  durationMs: int,
+  mutable availableCalls: int,
+  mutable windowStartTime: float,
+  mutable queueCount: int,
+  mutable nextWindowPromise: option<promise<unit>>,
+}
+
+type effectCacheInMemTable = {
+  mutable idsToStore: array<string>,
+  mutable invalidationsCount: int,
+  mutable dict: dict<Change.t<Internal.effectOutput>>,
+  mutable changesCount: float,
+  effect: Internal.effect,
+  scope: Internal.chainScope,
+  table: Table.table,
+  rateLimitState: option<effectRateLimitState>,
+  mutable activeCallsCount: int,
+  mutable prevCallStartTimerRef: Performance.timeRef,
+}
+
+// Opaque store of all per-(effect, scope) runtime state.
+type t
+
+let make: unit => t
+// Get, or lazily create, the in-mem table for an (effect, scope). Reuses the
+// surviving rate-limit window when the table is recreated after a rollback.
+let getTable: (t, ~effect: Internal.effect, ~scope: Internal.chainScope) => effectCacheInMemTable
+let forEach: (t, effectCacheInMemTable => unit) => unit
+// Drop the cache tables but keep the rate-limit windows across a rollback.
+let resetForRollback: t => unit

package/src/Envio.res

@@ -186,11 +186,19 @@ and effectOptions<'input, 'output> = {
   rateLimit: rateLimit,
   /** Whether the effect should be cached. */
   cache?: bool,
+  /** Whether the effect's cache is shared across all chains. Defaults to `true`.
+   Set to `false` to isolate the cache (and rate limiting) per chain and enable
+   `context.chain.id` inside the handler. */
+  crossChain?: bool,
 }
+and effectChain = {id: int}
 and effectContext = {
   log: logger,
   effect: 'input 'output. (effect<'input, 'output>, 'input) => promise<'output>,
   mutable cache: bool,
+  /** The chain the effect was called on. Only available on effects created with
+   `crossChain: false`; accessing it on a cross-chain effect throws. */
+  chain: effectChain,
 }
 and effectArgs<'input> = {
   input: 'input,
@@ -205,10 +213,23 @@ let durationToMs = (duration: rateLimitDuration) =>
   | Milliseconds(ms) => ms
   }
 
+// The name becomes both a Postgres cache-table suffix and a .envio/cache file
+// path segment. Path separators are excluded from the charset and a leading dot
+// is disallowed, so a name can never be "." / ".." or otherwise traverse out of
+// the cache dir; dots elsewhere are fine and keep existing names like
+// "token.metadata" working, since the (name, scope) <-> table <-> path mapping
+// stays reversible.
+let effectNameRe = /^[A-Za-z0-9_-][A-Za-z0-9_.-]*$/
+
 let createEffect = (
   options: effectOptions<'input, 'output>,
   handler: effectArgs<'input> => promise<'output>,
 ) => {
+  if !(effectNameRe->RegExp.test(options.name)) {
+    JsError.throwWithMessage(
+      `Invalid effect name "${options.name}". Effect names may contain letters, numbers, underscores, hyphens and dots (but must not start with a dot or contain a path separator), because the name is used as the cache table name and cache file path.`,
+    )
+  }
   let outputSchema =
     S.schema(_ => options.output)->(Utils.magic: S.t<S.t<'output>> => S.t<Internal.effectOutput>)
   let itemSchema = S.schema((s): Internal.effectCacheItem => {
@@ -222,8 +243,6 @@ let createEffect = (
         Internal.effectOutput,
       >
     ),
-    activeCallsCount: 0,
-    prevCallStartTimerRef: %raw(`null`),
     // This is the way to make the createEffect API
     // work without the need for users to call S.schema themselves,
     // but simply pass the desired object/tuple/etc.
@@ -233,7 +252,6 @@ let createEffect = (
     ),
     output: outputSchema,
     storageMeta: {
-      table: Internal.makeCacheTable(~effectName=options.name),
       outputSchema,
       itemSchema,
     },
@@ -241,16 +259,16 @@ let createEffect = (
     | Some(true) => true
     | _ => false
     },
+    crossChain: switch options.crossChain {
+    | Some(false) => false
+    | _ => true
+    },
     rateLimit: switch options.rateLimit {
     | Disable => None
     | Enable({calls, per}) =>
       Some({
         callsPerDuration: calls,
         durationMs: per->durationToMs,
-        availableCalls: calls,
-        windowStartTime: Date.now(),
-        queueCount: 0,
-        nextWindowPromise: None,
       })
     },
   }->(Utils.magic: Internal.effect => effect<'input, 'output>)

package/src/Envio.res.mjs

@@ -2,8 +2,8 @@
 
 import * as Address from "./Address.res.mjs";
 import * as Process from "process";
-import * as Internal from "./Internal.res.mjs";
 import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js";
+import * as Stdlib_JsError from "@rescript/runtime/lib/es6/Stdlib_JsError.js";
 import * as S$RescriptSchema from "rescript-schema/src/S.res.mjs";
 
 function durationToMs(duration) {
@@ -18,41 +18,36 @@ function durationToMs(duration) {
   }
 }
 
+let effectNameRe = /^[A-Za-z0-9_-][A-Za-z0-9_.-]*$/;
+
 function createEffect(options, handler) {
+  if (!effectNameRe.test(options.name)) {
+    Stdlib_JsError.throwWithMessage(`Invalid effect name "` + options.name + `". Effect names may contain letters, numbers, underscores, hyphens and dots (but must not start with a dot or contain a path separator), because the name is used as the cache table name and cache file path.`);
+  }
   let outputSchema = S$RescriptSchema.schema(param => options.output);
   let itemSchema = S$RescriptSchema.schema(s => ({
     id: s.m(S$RescriptSchema.string),
     output: s.m(outputSchema)
   }));
   let match = options.cache;
-  let match$1 = options.rateLimit;
+  let match$1 = options.crossChain;
+  let match$2 = options.rateLimit;
   let tmp;
-  if (match$1 === false) {
-    tmp = undefined;
-  } else {
-    let calls = match$1.calls;
-    tmp = {
-      callsPerDuration: calls,
-      durationMs: durationToMs(match$1.per),
-      availableCalls: calls,
-      windowStartTime: Date.now(),
-      queueCount: 0,
-      nextWindowPromise: undefined
-    };
-  }
+  tmp = match$2 === false ? undefined : ({
+      callsPerDuration: match$2.calls,
+      durationMs: durationToMs(match$2.per)
+    });
   return {
     name: options.name,
     handler: handler,
     storageMeta: {
       itemSchema: itemSchema,
-      outputSchema: outputSchema,
-      table: Internal.makeCacheTable(options.name)
+      outputSchema: outputSchema
     },
     defaultShouldCache: match !== undefined ? match : false,
+    crossChain: match$1 !== undefined ? match$1 : true,
     output: outputSchema,
     input: S$RescriptSchema.schema(param => options.input),
-    activeCallsCount: 0,
-    prevCallStartTimerRef: null,
     rateLimit: tmp
   };
 }
@@ -102,6 +97,7 @@ let TestHelpers = {
 
 export {
   durationToMs,
+  effectNameRe,
   createEffect,
   isNonInteractive,
   TestHelpers,

package/src/FetchState.res

@@ -47,17 +47,19 @@ type partition = {
   dynamicContract: option<string>,
   // Mutable array for SourceManager sync - queries exist only while being fetched
   mutPendingQueries: array<pendingQuery>,
-  // Track last 3 successful query ranges for chunking heuristic (0 means no data)
-  prevQueryRange: int,
-  prevPrevQueryRange: int,
-  // Item count of the response that set prevQueryRange. Paired with it to derive
-  // the partition's event density (prevRangeSize / prevQueryRange) for sizing
-  // queries against the buffer budget.
-  prevRangeSize: int,
+  // The last two ranges that measured how many blocks the source could return.
+  // Both must be non-zero before the minimum is trusted for chunk sizing.
+  sourceRangeCapacity: int,
+  prevSourceRangeCapacity: int,
+  // Smoothed items/block observed in responses. This is independent from the
+  // source's range capacity: even a response truncated by our own itemsTarget
+  // cap is useful density evidence while saying nothing about source capacity.
+  // None distinguishes a new partition from a real zero-density observation.
+  eventDensity: option<float>,
   // Tracks the latestFetchedBlock.blockNumber of the most recent response
-  // that updated prevQueryRange. Prevents degradation of the chunking heuristic
-  // when parallel query responses arrive out of order.
-  latestBlockRangeUpdateBlock: int,
+  // that updated sourceRangeCapacity. Prevents degradation of the chunking
+  // heuristic when parallel query responses arrive out of order.
+  latestSourceRangeCapacityUpdateBlock: int,
 }
 
 type query = {
@@ -111,22 +113,17 @@ let densityItemsTarget = (~density, ~fromBlock, ~toBlock, ~chainTargetBlock) =>
   )
 }
 
-// Calculate the chunk range from history using min-of-last-3-ranges heuristic
+// Calculate the chunk range from the last two source-capacity observations.
 let getMinHistoryRange = (p: partition) => {
-  switch (p.prevQueryRange, p.prevPrevQueryRange) {
+  switch (p.sourceRangeCapacity, p.prevSourceRangeCapacity) {
   | (0, _) | (_, 0) => None
   | (a, b) => Some(a < b ? a : b)
   }
 }
 
-// Density (items/block) from the last response, trusted only after two
-// responses — a single sample is too noisy to size the next query by.
-let getTrustedDensity = (p: partition) => {
-  switch (p.prevQueryRange, p.prevPrevQueryRange) {
-  | (0, _) | (_, 0) => None
-  | (prevQueryRange, _) => Some(p.prevRangeSize->Int.toFloat /. prevQueryRange->Int.toFloat)
-  }
-}
+// Density has its own initialization signal and is useful independently from
+// source-capacity history, including when an itemsTarget cap truncates a response.
+let getTrustedDensity = (p: partition) => p.eventDensity
 
 let getMinQueryRange = (partitions: array<partition>) => {
   let min = ref(0)
@@ -134,8 +131,8 @@ let getMinQueryRange = (partitions: array<partition>) => {
     let p = partitions->Array.getUnsafe(i)
 
     // Even if it's fetching, set dynamicContract field
-    let a = p.prevQueryRange
-    let b = p.prevPrevQueryRange
+    let a = p.sourceRangeCapacity
+    let b = p.prevSourceRangeCapacity
     if a > 0 && (min.contents == 0 || a < min.contents) {
       min := a
     }
@@ -208,11 +205,14 @@ module OptimizedPartitions = {
       nextPartitionIndexRef := nextPartitionIndexRef.contents + 1
       let minRange = getMinQueryRange([p1, p2])
       // The merged partition indexes both parents' addresses, so its expected
-      // event rate is the sum of their densities. Parents without a trusted
-      // density contribute 0; if none has one, prevRangeSize stays 0 and the
-      // partition probes for a fresh signal instead of chunking.
-      let inheritedDensity =
-        p1->getTrustedDensity->Option.getOr(0.) +. p2->getTrustedDensity->Option.getOr(0.)
+      // event rate is the sum of their densities. If neither parent has a
+      // trusted density, the merged partition probes for a fresh signal
+      // instead of treating zero as an observed density.
+      let inheritedDensity = switch (p1->getTrustedDensity, p2->getTrustedDensity) {
+      | (Some(p1Density), Some(p2Density)) => Some(p1Density +. p2Density)
+      | (Some(density), None) | (None, Some(density)) => Some(density)
+      | (None, None) => None
+      }
       {
         id: newId,
         dynamicContract: Some(contractName),
@@ -221,10 +221,10 @@ module OptimizedPartitions = {
         mergeBlock: None,
         addressesByContractName: Dict.make(), // set below
         mutPendingQueries: [],
-        prevQueryRange: minRange,
-        prevPrevQueryRange: minRange,
-        prevRangeSize: (inheritedDensity *. minRange->Int.toFloat)->Math.ceil->Float.toInt,
-        latestBlockRangeUpdateBlock: 0,
+        sourceRangeCapacity: minRange,
+        prevSourceRangeCapacity: minRange,
+        eventDensity: inheritedDensity,
+        latestSourceRangeCapacityUpdateBlock: 0,
       }
     }
 
@@ -472,14 +472,28 @@ module OptimizedPartitions = {
     pendingQuery.fetchedBlock = Some(latestFetchedBlock)
 
     let blockRange = latestFetchedBlock.blockNumber - query.fromBlock + 1
-    // Skip updating block range if a later response already updated it.
+    // Update density for every response, independently from whether this range
+    // is valid evidence of source capacity. A cap hit is still useful density
+    // evidence because it reports items returned across the scanned range.
+    let observedEventDensity = itemsCount->Int.toFloat /. blockRange->Int.toFloat
+    // Seed from the first observation, then smooth every later observation
+    // with a 1:1 moving average. Keeping initialization explicit is important:
+    // zero is valid density evidence and must participate in the next blend.
+    let updatedEventDensity = switch p.eventDensity {
+    | None => Some(observedEventDensity)
+    | Some(eventDensity) => Some((eventDensity +. observedEventDensity) /. 2.)
+    }
+
+    // Skip updating source capacity if a later response already updated it.
     // Prevents degradation of the chunking heuristic when parallel query
     // responses arrive out of order (e.g. earlier query with smaller range
     // arriving after a later query with bigger range).
-    let shouldUpdateBlockRange =
-      latestFetchedBlock.blockNumber > p.latestBlockRangeUpdateBlock &&
+    let shouldUpdateSourceRangeCapacity =
+      latestFetchedBlock.blockNumber > p.latestSourceRangeCapacityUpdateBlock &&
         switch query.toBlock {
-        | None => latestFetchedBlock.blockNumber < knownHeight - 10 // Don't update block range when very close to the head
+        | None =>
+          // Don't update source capacity when very close to the head.
+          latestFetchedBlock.blockNumber < knownHeight - 10
         | Some(queryToBlock) =>
           if latestFetchedBlock.blockNumber < queryToBlock {
             // Partial response is direct capacity evidence — unless it was
@@ -497,9 +511,12 @@ module OptimizedPartitions = {
             }
           }
         }
-    let updatedPrevQueryRange = shouldUpdateBlockRange ? blockRange : p.prevQueryRange
-    let updatedPrevPrevQueryRange = shouldUpdateBlockRange ? p.prevQueryRange : p.prevPrevQueryRange
-    let updatedPrevRangeSize = shouldUpdateBlockRange ? itemsCount : p.prevRangeSize
+    let updatedSourceRangeCapacity = shouldUpdateSourceRangeCapacity
+      ? blockRange
+      : p.sourceRangeCapacity
+    let updatedPrevSourceRangeCapacity = shouldUpdateSourceRangeCapacity
+      ? p.sourceRangeCapacity
+      : p.prevSourceRangeCapacity
 
     // Process fetched queries from front of queue for main partition
     let updatedLatestFetchedBlock = consumeFetchedQueries(
@@ -519,12 +536,12 @@ module OptimizedPartitions = {
       let updatedMainPartition = {
         ...p,
         latestFetchedBlock: updatedLatestFetchedBlock,
-        prevQueryRange: updatedPrevQueryRange,
-        prevPrevQueryRange: updatedPrevPrevQueryRange,
-        prevRangeSize: updatedPrevRangeSize,
-        latestBlockRangeUpdateBlock: shouldUpdateBlockRange
+        sourceRangeCapacity: updatedSourceRangeCapacity,
+        prevSourceRangeCapacity: updatedPrevSourceRangeCapacity,
+        eventDensity: updatedEventDensity,
+        latestSourceRangeCapacityUpdateBlock: shouldUpdateSourceRangeCapacity
           ? latestFetchedBlock.blockNumber
-          : p.latestBlockRangeUpdateBlock,
+          : p.latestSourceRangeCapacityUpdateBlock,
       }
 
       mutEntities->Dict.set(p.id, updatedMainPartition)
@@ -986,10 +1003,10 @@ OptimizedPartitions.t => {
             addressesByContractName,
             mergeBlock: None,
             mutPendingQueries: [],
-            prevQueryRange: 0,
-            prevPrevQueryRange: 0,
-            prevRangeSize: 0,
-            latestBlockRangeUpdateBlock: 0,
+            sourceRangeCapacity: 0,
+            prevSourceRangeCapacity: 0,
+            eventDensity: None,
+            latestSourceRangeCapacityUpdateBlock: 0,
           })
           nextPartitionIndexRef := nextPartitionIndexRef.contents + 1
         }
@@ -1322,10 +1339,10 @@ let registerDynamicContracts = (
                         addressesByContractName,
                         mergeBlock: None,
                         mutPendingQueries: p.mutPendingQueries,
-                        prevQueryRange: p.prevQueryRange,
-                        prevPrevQueryRange: p.prevPrevQueryRange,
-                        prevRangeSize: p.prevRangeSize,
-                        latestBlockRangeUpdateBlock: p.latestBlockRangeUpdateBlock,
+                        sourceRangeCapacity: p.sourceRangeCapacity,
+                        prevSourceRangeCapacity: p.prevSourceRangeCapacity,
+                        eventDensity: p.eventDensity,
+                        latestSourceRangeCapacityUpdateBlock: p.latestSourceRangeCapacityUpdateBlock,
                       })
                     }
                   }
@@ -1448,6 +1465,11 @@ let startFetchingQueries = ({optimizedPartitions}: t, ~queries: array<query>) =>
 // Most parallel in-flight chunk queries a single partition may have at once.
 let maxPendingChunksPerPartition = 12
 
+// Most parallel in-flight queries a single chain may have at once, across all
+// its partitions. Bounds source load on chains with many partitions, where the
+// per-partition cap alone would admit thousands of concurrent queries.
+let maxChainConcurrency = 100
+
 // Generates candidate queries for a gap range (a hole left between
 // completed/pending chunks, e.g. from an out-of-order partial response). Gaps
 // carry the range's low fromBlock, so the acceptance pass takes them before
@@ -1582,24 +1604,25 @@ type partitionFillState = {
 // query with no other ceiling: the true hard bounds stay
 // endBlock/mergeBlock/the lagged head.
 //
-// The tick's budget is chainTargetItems minus what's already in flight. Every
-// candidate query — gap-fill holes, plus each in-range partition's chunks or
-// open-ended probe toward the target — is generated with no budget check, then
-// the candidates are sorted by fromBlock and accepted in that order while the
-// budget stays positive. The query that tips it negative is still accepted (a
-// single overshoot); everything after it waits for a tick with more budget.
+// The tick's budget is chainTargetItems minus what's already in flight. A
+// non-positive budget only resolves the wait action and generates no query
+// candidates. With a positive budget, every candidate query — gap-fill holes,
+// plus each in-range partition's chunks or open-ended probe toward the target —
+// is generated with no budget check, then the candidates are sorted by
+// fromBlock and accepted in that order while the budget stays positive. The
+// query that tips it negative is still accepted (a single overshoot);
+// everything after it waits for a tick with more budget.
 // Sorting by fromBlock spends the budget on the earliest blocks across all
 // partitions first, so no partition is starved by generation order and the
 // frontier advances evenly. In-flight reservations release as responses land,
 // so acceptance redistributes across ticks.
 //
-// A partition with a trusted positive density (two or more responses — see
-// getMinHistoryRange) generates real, density-sized chunks toward the target.
-// Any other partition (no signal, one noisy sample, or a density-0 estimate)
-// generates one open-ended probe sized to the events its range to the target is
-// expected to hold — rangeTargetDensity × (chainTargetBlock − fromBlock + 1) /
-// inRangeCount — so several unknown-density partitions probe in parallel within
-// one budget, each scaled by how much of the range it still has to cover.
+// A partition with source-capacity history and a positive density generates
+// density-sized chunks toward the target. Any other partition (no signal, no
+// capacity history, or a density-0 estimate) generates one open-ended probe
+// sized to the events its range to the target is expected to hold —
+// rangeTargetDensity × (chainTargetBlock − fromBlock + 1) / inRangeCount — so
+// unknown-density partitions probe in parallel within one budget.
 let getNextQuery = (
   {optimizedPartitions, blockLag, latestOnBlockBlockNumber, knownHeight, endBlock}: t,
   ~chainTargetBlock: int,
@@ -1639,6 +1662,12 @@ let getNextQuery = (
       }
     }
 
+    // A zero budget is an admission check: preserve the wait-state scan above,
+    // but make every query-generation pass below empty. Caught-up chains also
+    // skip those passes because their action is already known.
+    let partitionsCount =
+      chainTargetItems <= 0. || shouldWaitForNewBlock.contents ? 0 : partitionsCount
+
     // One bucket per partition, in idsInAscOrder order — gap-fill and the
     // budget pass both push into a partition's own bucket, so flattening at
     // the end (see below) reproduces idsInAscOrder without a sort.
@@ -1788,10 +1817,9 @@ let getNextQuery = (
       inRangeCount > 0 && rangeToTarget > 0 ? freshBudget /. rangeToTarget->Int.toFloat : 0.
 
     // Phase B: generate each in-range partition's candidates — strict chunks
-    // toward the target sized by real density (up to the pending-chunk cap), or,
-    // for a partition without a trusted positive density, a single open-ended
-    // probe at its even share of the budget. No budget check here; the
-    // acceptance pass below decides which candidates make the cut.
+    // when both source-capacity history and density are known, or an open-ended
+    // budget probe otherwise. No budget check here; the acceptance pass below
+    // decides which candidates make the cut.
     //
     // Chunks require a POSITIVE trusted density: density 0 prices every chunk at
     // ~nothing, so an open-ended probe (full server scan range in one response)
@@ -1916,12 +1944,21 @@ let getNextQuery = (
     let streamCount = acceptanceStream->Array.length
     let remainingBudget = ref(chainTargetItems)
     let acceptIdx = ref(0)
-    while remainingBudget.contents > 0. && acceptIdx.contents < streamCount {
+    // In-flight queries count against the chain's concurrency cap alongside the
+    // ones accepted this tick; once the cap is reached no later candidate can be
+    // accepted (they're only later in fromBlock order), so the walk stops.
+    let usedConcurrency = ref(reservations->Array.length)
+    while (
+      remainingBudget.contents > 0. &&
+      acceptIdx.contents < streamCount &&
+      usedConcurrency.contents < maxChainConcurrency
+    ) {
       let (_, itemsEst, maybeQuery) = acceptanceStream->Array.getUnsafe(acceptIdx.contents)
       switch maybeQuery {
       | Some(query) =>
         let partitionIdx = partitionIndexById->Dict.getUnsafe(query.partitionId)
         queriesByPartitionIndex->Array.getUnsafe(partitionIdx)->Array.push(query)->ignore
+        usedConcurrency := usedConcurrency.contents + 1
       | None => ()
       }
       remainingBudget := remainingBudget.contents -. itemsEst->Int.toFloat
@@ -2022,10 +2059,10 @@ let make = (
       mergeBlock: None,
       dynamicContract: None,
       mutPendingQueries: [],
-      prevQueryRange: 0,
-      prevPrevQueryRange: 0,
-      prevRangeSize: 0,
-      latestBlockRangeUpdateBlock: 0,
+      sourceRangeCapacity: 0,
+      prevSourceRangeCapacity: 0,
+      eventDensity: None,
+      latestSourceRangeCapacityUpdateBlock: 0,
     })
   }
 
@@ -2345,13 +2382,15 @@ let isReadyToEnterReorgThreshold = ({endBlock, blockLag, buffer, knownHeight} as
   buffer->Utils.Array.isEmpty
 }
 
-// Lower progress percentage = further behind = higher priority. Shared by the
-// batch ordering and the cross-chain fetch priority.
+// Lower progress percentage = further behind = higher priority. Progress is
+// relative to the head this chain can actually fetch, so a chain at its lagged
+// head does not look behind relative to unavailable blocks. Shared by the batch
+// ordering and the cross-chain fetch priority.
 let getProgressPercentage = (fetchState: t) => {
   switch fetchState.firstEventBlock {
   | None => 0.
   | Some(firstEventBlock) =>
-    let totalRange = fetchState.knownHeight - firstEventBlock
+    let totalRange = fetchState.knownHeight - fetchState.blockLag - firstEventBlock
     if totalRange <= 0 {
       0.
     } else {