envio 3.3.0-alpha.7 → 3.3.0-alpha.8

package/src/Config.res.mjs

@@ -113,6 +113,7 @@ let svmEventDescriptorSchema = S$RescriptSchema.schema(s => ({
   discriminator: s.m(S$RescriptSchema.option(S$RescriptSchema.string)),
   discriminatorByteLen: s.m(S$RescriptSchema.int),
   transactionFields: s.m(S$RescriptSchema.array(Internal.svmTransactionFieldSchema)),
+  blockFields: s.m(S$RescriptSchema.option(S$RescriptSchema.array(Internal.svmBlockFieldSchema))),
   includeLogs: s.m(S$RescriptSchema.bool),
   accountFilters: s.m(S$RescriptSchema.option(S$RescriptSchema.array(S$RescriptSchema.schema(s => ({
     position: s.m(S$RescriptSchema.int),
@@ -490,7 +491,7 @@ function fromPublic(publicConfigJson) {
       };
     });
   }
-  let buildContractEvents = (contractName, events, abi, chainId, startBlock, addresses, svmDefinedTypesOpt) => {
+  let buildContractEvents = (contractName, events, abi, chainId, addresses, svmDefinedTypesOpt) => {
     let svmDefinedTypes = svmDefinedTypesOpt !== undefined ? svmDefinedTypesOpt : null;
     if (events !== undefined) {
       return events.map(eventItem => {
@@ -500,10 +501,10 @@ function fromPublic(publicConfigJson) {
         let kind = eventItem.kind;
         switch (ecosystemName) {
           case "evm" :
-            return EventConfigBuilder.buildEvmEventConfig(contractName, eventName, sighash, params, false, undefined, undefined, undefined, chainId, ecosystem.onEventBlockFilterSchema, eventItem.blockFields, eventItem.transactionFields, startBlock, Primitive_option.some(globalBlockFieldsSet), Primitive_option.some(globalTransactionFieldsSet));
+            return EventConfigBuilder.buildEvmEventConfig(contractName, eventName, sighash, params, eventItem.blockFields, eventItem.transactionFields, Primitive_option.some(globalBlockFieldsSet), Primitive_option.some(globalTransactionFieldsSet));
           case "fuel" :
             if (kind !== undefined) {
-              return EventConfigBuilder.buildFuelEventConfig(contractName, eventName, kind, sighash, abi, false, undefined, undefined, startBlock);
+              return EventConfigBuilder.buildFuelEventConfig(contractName, eventName, kind, sighash, abi);
             } else {
               return Stdlib_JsError.throwWithMessage(`Fuel event ` + contractName + `.` + eventName + ` is missing "kind" in internal config`);
             }
@@ -518,7 +519,7 @@ function fromPublic(publicConfigJson) {
               position: af.position,
               values: af.values
             })));
-            return EventConfigBuilder.buildSvmInstructionEventConfig(contractName, eventName, programId, svm.discriminator, svm.discriminatorByteLen, svm.includeLogs, svm.transactionFields, accountFilters, svm.isInner, false, undefined, undefined, Stdlib_Option.getOr(svm.accounts, []), Stdlib_Option.getOr(svm.args, null), svmDefinedTypes, startBlock);
+            return EventConfigBuilder.buildSvmInstructionEventConfig(contractName, eventName, programId, svm.discriminator, svm.discriminatorByteLen, svm.includeLogs, svm.transactionFields, svm.blockFields, accountFilters, svm.isInner, Stdlib_Option.getOr(svm.accounts, []), Stdlib_Option.getOr(svm.args, null), svmDefinedTypes);
         }
       });
     } else {
@@ -559,7 +560,7 @@ function fromPublic(publicConfigJson) {
       let rawAddresses = Stdlib_Option.getOr(Stdlib_Option.flatMap(chainContract, cc => cc.addresses), []);
       let addresses = rawAddresses.map(parseAddress);
       let startBlock = Stdlib_Option.flatMap(chainContract, cc => cc.startBlock);
-      let events = buildContractEvents(capitalizedName, contractData.events, contractData.abi, chainId, startBlock, rawAddresses, Stdlib_Option.getOr(Stdlib_Option.map(contractData.svmAbi, a => a.definedTypes), null));
+      let events = buildContractEvents(capitalizedName, contractData.events, contractData.abi, chainId, rawAddresses, Stdlib_Option.getOr(Stdlib_Option.map(contractData.svmAbi, a => a.definedTypes), null));
       return {
         name: capitalizedName,
         abi: contractData.abi,
@@ -1011,7 +1012,7 @@ function throwIfIncompatible(changedPaths, resetCommand, runCommand, hasClickhou
   Stdlib_JsError.throwWithMessage(`The following config changes are incompatible with the existing indexer data:\n\n` + bullets + `\n\nPick one:\n  1. ` + padTo(option1, col) + `# resume indexing where it left off\n  2. ` + padTo(resetCommand, col) + `# delete all indexed data and start over` + option3);
 }
 
-function loadWithoutRegistrations() {
+function load() {
   let c = cached.contents;
   if (c !== undefined) {
     return c;
@@ -1060,7 +1061,7 @@ export {
   canonicalJson,
   diffPaths,
   throwIfIncompatible,
-  loadWithoutRegistrations,
+  load,
   getPgUserEntities,
 }
 /* schema Not a pure module */

package/src/Core.res

@@ -9,6 +9,7 @@ type evmRpcClientCtor
 type svmHypersyncClientCtor
 type hyperfuelClientCtor
 type transactionStoreCtor
+type blockStoreCtor
 
 type addon = {
   getConfigJson: (~configPath: Null.t<string>, ~directory: Null.t<string>) => string,
@@ -23,10 +24,16 @@ type addon = {
   hyperfuelClient: hyperfuelClientCtor,
   @as("TransactionStore")
   transactionStore: transactionStoreCtor,
+  @as("BlockStore")
+  blockStore: blockStoreCtor,
   // Ordered transaction-field names exposed for the field-code contract test
   // (the ReScript `transactionFields` arrays must match the Rust ordinals).
   evmTransactionFieldNames: unit => array<string>,
   svmTransactionFieldNames: unit => array<string>,
+  // Ordered block-field names for the same contract test (`blockFields` arrays
+  // must match the Rust ordinals).
+  evmBlockFieldNames: unit => array<string>,
+  svmBlockFieldNames: unit => array<string>,
 }
 
 @module("node:module") external createRequire: string => {..} = "createRequire"

package/src/CrossChainState.res

@@ -135,13 +135,13 @@ let enterReorgThreshold = (crossChainState: t) => {
 // whole indexer. A chain is marked caught up only once EVERY chain is caught up
 // (reached endblock or fetched/processed to head) with no processable events
 // left — so no chain flips to ready while another is still backfilling.
-let applyBatchProgress = (crossChainState: t, ~batch: Batch.t) => {
+let applyBatchProgress = (crossChainState: t, ~batch: Batch.t, ~blockTimestampName: string) => {
   let chainIds = crossChainState.chainIds
 
   let everyChainCaughtUp = ref(true)
   for i in 0 to chainIds->Array.length - 1 {
     let cs = crossChainState->getChainState(chainIds->Array.getUnsafe(i))
-    cs->ChainState.applyBatchProgress(~batch)
+    cs->ChainState.applyBatchProgress(~batch, ~blockTimestampName)
     if !(cs->ChainState.hasProcessedToEndblock || cs->ChainState.isProgressAtHead) {
       everyChainCaughtUp := false
     }
@@ -190,98 +190,71 @@ let totalReservedSize = (crossChainState: t) => {
   total.contents
 }
 
-// Furthest-behind first: order candidate queries by the chain progress % at
-// their fromBlock, so the most behind ranges across all chains are admitted
-// before the rest.
-let compareByProgress = (a: FetchState.query, b: FetchState.query) =>
-  Float.compare(a.progress, b.progress)
-
 // Dispatch a fetch tick across the whole indexer from one shared pool of
-// ~targetBufferSize ready events. Every chain proposes its candidate queries
-// (each carrying an estimated response size) up to its own natural ceiling
-// (head/endBlock/mergeBlock), unconstrained by the shared budget; the
-// candidates are then pooled, ordered by chain progress (furthest-behind first),
-// and admitted until the budget is consumed. So the budget is split per query
-// across chains rather than per chain — a chain that can only use a little
-// leaves the rest for the others automatically.
+// ~targetBufferSize ready events, as a waterfall: visit chains furthest-behind
+// first, hand each the budget remaining at that point (plus its own
+// already-reserved share, since a chain's pending queries aren't "spent" —
+// they're this chain's), let it turn that into queries sized against its own
+// chain-density-derived target block, then subtract what it actually used
+// before moving to the next chain. So a chain that can only use a little
+// (density too low, or already caught up) leaves the rest for the others
+// automatically — unlike a per-query pool, this can starve a near-head chain
+// while an earlier one is deep in backfill, which is the intended tradeoff.
 let checkAndFetch = async (
   crossChainState: t,
   ~dispatchChain: (~chain: ChainMap.Chain.t, ~action: FetchState.nextQuery) => promise<unit>,
 ) => {
-  let remaining = Pervasives.max(
-    0,
-    crossChainState.targetBufferSize -
-    crossChainState->totalReadyCount -
-    crossChainState->totalReservedSize->Float.toInt,
+  let remaining = ref(
+    Pervasives.max(
+      0.,
+      crossChainState.targetBufferSize->Int.toFloat -.
+      crossChainState->totalReadyCount->Int.toFloat -.
+      crossChainState->totalReservedSize,
+    ),
   )
 
-  let chainIds = crossChainState.chainIds
   let actionByChain = Dict.make()
-  // Candidate queries from every chain. Each query carries its chain id and the
-  // chain progress % at its fromBlock (the admission sort key), set here so the
-  // pool can be ordered without a side tuple per query.
-  let candidates = []
-  for i in 0 to chainIds->Array.length - 1 {
-    let chainId = chainIds->Array.getUnsafe(i)
-    let cs = crossChainState->getChainState(chainId)
-    switch cs->ChainState.getNextQuery {
+  crossChainState
+  ->priorityOrder
+  ->Array.forEach(cs => {
+    let chainId = (cs->ChainState.chainConfig).id
+    let chainTargetItems = remaining.contents +. cs->ChainState.pendingBudget
+    switch cs->ChainState.getNextQuery(~chainTargetItems) {
     | (WaitingForNewBlock | NothingToQuery) as action =>
       actionByChain->Utils.Dict.setByInt(chainId, action)
-    | Ready(queries) =>
-      // Default to NothingToQuery; replaced below if any candidate is admitted.
-      actionByChain->Utils.Dict.setByInt(chainId, FetchState.NothingToQuery)
-      queries->Array.forEach(query => {
-        query.chainId = chainId
-        query.progress = cs->ChainState.getProgressPercentageAt(~blockNumber=query.fromBlock)
-        candidates->Array.push(query)
-      })
+    | Ready(queries) => {
+        let consumed =
+          queries->Array.reduce(0., (acc, query: FetchState.query) => acc +. query.itemsTarget)
+
+        let partitions = Dict.make()
+        queries->Array.forEach((query: FetchState.query) =>
+          partitions->Dict.set(
+            query.partitionId,
+            {
+              "fromBlock": query.fromBlock,
+              "targetBlock": query.toBlock,
+              "targetEvents": query.itemsTarget->Math.round->Float.toInt,
+            },
+          )
+        )
+        Logging.trace({
+          "msg": "Started querying",
+          "chainId": chainId,
+          "partitions": partitions,
+        })
+
+        actionByChain->Utils.Dict.setByInt(chainId, FetchState.Ready(queries))
+        // Mark the queries in flight and reserve their size against the shared
+        // budget; released as each response lands in handleQueryResult.
+        cs->ChainState.startFetchingQueries(~queries)
+        remaining := Pervasives.max(0., remaining.contents -. consumed)
+      }
     }
-  }
-
-  candidates->Array.sort(compareByProgress)
-
-  // Admit furthest-behind first until the budget runs out. The condition is
-  // checked before each query, so as long as there's any budget left we admit
-  // the next one even when its estimate alone exceeds the remainder — otherwise a
-  // chain whose only query is bigger than the budget would never make progress.
-  let admittedByChain = Dict.make()
-  let running = ref(0.)
-  let remainingF = remaining->Int.toFloat
-  let idx = ref(0)
-  while running.contents < remainingF && idx.contents < candidates->Array.length {
-    let query = candidates->Array.getUnsafe(idx.contents)
-    admittedByChain->Utils.Dict.push(query.chainId->Int.toString, query)
-    running := running.contents +. query.estResponseSize
-    idx := idx.contents + 1
-  }
-  admittedByChain->Dict.forEachWithKey((queries, chainId) => {
-    let partitions = Dict.make()
-    queries->Array.forEach((query: FetchState.query) =>
-      partitions->Dict.set(
-        query.partitionId,
-        {
-          "fromBlock": query.fromBlock,
-          "targetBlock": query.toBlock,
-          "targetEvents": query.estResponseSize->Math.round->Float.toInt,
-        },
-      )
-    )
-    Logging.trace({
-      "msg": "Started querying",
-      "chainId": chainId->Int.fromString->Option.getUnsafe,
-      "partitions": partitions,
-    })
-    actionByChain->Dict.set(chainId, FetchState.Ready(queries))
-    // Mark the admitted queries in flight and reserve their size against the
-    // shared budget; released as each response lands in handleQueryResult.
-    crossChainState
-    ->getChainState(chainId->Int.fromString->Option.getUnsafe)
-    ->ChainState.startFetchingQueries(~queries)
   })
 
   let promises = []
-  for i in 0 to chainIds->Array.length - 1 {
-    let chainId = chainIds->Array.getUnsafe(i)
+  for i in 0 to crossChainState.chainIds->Array.length - 1 {
+    let chainId = crossChainState.chainIds->Array.getUnsafe(i)
     switch actionByChain->Utils.Dict.dangerouslyGetByIntNonOption(chainId) {
     | Some(NothingToQuery)
     | None => ()

package/src/CrossChainState.res.mjs

@@ -7,9 +7,7 @@ import * as Logging from "./Logging.res.mjs";
 import * as ChainMap from "./ChainMap.res.mjs";
 import * as ChainState from "./ChainState.res.mjs";
 import * as Prometheus from "./Prometheus.res.mjs";
-import * as Stdlib_Int from "@rescript/runtime/lib/es6/Stdlib_Int.js";
-import * as Stdlib_Dict from "@rescript/runtime/lib/es6/Stdlib_Dict.js";
-import * as Primitive_int from "@rescript/runtime/lib/es6/Primitive_int.js";
+import * as Stdlib_Array from "@rescript/runtime/lib/es6/Stdlib_Array.js";
 import * as Primitive_float from "@rescript/runtime/lib/es6/Primitive_float.js";
 import * as SafeCheckpointTracking from "./SafeCheckpointTracking.res.mjs";
 
@@ -95,12 +93,12 @@ function enterReorgThreshold(crossChainState) {
   crossChainState.isInReorgThreshold = true;
 }
 
-function applyBatchProgress(crossChainState, batch) {
+function applyBatchProgress(crossChainState, batch, blockTimestampName) {
   let chainIds = crossChainState.chainIds;
   let everyChainCaughtUp = true;
   for (let i = 0, i_finish = chainIds.length; i < i_finish; ++i) {
     let cs = crossChainState.chainStates[chainIds[i]];
-    ChainState.applyBatchProgress(cs, batch);
+    ChainState.applyBatchProgress(cs, batch, blockTimestampName);
     if (!(ChainState.hasProcessedToEndblock(cs) || ChainState.isProgressAtHead(cs))) {
       everyChainCaughtUp = false;
     }
@@ -135,73 +133,59 @@ function totalReservedSize(crossChainState) {
   return total;
 }
 
-function compareByProgress(a, b) {
-  return Primitive_float.compare(a.progress, b.progress);
-}
-
 async function checkAndFetch(crossChainState, dispatchChain) {
-  let remaining = Primitive_int.max(0, (crossChainState.targetBufferSize - totalReadyCount(crossChainState) | 0) - (totalReservedSize(crossChainState) | 0) | 0);
-  let chainIds = crossChainState.chainIds;
+  let remaining = {
+    contents: Primitive_float.max(0, crossChainState.targetBufferSize - totalReadyCount(crossChainState) - totalReservedSize(crossChainState))
+  };
   let actionByChain = {};
-  let candidates = [];
-  for (let i = 0, i_finish = chainIds.length; i < i_finish; ++i) {
-    let chainId = chainIds[i];
-    let cs = crossChainState.chainStates[chainId];
-    let action = ChainState.getNextQuery(cs);
+  priorityOrder(crossChainState).forEach(cs => {
+    let chainId = ChainState.chainConfig(cs).id;
+    let chainTargetItems = remaining.contents + ChainState.pendingBudget(cs);
+    let action = ChainState.getNextQuery(cs, chainTargetItems);
     if (typeof action !== "object") {
+      if (action === "WaitingForNewBlock") {
+        actionByChain[chainId] = action;
+        return;
+      }
       actionByChain[chainId] = action;
+      return;
     } else {
-      actionByChain[chainId] = "NothingToQuery";
-      action._0.forEach(query => {
-        query.chainId = chainId;
-        query.progress = ChainState.getProgressPercentageAt(cs, query.fromBlock);
-        candidates.push(query);
+      let queries = action._0;
+      let consumed = Stdlib_Array.reduce(queries, 0, (acc, query) => acc + query.itemsTarget);
+      let partitions = {};
+      queries.forEach(query => {
+        partitions[query.partitionId] = {
+          fromBlock: query.fromBlock,
+          targetBlock: query.toBlock,
+          targetEvents: Math.round(query.itemsTarget) | 0
+        };
       });
-    }
-  }
-  candidates.sort(compareByProgress);
-  let admittedByChain = {};
-  let running = 0;
-  let idx = 0;
-  while (running < remaining && idx < candidates.length) {
-    let query = candidates[idx];
-    Utils.Dict.push(admittedByChain, query.chainId.toString(), query);
-    running = running + query.estResponseSize;
-    idx = idx + 1 | 0;
-  };
-  Stdlib_Dict.forEachWithKey(admittedByChain, (queries, chainId) => {
-    let partitions = {};
-    queries.forEach(query => {
-      partitions[query.partitionId] = {
-        fromBlock: query.fromBlock,
-        targetBlock: query.toBlock,
-        targetEvents: Math.round(query.estResponseSize) | 0
+      Logging.trace({
+        msg: "Started querying",
+        chainId: chainId,
+        partitions: partitions
+      });
+      actionByChain[chainId] = {
+        TAG: "Ready",
+        _0: queries
       };
-    });
-    Logging.trace({
-      msg: "Started querying",
-      chainId: Stdlib_Int.fromString(chainId, undefined),
-      partitions: partitions
-    });
-    actionByChain[chainId] = {
-      TAG: "Ready",
-      _0: queries
-    };
-    let chainId$1 = Stdlib_Int.fromString(chainId, undefined);
-    ChainState.startFetchingQueries(crossChainState.chainStates[chainId$1], queries);
+      ChainState.startFetchingQueries(cs, queries);
+      remaining.contents = Primitive_float.max(0, remaining.contents - consumed);
+      return;
+    }
   });
   let promises = [];
-  for (let i$1 = 0, i_finish$1 = chainIds.length; i$1 < i_finish$1; ++i$1) {
-    let chainId$1 = chainIds[i$1];
-    let action$1 = actionByChain[chainId$1];
-    if (action$1 !== undefined) {
+  for (let i = 0, i_finish = crossChainState.chainIds.length; i < i_finish; ++i) {
+    let chainId = crossChainState.chainIds[i];
+    let action = actionByChain[chainId];
+    if (action !== undefined) {
       let exit = 0;
-      if (typeof action$1 === "object" || action$1 === "WaitingForNewBlock") {
+      if (typeof action === "object" || action === "WaitingForNewBlock") {
         exit = 1;
       }
       if (exit === 1) {
-        let chain = ChainMap.Chain.makeUnsafe(chainId$1);
-        promises.push(dispatchChain(chain, action$1));
+        let chain = ChainMap.Chain.makeUnsafe(chainId);
+        promises.push(dispatchChain(chain, action));
       }
     }
   }

package/src/CrossChainState.resi

@@ -29,7 +29,7 @@ let createBatch: (
   ~isRollback: bool,
 ) => Batch.t
 let enterReorgThreshold: t => unit
-let applyBatchProgress: (t, ~batch: Batch.t) => unit
+let applyBatchProgress: (t, ~batch: Batch.t, ~blockTimestampName: string) => unit
 
 // Fetch control.
 let priorityOrder: t => array<ChainState.t>

package/src/Ecosystem.res

@@ -69,12 +69,12 @@ let getItemLogger = {
     | None =>
       let logger = switch item {
       | Internal.Event(_) => ecosystem.toEventLogger(item->Internal.castUnsafeEventItem)
-      | Block({blockNumber, onBlockConfig}) =>
+      | Block({blockNumber, onBlockRegistration}) =>
         Logging.createChildFrom(
           ~logger=ecosystem.logger,
           ~params={
-            "onBlock": onBlockConfig.name,
-            "chainId": onBlockConfig.chainId,
+            "onBlock": onBlockRegistration.name,
+            "chainId": onBlockRegistration.chainId,
             "block": blockNumber,
           },
         )

package/src/Ecosystem.res.mjs

@@ -26,10 +26,10 @@ function getItemLogger(item, ecosystem) {
   if (item.kind === 0) {
     logger$1 = ecosystem.toEventLogger(item);
   } else {
-    let onBlockConfig = item.onBlockConfig;
+    let onBlockRegistration = item.onBlockRegistration;
     logger$1 = Logging.createChildFrom(ecosystem.logger, {
-      onBlock: onBlockConfig.name,
-      chainId: onBlockConfig.chainId,
+      onBlock: onBlockRegistration.name,
+      chainId: onBlockRegistration.chainId,
       block: item.blockNumber
     });
   }

package/src/Envio.res

@@ -66,18 +66,21 @@ type svmLog = {
   message: string,
 }
 
-/** Block context for a matched instruction. `time`/`hash` follow the
- EVM/Fuel field names so the shared `Ecosystem.t` getters in `Svm.res` read
- them uniformly. */
+/** Block context for a matched instruction. `slot`/`hash` are always present;
+ `time` can still be absent — HyperSync/Solana may not report a block time for
+ every slot. Every other field is opt-in via `field_selection.block_fields`,
+ materialised from the per-chain block store at batch prep. */
 type svmInstructionBlock = {
   /** Slot this instruction's block was matched in. */
   slot: int,
-  /** Unix block time (seconds). `0` when HyperSync didn't return a block
-   for this instruction's slot. */
-  time: int,
-  /** Block hash. Currently always empty — populated by the future
-   reorg-guard `queryBlockHash(slot)` route. */
+  /** Unix block time (seconds). */
+  time?: int,
   hash: string,
+  /** Block height (distinct from slot). Absent when not selected via
+   `field_selection.block_fields`. */
+  height?: int,
+  parentSlot?: int,
+  parentHash?: string,
 }
 
 /** The per-instruction payload handlers receive as their `instruction`
@@ -111,7 +114,9 @@ type svmInstruction = {
   /** Program log entries scoped to this instruction. Absent when the
    per-instruction `include_logs` flag is `false`. */
   logs?: array<svmLog>,
-  block: svmInstructionBlock,
+  // Omitted on construction; materialised onto the payload at batch prep,
+  // before a handler ever reads it.
+  block?: svmInstructionBlock,
 }
 
 /** Arguments passed to handlers registered via `indexer.onInstruction`. */