envio 3.3.0-alpha.10 → 3.3.0-alpha.12

package/src/Config.res.mjs

@@ -115,10 +115,10 @@ let svmEventDescriptorSchema = S$RescriptSchema.schema(s => ({
   transactionFields: s.m(S$RescriptSchema.array(Internal.svmTransactionFieldSchema)),
   blockFields: s.m(S$RescriptSchema.option(S$RescriptSchema.array(Internal.svmBlockFieldSchema))),
   includeLogs: s.m(S$RescriptSchema.bool),
-  accountFilters: s.m(S$RescriptSchema.option(S$RescriptSchema.array(S$RescriptSchema.schema(s => ({
+  accountFilters: s.m(S$RescriptSchema.option(S$RescriptSchema.array(S$RescriptSchema.array(S$RescriptSchema.schema(s => ({
     position: s.m(S$RescriptSchema.int),
     values: s.m(S$RescriptSchema.array(S$RescriptSchema.string))
-  }))))),
+  })))))),
   isInner: s.m(S$RescriptSchema.option(S$RescriptSchema.bool)),
   accounts: s.m(S$RescriptSchema.option(S$RescriptSchema.array(S$RescriptSchema.string))),
   args: s.m(S$RescriptSchema.option(S$RescriptSchema.json(false)))

package/src/Core.res

@@ -10,9 +10,17 @@ type svmHypersyncClientCtor
 type hyperfuelClientCtor
 type transactionStoreCtor
 type blockStoreCtor
+type parseConfigYamlOptions = {
+  schema?: string,
+  env?: dict<string>,
+  files?: dict<string>,
+  isRescript?: bool,
+}
 
 type addon = {
   getConfigJson: (~configPath: Null.t<string>, ~directory: Null.t<string>) => string,
+  encodeIndexedTopic: (~abiType: string, ~value: unknown) => EvmTypes.Hex.t,
+  parseConfigYaml: (string, parseConfigYamlOptions) => string,
   runCli: (~args: array<string>, ~envioPackageDir: Null.t<string>) => promise<Null.t<string>>,
   @as("EvmHypersyncClient")
   evmHypersyncClient: evmHypersyncClientCtor,
@@ -197,6 +205,18 @@ let getConfigJson = (~configPath=?, ~directory=?) => {
   )
 }
 
+// Pure config entry point: no cwd, config file, schema file, .env, or process env lookup.
+// A supplied schema is authoritative; omitted or blank schema text means an empty schema.
+let parseConfigYaml = (~schema=?, ~env=?, ~files=?, ~isRescript=false, yaml) => {
+  let addon = getAddon()
+  addon.parseConfigYaml(yaml, {
+    ?schema,
+    ?env,
+    ?files,
+    isRescript,
+  })
+}
+
 let runCli = args => {
   let addon = getAddon()
   addon.runCli(~args, ~envioPackageDir=Null.make(envioPackageDir))

package/src/Core.res.mjs

@@ -180,6 +180,17 @@ function getConfigJson(configPath, directory) {
   return addon.getConfigJson(Stdlib_Null.fromOption(configPath), Stdlib_Null.fromOption(directory));
 }
 
+function parseConfigYaml(schema, env, files, isRescriptOpt, yaml) {
+  let isRescript = isRescriptOpt !== undefined ? isRescriptOpt : false;
+  let addon = getAddon();
+  return addon.parseConfigYaml(yaml, {
+    schema: schema,
+    env: env,
+    files: files,
+    isRescript: isRescript
+  });
+}
+
 function runCli(args) {
   let addon = getAddon();
   return addon.runCli(args, envioPackageDir);
@@ -196,6 +207,7 @@ export {
   addonRef,
   getAddon,
   getConfigJson,
+  parseConfigYaml,
   runCli,
 }
 /* envioPackageDir Not a pure module */

package/src/CrossChainState.res

@@ -170,7 +170,7 @@ let applyBatchProgress = (crossChainState: t, ~batch: Batch.t, ~blockTimestampNa
 // --- Fetch control. ---
 
 // Chains ordered furthest-behind first, so the shared buffer pool goes to the
-// chains with the most backfill work before the rest.
+// chains with the most fetchable backfill work before the rest.
 let priorityOrder = (crossChainState: t) =>
   crossChainState.chainStates
   ->Dict.valuesToArray
@@ -198,29 +198,37 @@ let totalReservedSize = (crossChainState: t) => {
 // chain-density-derived target block, then subtract what it actually used
 // before moving to the next chain. So a chain that can only use a little
 // (density too low, or already caught up) leaves the rest for the others
-// automatically. A chain visited after the budget ran out simply doesn't
-// query this round — the leader's reservations release as its responses land,
-// so the next tick redistributes. Chains that do get budget are additionally
-// capped at the most-behind chain's target progress mapped onto their own
-// range, so no chain runs further ahead than the chain the pool is
-// prioritizing.
+// automatically. Starting a new query requires at least 10% of the target pool
+// to be free. A chain visited after the budget falls below that admission unit
+// doesn't query this round — reservations release as responses land, so the
+// next tick redistributes. Chains that do get budget are additionally capped
+// at the first querying chain's target progress mapped onto their own range, so
+// no chain runs further ahead than the chain currently using the pool. Waiting
+// and idle chains do not establish this alignment line.
 let checkAndFetch = async (
   crossChainState: t,
   ~dispatchChain: (~chain: ChainMap.Chain.t, ~action: FetchState.nextQuery) => promise<unit>,
 ) => {
+  let targetBudget = crossChainState.targetBufferSize->Int.toFloat
   let remaining = ref(
     Pervasives.max(
       0.,
-      crossChainState.targetBufferSize->Int.toFloat -.
+      targetBudget -.
       crossChainState->totalReadyCount->Int.toFloat -.
       crossChainState->totalReservedSize,
     ),
   )
 
+  // New fetch work is admitted in units of 10% of the target pool. Waiting
+  // below this floor avoids spending the last few free items on undersized
+  // queries; response and batch completion schedule another tick after they
+  // release enough budget.
+  let minimumAdmissionBudget = targetBudget *. 0.1
+
   // A chain with no density signal probes blind, so it only gets a bounded
   // slice of the pool — one unknown chain shouldn't hold the whole budget
-  // while it takes its first measurements.
-  let coldChainBudget = Pervasives.min(5_000., crossChainState.targetBufferSize->Int.toFloat)
+  // while it takes its first measurements. Its probe is one admission unit.
+  let coldChainBudget = minimumAdmissionBudget
 
   // Chunk reservations get headroom over the density estimate so a
   // denser-than-expected range doesn't truncate at the server cap; realtime
@@ -233,30 +241,33 @@ let checkAndFetch = async (
   ->priorityOrder
   ->Array.forEach(cs => {
     let chainId = (cs->ChainState.chainConfig).id
-    if cs->ChainState.knownHeight == 0 {
-      // No height yet — nothing to size a budget or an alignment line
-      // against. Skip without consuming budget or claiming leadership, so a
-      // chain whose source hasn't reported doesn't unconstrain everyone else.
-      actionByChain->Utils.Dict.setByInt(chainId, FetchState.WaitingForNewBlock)
+    if remaining.contents < minimumAdmissionBudget {
+      // More than 90% of the target pool is ready or reserved. Do not attempt
+      // any chain action until a full admission unit becomes free.
+      actionByChain->Utils.Dict.setByInt(chainId, FetchState.NothingToQuery)
+    } else if cs->ChainState.knownHeight == 0 {
+      // Let getNextQuery own the waiting decision without trying to size work
+      // against an unknown height.
+      actionByChain->Utils.Dict.setByInt(
+        chainId,
+        cs->ChainState.getNextQuery(~chainTargetItems=0., ~chunkItemsMultiplier),
+      )
     } else {
       let isCold = cs->ChainState.effectiveDensity === None
       let chainTargetItems =
         (isCold ? Pervasives.min(remaining.contents, coldChainBudget) : remaining.contents) +.
         cs->ChainState.pendingBudget
+      let candidateProgress = switch maxProgress.contents {
+      | None if !isCold =>
+        Some(
+          cs->ChainState.progressAtBlock(
+            ~blockNumber=cs->ChainState.targetBlock(~chainTargetItems),
+          ),
+        )
+      | _ => None
+      }
       let maxTargetBlock = switch maxProgress.contents {
-      | None if isCold =>
-        // A cold chain's target is a guess — it doesn't set the alignment
-        // line; the first density-bearing chain after it does.
-        None
-      | None =>
-        // The most-behind chain sets the alignment line for everyone after it.
-        maxProgress :=
-          Some(
-            cs->ChainState.progressAtBlock(
-              ~blockNumber=cs->ChainState.targetBlock(~chainTargetItems),
-            ),
-          )
-        None
+      | None => None
       // 5% margin past the leader's line: chains whose progress tracks the
       // leader closely would otherwise flap in and out of the clamp as
       // leadership alternates, stalling their pipeline every other tick.
@@ -267,9 +278,33 @@ let checkAndFetch = async (
         ~chunkItemsMultiplier,
         ~maxTargetBlock?,
       ) {
-      | (WaitingForNewBlock | NothingToQuery) as action =>
+      | WaitingForNewBlock as action => actionByChain->Utils.Dict.setByInt(chainId, action)
+      | NothingToQuery =>
+        // A chain below its head that emitted no query this tick — its budget
+        // went to more-behind chains, or the cross-chain alignment clamped its
+        // range to nothing — and has nothing else to wake it must keep polling
+        // for new blocks instead of going silent. NothingToQuery isn't
+        // dispatched, so such a chain would never be re-scheduled and its head
+        // tracking would freeze. A chain is genuinely idle — and correctly left
+        // undispatched — when it is caught up to its head/endblock, still
+        // draining in-flight queries, or holding ready items that batch
+        // processing will drain and re-schedule from.
+        let action =
+          cs->ChainState.isFetchingAtHead ||
+          cs->ChainState.pendingBudget > 0. ||
+          cs->ChainState.bufferReadyCount > 0
+            ? FetchState.NothingToQuery
+            : FetchState.WaitingForNewBlock
         actionByChain->Utils.Dict.setByInt(chainId, action)
       | Ready(queries) => {
+          // A density-bearing chain establishes the alignment line only after
+          // proving that it can use the pool. Waiting and idle chains leave it
+          // open for the next chain; cold-chain targets remain guesses and do
+          // not lead, as before.
+          switch candidateProgress {
+          | Some(progress) => maxProgress := Some(progress)
+          | None => ()
+          }
           let consumed =
             queries->Array.reduce(0., (acc, query: FetchState.query) =>
               acc +. query.itemsEst->Int.toFloat

package/src/CrossChainState.res.mjs

@@ -134,10 +134,11 @@ function totalReservedSize(crossChainState) {
 }
 
 async function checkAndFetch(crossChainState, dispatchChain) {
+  let targetBudget = crossChainState.targetBufferSize;
   let remaining = {
-    contents: Primitive_float.max(0, crossChainState.targetBufferSize - totalReadyCount(crossChainState) - totalReservedSize(crossChainState))
+    contents: Primitive_float.max(0, targetBudget - totalReadyCount(crossChainState) - totalReservedSize(crossChainState))
   };
-  let coldChainBudget = Primitive_float.min(5000, crossChainState.targetBufferSize);
+  let minimumAdmissionBudget = targetBudget * 0.1;
   let chunkItemsMultiplier = crossChainState.isRealtime ? 3 : 1.5;
   let actionByChain = {};
   let maxProgress = {
@@ -145,28 +146,36 @@ async function checkAndFetch(crossChainState, dispatchChain) {
   };
   priorityOrder(crossChainState).forEach(cs => {
     let chainId = ChainState.chainConfig(cs).id;
+    if (remaining.contents < minimumAdmissionBudget) {
+      actionByChain[chainId] = "NothingToQuery";
+      return;
+    }
     if (ChainState.knownHeight(cs) === 0) {
-      actionByChain[chainId] = "WaitingForNewBlock";
+      actionByChain[chainId] = ChainState.getNextQuery(cs, 0, chunkItemsMultiplier, undefined);
       return;
     }
     let isCold = ChainState.effectiveDensity(cs) === undefined;
     let chainTargetItems = (
-      isCold ? Primitive_float.min(remaining.contents, coldChainBudget) : remaining.contents
+      isCold ? Primitive_float.min(remaining.contents, minimumAdmissionBudget) : remaining.contents
     ) + ChainState.pendingBudget(cs);
+    let match = maxProgress.contents;
+    let candidateProgress = match !== undefined || isCold ? undefined : ChainState.progressAtBlock(cs, ChainState.targetBlock(cs, chainTargetItems));
     let progress = maxProgress.contents;
-    let maxTargetBlock = progress !== undefined ? ChainState.blockAtProgress(cs, progress + 0.05) : (
-        isCold ? undefined : (maxProgress.contents = ChainState.progressAtBlock(cs, ChainState.targetBlock(cs, chainTargetItems)), undefined)
-      );
+    let maxTargetBlock = progress !== undefined ? ChainState.blockAtProgress(cs, progress + 0.05) : undefined;
     let action = ChainState.getNextQuery(cs, chainTargetItems, chunkItemsMultiplier, maxTargetBlock);
     if (typeof action !== "object") {
       if (action === "WaitingForNewBlock") {
         actionByChain[chainId] = action;
         return;
       }
-      actionByChain[chainId] = action;
+      let action$1 = ChainState.isFetchingAtHead(cs) || ChainState.pendingBudget(cs) > 0 || ChainState.bufferReadyCount(cs) > 0 ? "NothingToQuery" : "WaitingForNewBlock";
+      actionByChain[chainId] = action$1;
       return;
     } else {
       let queries = action._0;
+      if (candidateProgress !== undefined) {
+        maxProgress.contents = candidateProgress;
+      }
       let consumed = Stdlib_Array.reduce(queries, 0, (acc, query) => acc + query.itemsEst);
       let partitions = {};
       queries.forEach(query => {

package/src/EventConfigBuilder.res

@@ -236,58 +236,30 @@ let buildSimulateParamsSchema = (params: array<paramMeta>): S.t<Internal.eventPa
 
 // ============== Build topic filter getters ==============
 
-let getTopicEncoder = (abiType: string): (unknown => EvmTypes.Hex.t) => {
-  // Handle array/tuple types - these get keccak256'd
-  if abiType->String.endsWith("]") || abiType->String.startsWith("(") {
-    TopicFilter.castToHexUnsafe->(Utils.magic: ('a => EvmTypes.Hex.t) => unknown => EvmTypes.Hex.t)
-  } else {
-    switch abiType {
-    | "address" =>
-      // Lowercase before encoding so mixed-case (checksummed) user input
-      // still matches the lowercase hex topics returned by sources.
-
-      (
-        (value: string) =>
-          value->String.toLowerCase->Address.unsafeFromString->TopicFilter.fromAddress
-      )->(Utils.magic: (string => EvmTypes.Hex.t) => unknown => EvmTypes.Hex.t)
-
-    | "bool" =>
-      TopicFilter.fromBool->(Utils.magic: (bool => EvmTypes.Hex.t) => unknown => EvmTypes.Hex.t)
-    | "string" =>
-      TopicFilter.fromDynamicString->(
-        Utils.magic: (string => EvmTypes.Hex.t) => unknown => EvmTypes.Hex.t
-      )
-
-    | "bytes" =>
-      TopicFilter.fromDynamicBytes->(
-        Utils.magic: (string => EvmTypes.Hex.t) => unknown => EvmTypes.Hex.t
-      )
-
-    | t if t->String.startsWith("uint") =>
-      TopicFilter.fromBigInt->(Utils.magic: (bigint => EvmTypes.Hex.t) => unknown => EvmTypes.Hex.t)
-    | t if t->String.startsWith("int") =>
-      TopicFilter.fromSignedBigInt->(
-        Utils.magic: (bigint => EvmTypes.Hex.t) => unknown => EvmTypes.Hex.t
-      )
-
-    | t if t->String.startsWith("bytes") =>
-      TopicFilter.castToHexUnsafe->(
-        Utils.magic: ('a => EvmTypes.Hex.t) => unknown => EvmTypes.Hex.t
-      )
-
-    | other => JsError.throwWithMessage(`Unsupported topic filter ABI type: ${other}`)
-    }
-  }
-}
+let getTopicEncoder = (abiType: string): (unknown => EvmTypes.Hex.t) => value =>
+  Core.getAddon().encodeIndexedTopic(~abiType, ~value)
 
 let buildTopicGetter = (p: paramMeta) => {
   let encoder = getTopicEncoder(p.abiType)
+  let isTuple = p.abiType->String.startsWith("(")
   (eventFilter: dict<JSON.t>) =>
     eventFilter
     ->Utils.Dict.dangerouslyGetNonOption(p.name)
-    ->Option.mapOr([], topicFilters =>
-      topicFilters->(Utils.magic: JSON.t => unknown)->normalizeOrThrow->Array.map(encoder)
-    )
+    ->Option.mapOr([], topicFilters => {
+      let raw = topicFilters->(Utils.magic: JSON.t => unknown)
+      // A tuple filter value is itself an array, so a directly-passed tuple is
+      // indistinguishable from an OR-list by shape alone. A single tuple is
+      // the common case, so try it first; when the value doesn't ABI-encode as
+      // one tuple it must be an OR-list of tuples.
+      if isTuple {
+        switch encoder(raw) {
+        | encoded => [encoded]
+        | exception _ => raw->normalizeOrThrow->Array.map(encoder)
+        }
+      } else {
+        raw->normalizeOrThrow->Array.map(encoder)
+      }
+    })
 }
 
 // ============== Field selection ==============
@@ -452,6 +424,7 @@ let buildEvmOnEventRegistration = (
   }
 
   {
+    index: -1,
     eventConfig: (eventConfig :> Internal.eventConfig),
     isWildcard,
     handler,
@@ -536,6 +509,7 @@ let buildSvmOnEventRegistration = (
   ~contractRegister: option<Internal.contractRegister>,
   ~startBlock: option<int>=?,
 ): Internal.svmOnEventRegistration => {
+  index: -1,
   eventConfig: (eventConfig :> Internal.eventConfig),
   handler,
   contractRegister,
@@ -613,6 +587,7 @@ let buildFuelOnEventRegistration = (
   ~contractRegister: option<Internal.contractRegister>,
   ~startBlock: option<int>=?,
 ): Internal.fuelOnEventRegistration => {
+  index: -1,
   eventConfig: (eventConfig :> Internal.eventConfig),
   handler,
   contractRegister,

package/src/EventConfigBuilder.res.mjs

@@ -2,11 +2,11 @@
 
 import * as Evm from "./sources/Evm.res.mjs";
 import * as Svm from "./sources/Svm.res.mjs";
+import * as Core from "./Core.res.mjs";
 import * as Utils from "./Utils.res.mjs";
 import * as Address from "./Address.res.mjs";
 import * as FuelSDK from "./sources/FuelSDK.res.mjs";
 import * as Internal from "./Internal.res.mjs";
-import * as TopicFilter from "./TopicFilter.res.mjs";
 import * as LogSelection from "./LogSelection.res.mjs";
 import * as Stdlib_Array from "@rescript/runtime/lib/es6/Stdlib_Array.js";
 import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js";
@@ -208,34 +208,24 @@ function buildSimulateParamsSchema(params) {
 }
 
 function getTopicEncoder(abiType) {
-  if (abiType.endsWith("]") || abiType.startsWith("(")) {
-    return TopicFilter.castToHexUnsafe;
-  }
-  switch (abiType) {
-    case "address" :
-      return value => TopicFilter.fromAddress(value.toLowerCase());
-    case "bool" :
-      return TopicFilter.fromBool;
-    case "bytes" :
-      return TopicFilter.fromDynamicBytes;
-    case "string" :
-      return TopicFilter.fromDynamicString;
-    default:
-      if (abiType.startsWith("uint")) {
-        return TopicFilter.fromBigInt;
-      } else if (abiType.startsWith("int")) {
-        return TopicFilter.fromSignedBigInt;
-      } else if (abiType.startsWith("bytes")) {
-        return TopicFilter.castToHexUnsafe;
-      } else {
-        return Stdlib_JsError.throwWithMessage(`Unsupported topic filter ABI type: ` + abiType);
-      }
-  }
+  return value => Core.getAddon().encodeIndexedTopic(abiType, value);
 }
 
 function buildTopicGetter(p) {
   let encoder = getTopicEncoder(p.abiType);
-  return eventFilter => Stdlib_Option.mapOr(eventFilter[p.name], [], topicFilters => normalizeOrThrow(topicFilters).map(encoder));
+  let isTuple = p.abiType.startsWith("(");
+  return eventFilter => Stdlib_Option.mapOr(eventFilter[p.name], [], topicFilters => {
+    if (!isTuple) {
+      return normalizeOrThrow(topicFilters).map(encoder);
+    }
+    let encoded;
+    try {
+      encoded = encoder(topicFilters);
+    } catch (exn) {
+      return normalizeOrThrow(topicFilters).map(encoder);
+    }
+    return [encoded];
+  });
 }
 
 let alwaysIncludedBlockFields = [
@@ -317,6 +307,7 @@ function buildEvmOnEventRegistration(eventConfig, isWildcard, handler, contractR
   let sb = resolvedWhere.startBlock;
   let resolvedStartBlock = sb !== undefined ? sb : startBlock;
   return {
+    index: -1,
     eventConfig: eventConfig,
     handler: handler,
     contractRegister: contractRegister,
@@ -369,6 +360,7 @@ function buildSvmInstructionEventConfig(contractName, instructionName, programId
 
 function buildSvmOnEventRegistration(eventConfig, isWildcard, handler, contractRegister, startBlock) {
   return {
+    index: -1,
     eventConfig: eventConfig,
     handler: handler,
     contractRegister: contractRegister,
@@ -438,6 +430,7 @@ function buildFuelEventConfig(contractName, eventName, kind, sighash, rawAbi) {
 
 function buildFuelOnEventRegistration(eventConfig, isWildcard, handler, contractRegister, startBlock) {
   return {
+    index: -1,
     eventConfig: eventConfig,
     handler: handler,
     contractRegister: contractRegister,

package/src/EventProcessing.res

@@ -73,9 +73,10 @@ let runEventHandlerOrThrow = async (
     )
   }
   let handlerDuration = timeBeforeHandler->Performance.secondsSince
+  let eventConfig = eventItem.onEventRegistration.eventConfig
   Prometheus.ProcessingHandler.increment(
-    ~contract=eventItem.onEventRegistration.eventConfig.contractName,
-    ~event=eventItem.onEventRegistration.eventConfig.name,
+    ~contract=eventConfig.contractName,
+    ~event=eventConfig.name,
     ~duration=handlerDuration,
   )
 }
@@ -121,8 +122,8 @@ let runHandlerOrThrow = async (
         }),
       )
     }
-  | Event({onEventRegistration}) =>
-    switch onEventRegistration.handler {
+  | Event(_) =>
+    switch (item->Internal.castUnsafeEventItem).onEventRegistration.handler {
     | Some(handler) =>
       await item->runEventHandlerOrThrow(
         ~handler,
@@ -161,7 +162,9 @@ let preloadBatchOrThrow = async (
     for idx in 0 to checkpointEventsProcessed - 1 {
       let item = batch.items->Array.getUnsafe(itemIdx.contents + idx)
       switch item {
-      | Event({onEventRegistration: {handler, eventConfig: {contractName, name: eventName}}}) =>
+      | Event(_) =>
+        let {handler, eventConfig: {contractName, name: eventName}} =
+          (item->Internal.castUnsafeEventItem).onEventRegistration
         switch handler {
         | None => ()
         | Some(handler) =>

package/src/EventProcessing.res.mjs

@@ -65,7 +65,8 @@ async function runEventHandlerOrThrow(item, checkpointId, handler, indexerState,
     };
   }
   let handlerDuration = Performance.secondsSince(timeBeforeHandler);
-  return Prometheus.ProcessingHandler.increment(item.onEventRegistration.eventConfig.contractName, item.onEventRegistration.eventConfig.name, handlerDuration);
+  let eventConfig = item.onEventRegistration.eventConfig;
+  return Prometheus.ProcessingHandler.increment(eventConfig.contractName, eventConfig.name, handlerDuration);
 }
 
 async function runHandlerOrThrow(item, checkpointId, indexerState, loadManager, persistence, config, chains) {