envio 3.5.0-alpha.0 → 3.5.0-alpha.2

package/src/Config.res

@@ -78,6 +78,9 @@ type t = {
   ecosystem: Ecosystem.t,
   enableRawEvents: bool,
   maxAddrInPartition: int,
+  // Per-contract registered-address count past which a dynamic contract switches
+  // to client-side filtering. Overridable in tests.
+  clientFilterAddressThreshold: int,
   batchSize: int,
   // Slack (in blocks) below the lagged head within which a chain still counts as
   // ready to enter the reorg threshold, absorbing head advances between catch-up
@@ -387,10 +390,6 @@ let getFieldTypeAndSchema = (prop, ~enumConfigsByName: dict<Table.enumConfig<Tab
       let enumConfig = enumConfigsByName->Dict.get(enumName)->Option.getOrThrow
       (Table.Enum({config: enumConfig}), enumConfig.schema->S.toUnknown)
     }
-  | "entity" => {
-      let entityName = prop["entity"]->Option.getOrThrow
-      (Table.Entity({name: entityName}), S.string->S.toUnknown)
-    }
   | other => JsError.throwWithMessage("Unknown field type in entity config: " ++ other)
   }
 
@@ -1033,6 +1032,7 @@ let fromPublic = (publicConfigJson: JSON.t) => {
     enableRawEvents: publicConfig["rawEvents"]->Option.getOr(false),
     ecosystem,
     maxAddrInPartition,
+    clientFilterAddressThreshold: Env.clientFilterAddressThreshold,
     batchSize: publicConfig["fullBatchSize"]->Option.getOr(5000),
     reorgThresholdReadyTolerance: 100,
     lowercaseAddresses,

package/src/Config.res.mjs

@@ -263,16 +263,6 @@ function getFieldTypeAndSchema(prop, enumConfigsByName) {
         Utils.Schema.dbDate
       ];
       break;
-    case "entity" :
-      let entityName = Stdlib_Option.getOrThrow(prop.entity, undefined);
-      match = [
-        {
-          type: "Entity",
-          name: entityName
-        },
-        S$RescriptSchema.string
-      ];
-      break;
     case "enum" :
       let enumName = Stdlib_Option.getOrThrow(prop.enum, undefined);
       let enumConfig = Stdlib_Option.getOrThrow(enumConfigsByName[enumName], undefined);
@@ -724,6 +714,7 @@ function fromPublic(publicConfigJson) {
     ecosystem: ecosystem,
     enableRawEvents: Stdlib_Option.getOr(publicConfig.rawEvents, false),
     maxAddrInPartition: Env.maxAddrInPartition,
+    clientFilterAddressThreshold: Env.clientFilterAddressThreshold,
     batchSize: Stdlib_Option.getOr(publicConfig.fullBatchSize, 5000),
     reorgThresholdReadyTolerance: 100,
     lowercaseAddresses: lowercaseAddresses,

package/src/Core.res

@@ -10,6 +10,7 @@ type svmHyperSyncClientCtor
 type fuelHyperSyncClientCtor
 type transactionStoreCtor
 type blockStoreCtor
+type addressStoreCtor
 type fromUserApiOptions = {
   schema?: string,
   env?: dict<string>,
@@ -27,7 +28,6 @@ type addon = {
   encodeIndexedTopic: (~abiType: string, ~value: unknown) => EvmTypes.Hex.t,
   fromUserApi: (string, fromUserApiOptions) => fromUserApiResult,
   runCli: (~args: array<string>, ~envioPackageDir: Null.t<string>) => promise<Null.t<string>>,
-  requestShutdown: unit => unit,
   @as("EvmHyperSyncClient")
   evmHyperSyncClient: evmHyperSyncClientCtor,
   @as("EvmRpcClient")
@@ -40,6 +40,8 @@ type addon = {
   transactionStore: transactionStoreCtor,
   @as("BlockStore")
   blockStore: blockStoreCtor,
+  @as("AddressStore")
+  addressStore: addressStoreCtor,
   // Ordered transaction-field names exposed for the field-code contract test
   // (the ReScript `transactionFields` arrays must match the Rust ordinals).
   evmTransactionFieldNames: unit => array<string>,
@@ -229,40 +231,7 @@ let fromUserApi = (~schema=?, ~env=?, ~files=?, ~withIndexerTypes=false, yaml) =
   )
 }
 
-let runWithShutdownSignals = %raw(`function(run, requestShutdown) {
-  var onShutdown = function() { requestShutdown(); };
-  process.once("SIGINT", onShutdown);
-  process.once("SIGTERM", onShutdown);
-  return run().finally(function() {
-    process.off("SIGINT", onShutdown);
-    process.off("SIGTERM", onShutdown);
-  });
-}`)
-
-// True only when the parsed subcommand is `serve`. The subcommand is the
-// first positional arg; global value-taking flags (`-d`/`--directory`/
-// `--config`) consume the token after them, and other leading flags are
-// skipped — so an unrelated arg value that happens to equal "serve" (e.g.
-// `envio init --name serve`) does not trigger the serve-only shutdown wiring.
-let isServeCommand: array<string> => bool = %raw(`function(args) {
-  var valueFlags = { "-d": true, "--directory": true, "--config": true };
-  for (var i = 0; i < args.length; i++) {
-    var a = args[i];
-    if (a[0] === "-") {
-      if (a.indexOf("=") === -1 && valueFlags[a]) { i++; }
-      continue;
-    }
-    return a === "serve";
-  }
-  return false;
-}`)
-
 let runCli = args => {
   let addon = getAddon()
-  let invoke = () => addon.runCli(~args, ~envioPackageDir=Null.make(envioPackageDir))
-  if isServeCommand(args) {
-    runWithShutdownSignals(invoke, () => addon.requestShutdown())
-  } else {
-    invoke()
-  }
+  addon.runCli(~args, ~envioPackageDir=Null.make(envioPackageDir))
 }

package/src/Core.res.mjs

@@ -191,37 +191,9 @@ function fromUserApi(schema, env, files, withIndexerTypesOpt, yaml) {
   });
 }
 
-let runWithShutdownSignals = (function(run, requestShutdown) {
-  var onShutdown = function() { requestShutdown(); };
-  process.once("SIGINT", onShutdown);
-  process.once("SIGTERM", onShutdown);
-  return run().finally(function() {
-    process.off("SIGINT", onShutdown);
-    process.off("SIGTERM", onShutdown);
-  });
-});
-
-let isServeCommand = (function(args) {
-  var valueFlags = { "-d": true, "--directory": true, "--config": true };
-  for (var i = 0; i < args.length; i++) {
-    var a = args[i];
-    if (a[0] === "-") {
-      if (a.indexOf("=") === -1 && valueFlags[a]) { i++; }
-      continue;
-    }
-    return a === "serve";
-  }
-  return false;
-});
-
 function runCli(args) {
   let addon = getAddon();
-  let invoke = () => addon.runCli(args, envioPackageDir);
-  if (isServeCommand(args)) {
-    return runWithShutdownSignals(invoke, () => addon.requestShutdown());
-  } else {
-    return addon.runCli(args, envioPackageDir);
-  }
+  return addon.runCli(args, envioPackageDir);
 }
 
 export {
@@ -236,8 +208,6 @@ export {
   getAddon,
   getConfigJson,
   fromUserApi,
-  runWithShutdownSignals,
-  isServeCommand,
   runCli,
 }
 /* envioPackageDir Not a pure module */

package/src/CrossChainState.res

@@ -249,19 +249,6 @@ let checkAndFetch = async (
   // while it takes its first measurements. Its probe is one admission unit.
   let coldChainBudget = minimumAdmissionBudget
 
-  // Chunk reservations get headroom over the density estimate so a
-  // denser-than-expected range doesn't truncate at the server cap; realtime
-  // gets more since a forced catch-up query there costs a head-poll roundtrip.
-  let chunkItemsMultiplier = crossChainState.isRealtime ? 3. : 1.5
-
-  // Server-cap floor for bounded queries: their block range is already the
-  // hard bound on the response, so a low density estimate shrinking the cap
-  // below this only buys self-truncated responses. Splitting the target pool
-  // across a chain's concurrency slots keeps the worst case — every in-flight
-  // bounded query returning a full floored response at once — at ~one buffer
-  // target.
-  let itemsTargetFloor = crossChainState.targetBufferSize / FetchState.maxChainConcurrency
-
   let prioritizedChainStates = crossChainState->priorityOrder
 
   // Alignment anchor: the first known-height chain in priority order — which,
@@ -308,12 +295,7 @@ let checkAndFetch = async (
         Some(cs->ChainState.blockAtProgress(~progress=progress +. 0.1))
       | _ => None
       }
-      switch cs->ChainState.getNextQuery(
-        ~chainTargetItems,
-        ~chunkItemsMultiplier,
-        ~itemsTargetFloor,
-        ~maxTargetBlock?,
-      ) {
+      switch cs->ChainState.getNextQuery(~chainTargetItems, ~maxTargetBlock?) {
       | WaitingForNewBlock as action => actionByChain->Utils.Dict.setByInt(chainId, action)
       | NothingToQuery =>
         // A chain below its head can emit no query when its budget went to
@@ -333,7 +315,7 @@ let checkAndFetch = async (
               {
                 "fromBlock": query.fromBlock,
                 "targetBlock": query.toBlock,
-                "targetEvents": query.itemsTarget,
+                "targetEvents": query.itemsEst,
               },
             )
           )

package/src/CrossChainState.res.mjs

@@ -6,9 +6,7 @@ import * as Utils from "./Utils.res.mjs";
 import * as Logging from "./Logging.res.mjs";
 import * as ChainMap from "./ChainMap.res.mjs";
 import * as ChainState from "./ChainState.res.mjs";
-import * as FetchState from "./FetchState.res.mjs";
 import * as Stdlib_Array from "@rescript/runtime/lib/es6/Stdlib_Array.js";
-import * as Primitive_int from "@rescript/runtime/lib/es6/Primitive_int.js";
 import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js";
 import * as Primitive_float from "@rescript/runtime/lib/es6/Primitive_float.js";
 import * as SafeCheckpointTracking from "./SafeCheckpointTracking.res.mjs";
@@ -147,8 +145,6 @@ async function checkAndFetch(crossChainState, dispatchChain) {
     contents: Primitive_float.max(0, targetBudget - totalReadyCount(crossChainState) - totalReservedSize(crossChainState))
   };
   let minimumAdmissionBudget = targetBudget * 0.1;
-  let chunkItemsMultiplier = crossChainState.isRealtime ? 3 : 1.5;
-  let itemsTargetFloor = Primitive_int.div(crossChainState.targetBufferSize, FetchState.maxChainConcurrency);
   let prioritizedChainStates = priorityOrder(crossChainState);
   let alignment = crossChainState.isRealtime ? undefined : Stdlib_Option.map(prioritizedChainStates.find(cs => ChainState.knownHeight(cs) !== 0), cs => [
       ChainState.chainConfig(cs).id,
@@ -170,7 +166,7 @@ async function checkAndFetch(crossChainState, dispatchChain) {
       isCold ? Primitive_float.min(remaining.contents, minimumAdmissionBudget) : remaining.contents
     ) + ChainState.pendingBudget(cs);
     let maxTargetBlock = alignment !== undefined && alignment[0] !== chainId ? ChainState.blockAtProgress(cs, alignment[1] + 0.1) : undefined;
-    let action = ChainState.getNextQuery(cs, chainTargetItems, chunkItemsMultiplier, itemsTargetFloor, maxTargetBlock);
+    let action = ChainState.getNextQuery(cs, chainTargetItems, maxTargetBlock);
     if (typeof action !== "object") {
       if (action === "WaitingForNewBlock") {
         actionByChain[chainId] = action;
@@ -186,7 +182,7 @@ async function checkAndFetch(crossChainState, dispatchChain) {
         partitions[query.partitionId] = {
           fromBlock: query.fromBlock,
           targetBlock: query.toBlock,
-          targetEvents: query.itemsTarget
+          targetEvents: query.itemsEst
         };
       });
       Logging.trace({

package/src/EntityId.res

@@ -0,0 +1,15 @@
+// Opaque representation of an entity id. The generated per-entity types expose
+// the real scalar (string / int / bigint); the generic storage and in-memory
+// layers work with any entity's id through this type without knowing which
+// scalar backs it. The runtime value is always the real id, never a stringified
+// form — `toKey` derives the string only where a JS object/dict key is needed.
+type t
+
+external unsafeOfAny: 'a => t = "%identity"
+external unsafeToAny: t => 'a = "%identity"
+external unsafeOfString: string => t = "%identity"
+
+// Stringified id used as a JS object/dict key. `String` matches how JS coerces
+// a value used as an object key, so an id indexes the same whether the raw
+// value or its key form is used for lookup, across string/int/bigint.
+let toKey: t => string = %raw(`String`)

package/src/EntityId.res.mjs

@@ -0,0 +1,9 @@
+// Generated by ReScript, PLEASE EDIT WITH CARE
+
+
+let toKey = String;
+
+export {
+  toKey,
+}
+/* toKey Not a pure module */

package/src/Env.res

@@ -9,6 +9,22 @@ let updateSyncTimeOnRestart =
 let targetBufferSize = envSafe->EnvSafe.get("ENVIO_INDEXING_MAX_BUFFER_SIZE", S.option(S.int))
 let maxAddrInPartition = envSafe->EnvSafe.get("MAX_PARTITION_SIZE", S.int, ~fallback=5_000)
 
+// Most parallel in-flight queries a single chain may have at once, across all
+// its partitions (consumed as FetchState.maxChainConcurrency).
+let maxChainConcurrency = 100
+
+// Switch a single contract to client-side address filtering
+// once its registered address count crosses this threshold. Keeping addresses
+// server-side spreads the contract across ceil(count / maxAddrInPartition)
+// partitions, each holding an in-flight query slot; capping a contract at half
+// the chain's concurrency budget stops one busy contract from monopolising them.
+let clientFilterAddressThreshold =
+  envSafe->EnvSafe.get(
+    "ENVIO_CLIENT_FILTER_ADDRESS_THRESHOLD",
+    S.int,
+    ~fallback=maxAddrInPartition * maxChainConcurrency / 2,
+  )
+
 // Target number of in-memory objects (uncommitted entity/effect changes plus
 // unwritten batch items) the store holds before processing waits for the write
 // cycle to catch up.

package/src/Env.res.mjs

@@ -19,6 +19,8 @@ let targetBufferSize = EnvSafe.get(envSafe, "ENVIO_INDEXING_MAX_BUFFER_SIZE", S$
 
 let maxAddrInPartition = EnvSafe.get(envSafe, "MAX_PARTITION_SIZE", S$RescriptSchema.int, undefined, 5000, undefined, undefined);
 
+let clientFilterAddressThreshold = EnvSafe.get(envSafe, "ENVIO_CLIENT_FILTER_ADDRESS_THRESHOLD", S$RescriptSchema.int, undefined, (maxAddrInPartition * 100 | 0) / 2 | 0, undefined, undefined);
+
 let inMemoryObjectsTarget = EnvSafe.get(envSafe, "ENVIO_IN_MEMORY_OBJECTS_TARGET", S$RescriptSchema.int, undefined, 100000, undefined, undefined);
 
 let serverPort = EnvSafe.get(envSafe, "ENVIO_INDEXER_PORT", S$RescriptSchema.port(S$RescriptSchema.int, undefined), undefined, EnvSafe.get(envSafe, "METRICS_PORT", S$RescriptSchema.port(S$RescriptSchema.int, undefined), undefined, 9898, undefined, undefined), undefined, undefined);
@@ -216,10 +218,14 @@ let ThrottleWrites = {
 
 EnvSafe.close(envSafe);
 
+let maxChainConcurrency = 100;
+
 export {
   updateSyncTimeOnRestart,
   targetBufferSize,
   maxAddrInPartition,
+  maxChainConcurrency,
+  clientFilterAddressThreshold,
   inMemoryObjectsTarget,
   serverPort,
   tuiEnvVar,

package/src/EventConfigBuilder.res

@@ -292,61 +292,6 @@ let resolveFieldSelection = (
   )
 }
 
-// ============== Client-side address filter ==============
-
-let compileAddressFilter: string => (
-  Internal.eventPayload,
-  int,
-  dict<Internal.indexingContract>,
-) => bool = %raw(`function (body) {
-  return new Function("event", "blockNumber", "indexingAddresses", body);
-}`)
-
-// Body of the client-side address filter. Two analogous registered-at-or-before
-// checks, ANDed: (1) for non-wildcard events, the log's srcAddress must itself be
-// registered (ownership is resolved structurally by partition, but the temporal
-// `effectiveStartBlock` gate lives here now); (2) a DNF of address-filtered param
-// names (OR of AND-groups) for events that filter an indexed address param. The
-// DNF is fixed here, so it's unrolled into one boolean expression — no per-event
-// closure, loop, or array. `None` only for wildcard events without a param
-// filter. Exposed for snapshotting.
-// `srcAddressExpr` is the JS expression for the event's owning address: EVM and
-// Fuel events expose `event.srcAddress`; SVM instructions expose `event.programId`.
-let buildAddressFilterBody = (
-  groups: array<array<string>>,
-  ~isWildcard: bool,
-  ~srcAddressExpr: string="event.srcAddress",
-): option<string> => {
-  let paramLeaf = name =>
-    `(ic = indexingAddresses[p[${JSON.stringify(
-        JSON.String(name),
-      )}]]) !== undefined && ic.effectiveStartBlock <= blockNumber`
-  let paramDnf = switch groups {
-  | [] => None
-  | _ =>
-    Some(
-      groups
-      ->Array.map(group => "(" ++ group->Array.map(paramLeaf)->Array.join(" && ") ++ ")")
-      ->Array.join(" || "),
-    )
-  }
-  let srcLeaf = `(ic = indexingAddresses[${srcAddressExpr}]) !== undefined && ic.effectiveStartBlock <= blockNumber`
-  switch (isWildcard, paramDnf) {
-  | (true, None) => None
-  | (true, Some(dnf)) => Some("var p = event.params, ic; return " ++ dnf ++ ";")
-  | (false, None) => Some("var ic; return " ++ srcLeaf ++ ";")
-  | (false, Some(dnf)) =>
-    Some("var p = event.params, ic; return " ++ srcLeaf ++ " && (" ++ dnf ++ ");")
-  }
-}
-
-let buildAddressFilter = (
-  groups: array<array<string>>,
-  ~isWildcard: bool,
-  ~srcAddressExpr: string="event.srcAddress",
-): option<(Internal.eventPayload, int, dict<Internal.indexingContract>) => bool> =>
-  buildAddressFilterBody(groups, ~isWildcard, ~srcAddressExpr)->Option.map(compileAddressFilter)
-
 // ============== Build complete EVM event config ==============
 
 let buildEvmEventConfig = (
@@ -431,7 +376,7 @@ let buildEvmOnEventRegistration = (
     contractRegister,
     resolvedWhere,
     filterByAddresses,
-    clientAddressFilter: ?buildAddressFilter(addressFilterParamGroups, ~isWildcard),
+    addressFilterParamGroups,
     dependsOnAddresses: Internal.dependsOnAddresses(~isWildcard, ~filterByAddresses),
     startBlock: resolvedStartBlock,
   }
@@ -516,7 +461,6 @@ let buildSvmOnEventRegistration = (
   isWildcard,
   filterByAddresses: false,
   dependsOnAddresses: Internal.dependsOnAddresses(~isWildcard, ~filterByAddresses=false),
-  clientAddressFilter: ?buildAddressFilter([], ~isWildcard, ~srcAddressExpr="event.programId"),
   startBlock,
 }
 
@@ -594,6 +538,5 @@ let buildFuelOnEventRegistration = (
   isWildcard,
   filterByAddresses: false,
   dependsOnAddresses: Internal.dependsOnAddresses(~isWildcard, ~filterByAddresses=false),
-  clientAddressFilter: ?buildAddressFilter([], ~isWildcard),
   startBlock,
 }

package/src/EventConfigBuilder.res.mjs

@@ -243,33 +243,6 @@ function resolveFieldSelection(blockFields, transactionFields, globalBlockFields
   ];
 }
 
-let compileAddressFilter = (function (body) {
-  return new Function("event", "blockNumber", "indexingAddresses", body);
-});
-
-function buildAddressFilterBody(groups, isWildcard, srcAddressExprOpt) {
-  let srcAddressExpr = srcAddressExprOpt !== undefined ? srcAddressExprOpt : "event.srcAddress";
-  let paramLeaf = name => `(ic = indexingAddresses[p[` + JSON.stringify(name) + `]]) !== undefined && ic.effectiveStartBlock <= blockNumber`;
-  let paramDnf = groups.length !== 0 ? groups.map(group => "(" + group.map(paramLeaf).join(" && ") + ")").join(" || ") : undefined;
-  let srcLeaf = `(ic = indexingAddresses[` + srcAddressExpr + `]) !== undefined && ic.effectiveStartBlock <= blockNumber`;
-  if (isWildcard) {
-    if (paramDnf !== undefined) {
-      return "var p = event.params, ic; return " + paramDnf + ";";
-    } else {
-      return;
-    }
-  } else if (paramDnf !== undefined) {
-    return "var p = event.params, ic; return " + srcLeaf + " && (" + paramDnf + ");";
-  } else {
-    return "var ic; return " + srcLeaf + ";";
-  }
-}
-
-function buildAddressFilter(groups, isWildcard, srcAddressExprOpt) {
-  let srcAddressExpr = srcAddressExprOpt !== undefined ? srcAddressExprOpt : "event.srcAddress";
-  return Stdlib_Option.map(buildAddressFilterBody(groups, isWildcard, srcAddressExpr), compileAddressFilter);
-}
-
 function buildEvmEventConfig(contractName, eventName, sighash, params, blockFields, transactionFields, globalBlockFieldsSetOpt, globalTransactionFieldsSetOpt) {
   let globalBlockFieldsSet = globalBlockFieldsSetOpt !== undefined ? Primitive_option.valFromOption(globalBlockFieldsSetOpt) : new Set();
   let globalTransactionFieldsSet = globalTransactionFieldsSetOpt !== undefined ? Primitive_option.valFromOption(globalTransactionFieldsSetOpt) : new Set();
@@ -314,7 +287,7 @@ function buildEvmOnEventRegistration(eventConfig, isWildcard, handler, contractR
     isWildcard: isWildcard,
     filterByAddresses: filterByAddresses,
     dependsOnAddresses: Internal.dependsOnAddresses(isWildcard, filterByAddresses),
-    clientAddressFilter: buildAddressFilter(match.addressFilterParamGroups, isWildcard, undefined),
+    addressFilterParamGroups: match.addressFilterParamGroups,
     startBlock: resolvedStartBlock,
     resolvedWhere: resolvedWhere
   };
@@ -367,7 +340,6 @@ function buildSvmOnEventRegistration(eventConfig, isWildcard, handler, contractR
     isWildcard: isWildcard,
     filterByAddresses: false,
     dependsOnAddresses: Internal.dependsOnAddresses(isWildcard, false),
-    clientAddressFilter: buildAddressFilter([], isWildcard, "event.programId"),
     startBlock: startBlock
   };
 }
@@ -437,7 +409,6 @@ function buildFuelOnEventRegistration(eventConfig, isWildcard, handler, contract
     isWildcard: isWildcard,
     filterByAddresses: false,
     dependsOnAddresses: Internal.dependsOnAddresses(isWildcard, false),
-    clientAddressFilter: buildAddressFilter([], isWildcard, undefined),
     startBlock: startBlock
   };
 }
@@ -457,9 +428,6 @@ export {
   buildTopicGetter,
   alwaysIncludedBlockFields,
   resolveFieldSelection,
-  compileAddressFilter,
-  buildAddressFilterBody,
-  buildAddressFilter,
   buildEvmEventConfig,
   buildEvmOnEventRegistration,
   alwaysIncludedSvmBlockFields,