envio 3.3.0-alpha.3 → 3.3.0-alpha.5

package/src/IndexingAddresses.res

@@ -0,0 +1,105 @@
+type indexingAddress = Internal.indexingContract
+
+type contractConfig = {startBlock: option<int>}
+
+type t = dict<indexingAddress>
+
+let deriveEffectiveStartBlock = (~registrationBlock: int, ~contractStartBlock: option<int>) => {
+  Pervasives.max(Pervasives.max(registrationBlock, 0), contractStartBlock->Option.getOr(0))
+}
+
+let makeContractConfigs = (~eventConfigs: array<Internal.eventConfig>): dict<contractConfig> => {
+  let contractConfigs: dict<contractConfig> = Dict.make()
+  eventConfigs->Array.forEach(ec => {
+    switch contractConfigs->Utils.Dict.dangerouslyGetNonOption(ec.contractName) {
+    | Some({startBlock}) =>
+      contractConfigs->Dict.set(
+        ec.contractName,
+        {
+          startBlock: switch (startBlock, ec.startBlock) {
+          | (Some(a), Some(b)) => Some(Pervasives.min(a, b))
+          | (Some(_) as s, None) | (None, Some(_) as s) => s
+          | (None, None) => None
+          },
+        },
+      )
+    | None =>
+      contractConfigs->Dict.set(
+        ec.contractName,
+        {
+          startBlock: ec.startBlock,
+        },
+      )
+    }
+  })
+  contractConfigs
+}
+
+let makeIndexingAddress = (
+  ~contract: Internal.indexingAddress,
+  ~contractConfigs: dict<contractConfig>,
+): indexingAddress => {
+  let contractStartBlock = switch contractConfigs->Utils.Dict.dangerouslyGetNonOption(
+    contract.contractName,
+  ) {
+  | Some({startBlock}) => startBlock
+  | None => None
+  }
+  {
+    address: contract.address,
+    contractName: contract.contractName,
+    registrationBlock: contract.registrationBlock,
+    effectiveStartBlock: deriveEffectiveStartBlock(
+      ~registrationBlock=contract.registrationBlock,
+      ~contractStartBlock,
+    ),
+  }
+}
+
+let make = (
+  ~contractConfigs: dict<contractConfig>,
+  ~addresses: array<Internal.indexingAddress>,
+): t => {
+  let indexingAddresses = Dict.make()
+  addresses->Array.forEach(contract => {
+    indexingAddresses->Dict.set(
+      contract.address->Address.toString,
+      makeIndexingAddress(~contract, ~contractConfigs),
+    )
+  })
+  indexingAddresses
+}
+
+let get = (indexingAddresses: t, address) =>
+  indexingAddresses->Utils.Dict.dangerouslyGetNonOption(address)
+
+let size = (indexingAddresses: t) => indexingAddresses->Utils.Dict.size
+
+let getContractAddresses = (indexingAddresses: t, ~contractName): array<Address.t> => {
+  let addresses = []
+  indexingAddresses->Utils.Dict.forEach(ia => {
+    if ia.contractName === contractName {
+      addresses->Array.push(ia.address)
+    }
+  })
+  addresses
+}
+
+// Underlying dict for the precompiled `clientAddressFilter` only — it does raw
+// `indexingAddresses[srcAddress]` access in generated JS and can't take the opaque
+// type. Don't reach for this elsewhere; use the domain accessors above.
+let rawForFilter = (indexingAddresses: t): dict<indexingAddress> => indexingAddresses
+
+let register = (indexingAddresses: t, additions: dict<indexingAddress>) => {
+  let _ = Utils.Dict.mergeInPlace(indexingAddresses, additions)
+}
+
+let rollbackInPlace = (indexingAddresses: t, ~targetBlockNumber: int): unit => {
+  // forEachWithKey is a `for..in`, so deleting the key currently being visited is
+  // safe — it doesn't affect enumeration of the remaining keys.
+  indexingAddresses->Utils.Dict.forEachWithKey((indexingContract, address) => {
+    if indexingContract.registrationBlock > targetBlockNumber {
+      indexingAddresses->Utils.Dict.deleteInPlace(address)
+    }
+  })
+}

package/src/IndexingAddresses.res.mjs

@@ -0,0 +1,100 @@
+// Generated by ReScript, PLEASE EDIT WITH CARE
+
+import * as Utils from "./Utils.res.mjs";
+import * as Primitive_int from "@rescript/runtime/lib/es6/Primitive_int.js";
+import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js";
+
+function deriveEffectiveStartBlock(registrationBlock, contractStartBlock) {
+  return Primitive_int.max(Primitive_int.max(registrationBlock, 0), Stdlib_Option.getOr(contractStartBlock, 0));
+}
+
+function makeContractConfigs(eventConfigs) {
+  let contractConfigs = {};
+  eventConfigs.forEach(ec => {
+    let match = contractConfigs[ec.contractName];
+    if (match !== undefined) {
+      let startBlock = match.startBlock;
+      let match$1 = ec.startBlock;
+      contractConfigs[ec.contractName] = {
+        startBlock: startBlock !== undefined ? (
+            match$1 !== undefined ? Primitive_int.min(startBlock, match$1) : startBlock
+          ) : (
+            match$1 !== undefined ? match$1 : undefined
+          )
+      };
+      return;
+    }
+    contractConfigs[ec.contractName] = {
+      startBlock: ec.startBlock
+    };
+  });
+  return contractConfigs;
+}
+
+function makeIndexingAddress(contract, contractConfigs) {
+  let match = contractConfigs[contract.contractName];
+  let contractStartBlock = match !== undefined ? match.startBlock : undefined;
+  return {
+    address: contract.address,
+    contractName: contract.contractName,
+    registrationBlock: contract.registrationBlock,
+    effectiveStartBlock: deriveEffectiveStartBlock(contract.registrationBlock, contractStartBlock)
+  };
+}
+
+function make(contractConfigs, addresses) {
+  let indexingAddresses = {};
+  addresses.forEach(contract => {
+    indexingAddresses[contract.address] = makeIndexingAddress(contract, contractConfigs);
+  });
+  return indexingAddresses;
+}
+
+function get(indexingAddresses, address) {
+  return indexingAddresses[address];
+}
+
+function size(indexingAddresses) {
+  return Utils.Dict.size(indexingAddresses);
+}
+
+function getContractAddresses(indexingAddresses, contractName) {
+  let addresses = [];
+  Utils.Dict.forEach(indexingAddresses, ia => {
+    if (ia.contractName === contractName) {
+      addresses.push(ia.address);
+      return;
+    }
+  });
+  return addresses;
+}
+
+function rawForFilter(indexingAddresses) {
+  return indexingAddresses;
+}
+
+function register(indexingAddresses, additions) {
+  Object.assign(indexingAddresses, additions);
+}
+
+function rollbackInPlace(indexingAddresses, targetBlockNumber) {
+  Utils.Dict.forEachWithKey(indexingAddresses, (indexingContract, address) => {
+    if (indexingContract.registrationBlock > targetBlockNumber) {
+      return Utils.Dict.deleteInPlace(indexingAddresses, address);
+    }
+  });
+}
+
+export {
+  deriveEffectiveStartBlock,
+  makeContractConfigs,
+  makeIndexingAddress,
+  make,
+  get,
+  size,
+  getContractAddresses,
+  rawForFilter,
+  register,
+  rollbackInPlace,
+}
+/* Utils Not a pure module */

package/src/IndexingAddresses.resi

@@ -0,0 +1,32 @@
+type indexingAddress = Internal.indexingContract
+
+type contractConfig = {startBlock: option<int>}
+
+type t
+
+let deriveEffectiveStartBlock: (~registrationBlock: int, ~contractStartBlock: option<int>) => int
+
+let makeContractConfigs: (~eventConfigs: array<Internal.eventConfig>) => dict<contractConfig>
+
+let makeIndexingAddress: (
+  ~contract: Internal.indexingAddress,
+  ~contractConfigs: dict<contractConfig>,
+) => indexingAddress
+
+let make: (~contractConfigs: dict<contractConfig>, ~addresses: array<Internal.indexingAddress>) => t
+
+let get: (t, string) => option<indexingAddress>
+
+let size: t => int
+
+let getContractAddresses: (t, ~contractName: string) => array<Address.t>
+
+// Underlying dict for the precompiled `clientAddressFilter` only, which does raw
+// `indexingAddresses[srcAddress]` access in generated JS. Don't use elsewhere —
+// prefer the domain accessors so the opaque type stays enforced.
+let rawForFilter: t => dict<indexingAddress>
+
+let register: (t, dict<indexingAddress>) => unit
+
+// Deletes addresses registered after `targetBlockNumber` from the index in place.
+let rollbackInPlace: (t, ~targetBlockNumber: int) => unit

package/src/Internal.res

@@ -420,6 +420,11 @@ type eventConfig = private {
   // string set so the shared mask logic is ecosystem-agnostic; sources recover
   // the typed view where they need it.
   selectedTransactionFields: Utils.Set.t<string>,
+  // `selectedTransactionFields` precompiled to the transaction-store selection
+  // bitmask (bit per ecosystem field code). Materialisation reads this per item
+  // so each transaction decodes only the fields its event selected. `0.` when
+  // nothing is selected or the ecosystem carries the transaction inline (Fuel).
+  transactionFieldMask: float,
 }
 
 type fuelEventKind =

package/src/Main.res

@@ -168,18 +168,7 @@ let buildChainsObject = (~config: Config.t) => {
             | Some(state) => {
                 let chain = ChainMap.Chain.makeUnsafe(~chainId=chainConfig.id)
                 let chainState = state->IndexerState.getChainState(~chain)
-                let indexingAddresses = chainState->ChainState.indexingAddresses
-
-                // Collect all addresses for this contract name from indexingAddresses
-                let addresses = []
-                let values = indexingAddresses->Dict.valuesToArray
-                for idx in 0 to values->Array.length - 1 {
-                  let indexingContract = values->Array.getUnsafe(idx)
-                  if indexingContract.contractName === contract.name {
-                    addresses->Array.push(indexingContract.address)->ignore
-                  }
-                }
-                addresses
+                chainState->ChainState.contractAddresses(~contractName=contract.name)
               }
             // Before the global state is available (eg during handler
             // module load after resume), combine static addresses from config
@@ -627,9 +616,9 @@ let startServer = (~getState, ~persistence: Persistence.t, ~isDevelopmentMode: b
   app->get("/metrics", (_req, res) => {
     res->set("Content-Type", PromClient.defaultRegister->PromClient.getContentType)
     let _ =
-      PromClient.defaultRegister
-      ->PromClient.metrics
-      ->Promise.thenResolve(metrics => res->endWithData(metrics))
+      Metrics.collect(~state=indexerStateRef.contents)->Promise.thenResolve(metrics =>
+        res->endWithData(metrics)
+      )
   })
 
   app->get("/metrics/runtime", (_req, res) => {
@@ -687,6 +676,10 @@ let dropSchema = async () => {
   await persistence.storage.close()
 }
 
+// Rejection carried by `onError`: the failure is already logged with full
+// context, so callers should act on it (exit / re-throw) without logging again.
+exception FatalError(exn)
+
 let start = async (
   ~persistence: option<Persistence.t>=?,
   ~reset=false,
@@ -742,17 +735,20 @@ let start = async (
   | None => config
   }
   // The single fatal-error handler, invoked once via IndexerState.errorExit.
+  // It logs the failure once (with chain context) and rejects the run wrapped in
+  // `FatalError` so callers know it's already logged — `Bin.res` just exits, the
+  // test worker unwraps and re-throws it to the parent thread. `runUntilFatalError`
+  // only ever rejects: on a clean run it stays pending and the process exits via
+  // ExitOnCaughtUp / when the indexer loop drains.
+  let onErrorReject = ref(None)
+  let runUntilFatalError: promise<unit> = Promise.make((_resolve, reject) =>
+    onErrorReject := Some(reject)
+  )
+  // `onErrorReject` is filled synchronously by `Promise.make` above, before the
+  // indexer can run and call `onError`, so it's always present here.
   let onError = (errHandler: ErrorHandling.t) => {
     errHandler->ErrorHandling.log
-    if isTest {
-      // The TestIndexer runs the indexer in a worker thread and reads the
-      // failure off the worker 'error' event. Re-throw the original error
-      // outside the promise chain on the next tick so the test sees its real
-      // message instead of a generic non-zero exit code.
-      NodeJs.setImmediate(() => errHandler->ErrorHandling.raiseExn)
-    } else {
-      NodeJs.process->NodeJs.exitWithCode(Failure)
-    }
+    (onErrorReject.contents->Option.getUnsafe)(FatalError(errHandler.exn->Utils.prettifyExn))
   }
   let envioVersion = Utils.EnvioPackage.value.version
   Prometheus.Info.set(~version=envioVersion)
@@ -793,4 +789,5 @@ let start = async (
   }
   indexerStateRef := Some(state)
   state->IndexerLoop.start
+  await runUntilFatalError
 }

package/src/Main.res.mjs

@@ -6,6 +6,7 @@ import * as Envio from "./Envio.res.mjs";
 import * as Utils from "./Utils.res.mjs";
 import * as Config from "./Config.res.mjs";
 import * as Logging from "./Logging.res.mjs";
+import * as Metrics from "./Metrics.res.mjs";
 import Express from "express";
 import * as Process from "process";
 import * as ChainMap from "./ChainMap.res.mjs";
@@ -165,29 +166,20 @@ function buildChainsObject(config) {
           if (state !== undefined) {
             let chain = ChainMap.Chain.makeUnsafe(chainConfig.id);
             let chainState = IndexerState.getChainState(Primitive_option.valFromOption(state), chain);
-            let indexingAddresses = ChainState.indexingAddresses(chainState);
-            let addresses = [];
-            let values = Object.values(indexingAddresses);
-            for (let idx = 0, idx_finish = values.length; idx < idx_finish; ++idx) {
-              let indexingContract = values[idx];
-              if (indexingContract.contractName === contract.name) {
-                addresses.push(indexingContract.address);
-              }
-            }
-            return addresses;
+            return ChainState.contractAddresses(chainState, contract.name);
           }
           let chainState$1 = getInitialChainState(chainConfig.id);
           if (chainState$1 === undefined) {
             return contract.addresses;
           }
-          let addresses$1 = contract.addresses.slice();
+          let addresses = contract.addresses.slice();
           chainState$1.indexingAddresses.forEach(dc => {
             if (dc.contractName === contract.name) {
-              addresses$1.push(dc.address);
+              addresses.push(dc.address);
               return;
             }
           });
-          return addresses$1;
+          return addresses;
         }
       });
       Object.defineProperty(chainObj, contract.name, {
@@ -486,7 +478,7 @@ function startServer(getState, persistence, isDevelopmentMode) {
   });
   app.get("/metrics", (_req, res) => {
     res.set("Content-Type", PromClient.register.contentType);
-    PromClient.register.metrics().then(metrics => res.end(metrics));
+    Metrics.collect(indexerStateRef.contents).then(metrics => res.end(metrics));
   });
   app.get("/metrics/runtime", (_req, res) => {
     res.set("Content-Type", runtimeRegistry.contentType);
@@ -522,6 +514,8 @@ async function dropSchema() {
   return await persistence.storage.close();
 }
 
+let FatalError = /* @__PURE__ */Primitive_exceptions.create("Main.FatalError");
+
 async function start(persistence, resetOpt, isTestOpt, exitAfterFirstEventBlockOpt, patchConfig) {
   let reset = resetOpt !== undefined ? resetOpt : false;
   let isTest = isTestOpt !== undefined ? isTestOpt : false;
@@ -562,13 +556,18 @@ async function start(persistence, resetOpt, isTestOpt, exitAfterFirstEventBlockO
       allEnums: config.allEnums
     }) : config;
   let config$2 = patchConfig !== undefined ? patchConfig(config$1, registrations) : config$1;
+  let onErrorReject = {
+    contents: undefined
+  };
+  let runUntilFatalError = new Promise((_resolve, reject) => {
+    onErrorReject.contents = reject;
+  });
   let onError = errHandler => {
     ErrorHandling.log(errHandler);
-    if (isTest) {
-      setImmediate(() => ErrorHandling.raiseExn(errHandler));
-    } else {
-      Process.exit(1);
-    }
+    onErrorReject.contents({
+      RE_EXN_ID: FatalError,
+      _1: Utils.prettifyExn(errHandler.exn)
+    });
   };
   let envioVersion = Utils.EnvioPackage.value.version;
   Prometheus.Info.set(envioVersion);
@@ -599,7 +598,8 @@ async function start(persistence, resetOpt, isTestOpt, exitAfterFirstEventBlockO
     Tui.start(() => state);
   }
   indexerStateRef.contents = Primitive_option.some(state);
-  return IndexerLoop.start(state);
+  IndexerLoop.start(state);
+  return await runUntilFatalError;
 }
 
 export {
@@ -616,6 +616,7 @@ export {
   getEnvioInfo,
   migrate,
   dropSchema,
+  FatalError,
   start,
 }
 /* chainDataSchema Not a pure module */

package/src/Metrics.res

@@ -0,0 +1,33 @@
+// Pull-based metrics computed from live indexer state at scrape time, merged with
+// the imperatively-updated prom-client default registry. Used because the address
+// index is mutated in place (no single update site to fire a gauge from).
+
+let indexingAddressesName = "envio_indexing_addresses"
+let indexingAddressesHelp = "The number of addresses indexed on chain. Includes both static and dynamic addresses."
+
+// Render a gauge straight from the per-chain dict, one sample per key, without
+// materialising an intermediate samples array. Accumulate into one string rather
+// than building a lines array to join: `++` compiles to JS `+=`, which V8 grows
+// as a ConsString instead of recopying.
+let renderGauge = (~name, ~help, ~chains: dict<'a>, ~value: 'a => int) => {
+  let out = ref(`# HELP ${name} ${help}\n# TYPE ${name} gauge`)
+  let prefix = `\n${name}{chainId="`
+  chains->Utils.Dict.forEachWithKey((chain, chainId) => {
+    out := out.contents ++ prefix ++ chainId ++ `"} ` ++ value(chain)->Int.toString
+  })
+  out.contents
+}
+
+let collect = async (~state: option<IndexerState.t>) => {
+  let base = await PromClient.defaultRegister->PromClient.metrics
+  switch state {
+  | None => base
+  | Some(state) =>
+    `${base}${renderGauge(
+        ~name=indexingAddressesName,
+        ~help=indexingAddressesHelp,
+        ~chains=state->IndexerState.chainStates,
+        ~value=cs => (cs->ChainState.toChainData).numAddresses,
+      )}\n`
+  }
+}

package/src/Metrics.res.mjs

@@ -0,0 +1,39 @@
+// Generated by ReScript, PLEASE EDIT WITH CARE
+
+import * as Utils from "./Utils.res.mjs";
+import * as ChainState from "./ChainState.res.mjs";
+import * as PromClient from "prom-client";
+import * as IndexerState from "./IndexerState.res.mjs";
+import * as Primitive_option from "@rescript/runtime/lib/es6/Primitive_option.js";
+
+let indexingAddressesName = "envio_indexing_addresses";
+
+let indexingAddressesHelp = "The number of addresses indexed on chain. Includes both static and dynamic addresses.";
+
+function renderGauge(name, help, chains, value) {
+  let out = {
+    contents: `# HELP ` + name + ` ` + help + `\n# TYPE ` + name + ` gauge`
+  };
+  let prefix = `\n` + name + `{chainId="`;
+  Utils.Dict.forEachWithKey(chains, (chain, chainId) => {
+    out.contents = out.contents + prefix + chainId + `"} ` + value(chain).toString();
+  });
+  return out.contents;
+}
+
+async function collect(state) {
+  let base = await PromClient.register.metrics();
+  if (state !== undefined) {
+    return base + renderGauge(indexingAddressesName, indexingAddressesHelp, IndexerState.chainStates(Primitive_option.valFromOption(state)), cs => ChainState.toChainData(cs).numAddresses) + `\n`;
+  } else {
+    return base;
+  }
+}
+
+export {
+  indexingAddressesName,
+  indexingAddressesHelp,
+  renderGauge,
+  collect,
+}
+/* Utils Not a pure module */

package/src/Prometheus.res

@@ -246,10 +246,7 @@ module PreloadHandler = {
       )
       operations->Utils.Dict.deleteInPlace(key)
     }
-    sumTimeCounter->SafeCounter.handleFloat(
-      ~labels,
-      ~value=timerRef->Performance.secondsSince,
-    )
+    sumTimeCounter->SafeCounter.handleFloat(~labels, ~value=timerRef->Performance.secondsSince)
     count->SafeCounter.increment(~labels)
   }
 }
@@ -339,18 +336,6 @@ module ProcessStartTimeSeconds = {
   }
 }
 
-module IndexingAddresses = {
-  let gauge = SafeGauge.makeOrThrow(
-    ~name="envio_indexing_addresses",
-    ~help="The number of addresses indexed on chain. Includes both static and dynamic addresses.",
-    ~labelSchema=chainIdLabelsSchema,
-  )
-
-  let set = (~addressesCount, ~chainId) => {
-    gauge->SafeGauge.handleInt(~labels=chainId, ~value=addressesCount)
-  }
-}
-
 module IndexingConcurrency = {
   let gauge = SafeGauge.makeOrThrow(
     ~name="envio_indexing_concurrency",
@@ -800,10 +785,7 @@ module StorageLoad = {
       )
       operations->Utils.Dict.deleteInPlace(key)
     }
-    sumTimeCounter->SafeCounter.handleFloat(
-      ~labels,
-      ~value=timerRef->Performance.secondsSince,
-    )
+    sumTimeCounter->SafeCounter.handleFloat(~labels, ~value=timerRef->Performance.secondsSince)
     counter->SafeCounter.increment(~labels)
     whereSizeCounter->SafeCounter.handleInt(~labels, ~value=whereSize)
     sizeCounter->SafeCounter.handleInt(~labels, ~value=size)