envio 3.7.0 → 3.9.0

package/src/Metrics.res

@@ -12,6 +12,8 @@ type chainMetrics = {
   startBlock: int,
   endBlock: option<int>,
   numAddresses: int,
+  // Per-contract registration counts, in the chain's contract-id order.
+  addressesByContract: array<(string, int)>,
   isReady: bool,
   // Raw source height, unlike knownHeight which is clamped to endBlock.
   sourceBlockNumber: int,
@@ -694,11 +696,23 @@ let renderMetrics = (b: builder, metrics: t) => {
   )
   b->series(
     ~name="envio_indexing_addresses",
-    ~help="The number of addresses indexed on chain. Includes both static and dynamic addresses.",
+    ~help="The number of address registrations on chain, static and dynamic. An address shared by N contracts counts N times.",
     ~kind="gauge",
     ~entries=chains,
     ~value=m => m.numAddresses->Int.toFloat,
   )
+  b->series(
+    ~name="envio_indexing_contract_addresses",
+    ~help="The number of address registrations per contract on chain, static and dynamic. An address shared by N contracts counts N times.",
+    ~kind="gauge",
+    ~entries=metrics.chains->Array.flatMap(m =>
+      m.addressesByContract->Array.map(((contract, count)) => (
+        `{chainId="${m.chainId->ChainId.toString}",contract="${contract->escapeLabelValue}"}`,
+        count,
+      ))
+    ),
+    ~value=count => count->Int.toFloat,
+  )
 }
 
 let contentType = "text/plain; version=0.0.4; charset=utf-8"

package/src/Metrics.res.mjs

@@ -229,7 +229,11 @@ function renderMetrics(b, metrics) {
   series(b, "envio_storage_load_size", "Cumulative number of records loaded from storage during the indexing process.", "counter", storageLoads, s => s.size);
   series(b, "envio_storage_write_seconds", "Cumulative time spent writing batch data to storage.", "counter", storageWrites, s => s.seconds);
   series(b, "envio_storage_write_total", "Cumulative number of successful storage write operations during the indexing process.", "counter", storageWrites, s => s.count);
-  series(b, "envio_indexing_addresses", "The number of addresses indexed on chain. Includes both static and dynamic addresses.", "gauge", chains, m => m.numAddresses);
+  series(b, "envio_indexing_addresses", "The number of address registrations on chain, static and dynamic. An address shared by N contracts counts N times.", "gauge", chains, m => m.numAddresses);
+  series(b, "envio_indexing_contract_addresses", "The number of address registrations per contract on chain, static and dynamic. An address shared by N contracts counts N times.", "gauge", metrics.chains.flatMap(m => m.addressesByContract.map(param => [
+    `{chainId="` + ChainId.toString(m.chainId) + `",contract="` + escapeLabelValue(param[0]) + `"}`,
+    param[1]
+  ])), count => count);
 }
 
 function collect(metrics) {

package/src/Persistence.res

@@ -27,22 +27,25 @@ type initialChainState = {
   numEventsProcessed: float,
   firstEventBlockNumber: option<int>,
   timestampCaughtUpToHeadOrEndblock: option<Date.t>,
-  indexingAddresses: array<Internal.indexingAddress>,
+  // Every address the chain indexes, columnar — config-declared and dynamically
+  // registered alike. The chain's address store seeds straight from it.
+  addressRows: AddressRows.seedRows,
   sourceBlockNumber: int,
 }
 
 type initialState = {
   cleanRun: bool,
+  // On a resume this is what the database holds, not what the config would
+  // derive — the ids must never reshuffle under stored rows.
+  contractMapping: ContractMapping.t,
+  // Public config snapshot, restored with the address rows. None when
+  // envio_info or envio_contracts is missing.
+  envioInfo: option<JSON.t>,
   cache: dict<effectCacheRecord>,
   chains: array<initialChainState>,
   checkpointId: Internal.checkpointId,
   // Needed to keep reorg detection logic between restarts
   reorgCheckpoints: array<Internal.reorgCheckpoint>,
-  // Public config snapshot read from envio_info, used by `Persistence.init`
-  // to compat-check a resume against the running config. None when the
-  // schema pre-dates envio_info or the row is missing — `init` treats that
-  // as a version mismatch.
-  envioInfo: option<JSON.t>,
 }
 
 // Carries the already-resolved cache address (`table`) rather than an effect +
@@ -58,6 +61,9 @@ type updatedEffectCache = {
 type rollback = {
   targetCheckpointId: Internal.checkpointId,
   diffCheckpointId: Internal.checkpointId,
+  // The address registrations the rollback dropped, as the chains' address
+  // stores resolved them. Deleted by primary key in the same transaction.
+  rolledBackAddresses: array<AddressRows.key>,
   // Last valid block per chain affected by the rollback. Read by
   // `RollbackCommit.fire` once the diff is durably written.
   progressBlockNumberByChainId: dict<int>,
@@ -92,6 +98,7 @@ type storage = {
     ~chainConfigs: array<Config.chain>=?,
     ~entities: array<Internal.entityConfig>=?,
     ~enums: array<Table.enumConfig<Table.enum>>=?,
+    ~contractMapping: ContractMapping.t,
     ~envioInfo: JSON.t,
   ) => promise<initialState>,
   resumeInitialState: unit => promise<initialState>,
@@ -147,11 +154,13 @@ type storage = {
       "new_progress_block_number": int,
     }>,
   >,
-  // Get rollback data for entity
+  // Rollback data for an entity, as decoded entities rather than storage rows:
+  // only the storage knows how it encoded them, so each one decodes its own
+  // before handing them back.
   getRollbackData: (
     ~entityConfig: Internal.entityConfig,
     ~rollbackTargetCheckpointId: Internal.checkpointId,
-  ) => promise<(array<rollbackRemoval>, array<unknown>)>,
+  ) => promise<(array<rollbackRemoval>, array<Internal.entity>)>,
   // Write batch to storage
   writeBatch: (
     ~batch: Batch.t,
@@ -161,6 +170,8 @@ type storage = {
     ~allEntities: array<Internal.entityConfig>,
     ~updatedEffectsCache: array<updatedEffectCache>,
     ~updatedEntities: array<updatedEntity>,
+    // Addresses this batch registered, with the checkpoint that covers them.
+    ~registeredAddresses: array<AddressRows.staged>,
     // Chain metadata stale since the last write, persisted in the same
     // transaction so it never races the batch write.
     ~chainMetaData: option<dict<InternalTable.Chains.metaFields>>,
@@ -194,7 +205,7 @@ let make = (
   ~allEnums,
   ~storage,
 ) => {
-  let allEntities = userEntities->Array.concat([InternalTable.EnvioAddresses.entityConfig])
+  let allEntities = userEntities
   let allEnums =
     allEnums->Array.concat([EntityHistory.RowAction.config->Table.fromGenericEnumConfig])
   {
@@ -207,7 +218,15 @@ let make = (
 }
 
 let init = {
-  async (persistence, ~chainConfigs, ~envioInfo, ~resetCommand, ~runCommand, ~reset=false) => {
+  async (
+    persistence,
+    ~chainConfigs,
+    ~contractMapping,
+    ~envioInfo,
+    ~resetCommand,
+    ~runCommand,
+    ~reset=false,
+  ) => {
     try {
       let shouldRun = switch persistence.storageStatus {
       | Unknown => true
@@ -229,6 +248,7 @@ let init = {
             ~entities=persistence.allEntities,
             ~enums=persistence.allEnums,
             ~chainConfigs,
+            ~contractMapping,
             ~envioInfo,
           )
           Logging.info(`The indexer storage is ready. Starting indexing!`)
@@ -243,17 +263,10 @@ let init = {
         ) {
           Logging.info(`Found existing indexer storage. Resuming indexing state...`)
           let initialState = await persistence.storage.resumeInitialState()
-          // Compat-check the running config against what was stored on the
-          // last successful initialize. None means the schema pre-dates
-          // envio_info (or the row was wiped out-of-band) and we can't
-          // compare — treat it as a version mismatch.
           let changedPaths = switch initialState.envioInfo {
-          | None => ["envio info is missing — storage initialized by an older envio"]
+          | None => ["storage was initialized by an older envio version"]
           | Some(stored) => Config.diffPaths(~stored, ~current=envioInfo)
           }
-          // `storage.clickhouse` is serialized as a plain bool by the
-          // public config (see Rust `StorageConfig`), so probe for
-          // `Boolean(true)`, not an object.
           let hasClickhouse = switch envioInfo {
           | Object(d) =>
             switch d->Dict.get("storage") {
@@ -267,6 +280,9 @@ let init = {
           | _ => false
           }
           Config.throwIfIncompatible(changedPaths, ~resetCommand, ~runCommand, ~hasClickhouse)
+          if !(initialState.contractMapping->ContractMapping.isEqual(contractMapping)) {
+            Config.throwIfIncompatible(["contracts"], ~resetCommand, ~runCommand, ~hasClickhouse)
+          }
           persistence.storageStatus = Ready(initialState)
           let progress = Dict.make()
           initialState.chains->Array.forEach(c => {

package/src/Persistence.res.mjs

@@ -5,23 +5,23 @@ import * as Logging from "./Logging.res.mjs";
 import * as EntityHistory from "./db/EntityHistory.res.mjs";
 import * as ErrorHandling from "./ErrorHandling.res.mjs";
 import * as Stdlib_JsError from "@rescript/runtime/lib/es6/Stdlib_JsError.js";
+import * as ContractMapping from "./ContractMapping.res.mjs";
 import * as Primitive_exceptions from "@rescript/runtime/lib/es6/Primitive_exceptions.js";
 
 let StorageError = /* @__PURE__ */Primitive_exceptions.create("Persistence.StorageError");
 
 function make(userEntities, allEnums, storage) {
-  let allEntities = userEntities.concat([Config.EnvioAddresses.entityConfig]);
   let allEnums$1 = allEnums.concat([EntityHistory.RowAction.config]);
   return {
     userEntities: userEntities,
-    allEntities: allEntities,
+    allEntities: userEntities,
     allEnums: allEnums$1,
     storageStatus: "Unknown",
     storage: storage
   };
 }
 
-async function init(persistence, chainConfigs, envioInfo, resetCommand, runCommand, resetOpt) {
+async function init(persistence, chainConfigs, contractMapping, envioInfo, resetCommand, runCommand, resetOpt) {
   let reset = resetOpt !== undefined ? resetOpt : false;
   try {
     let promise = persistence.storageStatus;
@@ -49,7 +49,7 @@ async function init(persistence, chainConfigs, envioInfo, resetCommand, runComma
     };
     if (reset || !await persistence.storage.isInitialized()) {
       Logging.info(`Initializing the indexer storage...`);
-      let initialState = await persistence.storage.initialize(chainConfigs, persistence.allEntities, persistence.allEnums, envioInfo);
+      let initialState = await persistence.storage.initialize(chainConfigs, persistence.allEntities, persistence.allEnums, contractMapping, envioInfo);
       Logging.info(`The indexer storage is ready. Starting indexing!`);
       persistence.storageStatus = {
         TAG: "Ready",
@@ -63,7 +63,7 @@ async function init(persistence, chainConfigs, envioInfo, resetCommand, runComma
         Logging.info(`Found existing indexer storage. Resuming indexing state...`);
         let initialState$1 = await persistence.storage.resumeInitialState();
         let stored = initialState$1.envioInfo;
-        let changedPaths = stored !== undefined ? Config.diffPaths(stored, envioInfo) : ["envio info is missing — storage initialized by an older envio"];
+        let changedPaths = stored !== undefined ? Config.diffPaths(stored, envioInfo) : ["storage was initialized by an older envio version"];
         let hasClickhouse;
         if (typeof envioInfo === "object" && envioInfo !== null && !Array.isArray(envioInfo)) {
           let match$1 = envioInfo["storage"];
@@ -77,6 +77,9 @@ async function init(persistence, chainConfigs, envioInfo, resetCommand, runComma
           hasClickhouse = false;
         }
         Config.throwIfIncompatible(changedPaths, resetCommand, runCommand, hasClickhouse);
+        if (!ContractMapping.isEqual(initialState$1.contractMapping, contractMapping)) {
+          Config.throwIfIncompatible(["contracts"], resetCommand, runCommand, hasClickhouse);
+        }
         persistence.storageStatus = {
           TAG: "Ready",
           _0: initialState$1