envio 3.8.0 → 3.9.0

package/src/MemoryStorage.res.mjs

@@ -6,6 +6,8 @@ import * as Config from "./Config.res.mjs";
 import * as ChainId from "./ChainId.res.mjs";
 import * as EntityId from "./EntityId.res.mjs";
 import * as Internal from "./Internal.res.mjs";
+import * as ChainState from "./ChainState.res.mjs";
+import * as AddressRows from "./AddressRows.res.mjs";
 import * as Stdlib_Null from "@rescript/runtime/lib/es6/Stdlib_Null.js";
 import * as EntityFilter from "./db/EntityFilter.res.mjs";
 import * as Stdlib_Array from "@rescript/runtime/lib/es6/Stdlib_Array.js";
@@ -26,6 +28,7 @@ function make() {
     cache: {},
     effectCache: {},
     envioInfo: undefined,
+    addresses: AddressRows.Table.make(),
     isInitialized: false
   };
 }
@@ -76,58 +79,17 @@ function registerEntities(state, entities) {
   });
 }
 
-function seedIndexingAddresses(state, chainConfigs) {
-  let dict = getEntityDict(state, Config.EnvioAddresses.name);
-  chainConfigs.forEach(chainConfig => {
-    chainConfig.contracts.forEach(contract => {
-      contract.addresses.forEach(address => {
-        let entity_id = Config.EnvioAddresses.makeId(chainConfig.id, address);
-        let entity_chain_id = chainConfig.id;
-        let entity_contract_name = contract.name;
-        let entity = {
-          id: entity_id,
-          chain_id: entity_chain_id,
-          registration_block: -1,
-          registration_log_index: -1,
-          contract_name: entity_contract_name
-        };
-        dict[entity_id] = entity;
-      });
-    });
-  });
-}
-
-function toIndexingAddress(dc) {
-  return {
-    address: Config.EnvioAddresses.getAddress(dc),
-    contractName: dc.contract_name,
-    registrationBlock: dc.registration_block
-  };
+function seedConfigAddresses(state, chainConfigs, contractMapping, ecosystem) {
+  AddressRows.Table.clear(state.addresses);
+  ChainState.seedConfigAddresses(state.addresses, chainConfigs, ecosystem, contractMapping);
 }
 
-function getIndexingAddressesByChain(state) {
-  let byChain = {};
-  let dcDict = state.entities[Config.EnvioAddresses.name];
-  if (dcDict !== undefined) {
-    Object.values(dcDict).forEach(entity => {
-      let chainIdStr = ChainId.toString(entity.chain_id);
-      let arr = byChain[chainIdStr];
-      let contracts;
-      if (arr !== undefined) {
-        contracts = arr;
-      } else {
-        let arr$1 = [];
-        byChain[chainIdStr] = arr$1;
-        contracts = arr$1;
-      }
-      contracts.push(toIndexingAddress(entity));
-    });
-  }
-  return byChain;
+function addressRowsByChain(state) {
+  return AddressRows.Table.groupByChain(state.addresses);
 }
 
 function toInitialChainStates(state) {
-  let addressesByChain = getIndexingAddressesByChain(state);
+  let addressesByChain = AddressRows.Table.groupByChain(state.addresses);
   return Object.values(state.chains).map(chain => ({
     id: chain.id,
     startBlock: chain.startBlock,
@@ -137,7 +99,7 @@ function toInitialChainStates(state) {
     numEventsProcessed: chain.numEventsProcessed,
     firstEventBlockNumber: chain.firstEventBlockNumber,
     timestampCaughtUpToHeadOrEndblock: chain.timestampCaughtUpToHeadOrEndblock,
-    indexingAddresses: Stdlib_Option.getOr(addressesByChain[ChainId.toString(chain.id)], []),
+    addressRows: Stdlib_Option.getOr(addressesByChain[ChainId.toString(chain.id)], AddressRows.emptySeedRows()),
     sourceBlockNumber: chain.sourceBlockNumber
   }));
 }
@@ -174,14 +136,15 @@ function reorgCheckpoints(state) {
   });
 }
 
-function toInitialState(state, cleanRun) {
+function toInitialState(state, cleanRun, contractMapping) {
   return {
     cleanRun: cleanRun,
+    contractMapping: contractMapping,
+    envioInfo: state.envioInfo,
     cache: state.cache,
     chains: toInitialChainStates(state),
     checkpointId: committedCheckpointId(state),
-    reorgCheckpoints: reorgCheckpoints(state),
-    envioInfo: state.envioInfo
+    reorgCheckpoints: reorgCheckpoints(state)
   };
 }
 
@@ -227,18 +190,20 @@ function backfillHistory(state, entityConfig, scope, entityId, rows) {
   }
 }
 
-function applyRollback(state, targetCheckpointId) {
+function applyRollback(state, targetCheckpointId, rolledBackAddresses) {
   state.checkpoints = state.checkpoints.filter(cp => cp.id <= targetCheckpointId);
+  rolledBackAddresses.forEach(key => AddressRows.Table.$$delete(state.addresses, key));
   Object.entries(state.history).forEach(param => {
     state.history[param[0]] = param[1].filter(row => row.checkpointId <= targetCheckpointId);
   });
 }
 
-function writeBatch(state, batch, rollback, isInReorgThreshold, config, updatedEntities, updatedEffectsCache, chainMetaData) {
+function writeBatch(state, batch, rollback, isInReorgThreshold, config, updatedEntities, registeredAddresses, updatedEffectsCache, chainMetaData) {
   let shouldSaveHistory = Config.shouldSaveHistory(config, isInReorgThreshold);
   if (rollback !== undefined) {
-    applyRollback(state, rollback.targetCheckpointId);
+    applyRollback(state, rollback.targetCheckpointId, rollback.rolledBackAddresses);
   }
+  registeredAddresses.forEach(param => AddressRows.Table.insert(state.addresses, param.row));
   let diffCheckpointId = Stdlib_Option.map(rollback, param => param.diffCheckpointId);
   let isDiff = change => {
     if (diffCheckpointId !== undefined) {
@@ -431,12 +396,12 @@ function toStorage(state, config) {
   return {
     name: "memory",
     isInitialized: async () => state.isInitialized,
-    initialize: async (chainConfigsOpt, entitiesOpt, $staropt$star, envioInfo) => {
+    initialize: async (chainConfigsOpt, entitiesOpt, $staropt$star, contractMapping, envioInfo) => {
       let chainConfigs = chainConfigsOpt !== undefined ? chainConfigsOpt : [];
       let entities = entitiesOpt !== undefined ? entitiesOpt : [];
       $staropt$star !== undefined;
       registerEntities(state, entities);
-      seedIndexingAddresses(state, chainConfigs);
+      seedConfigAddresses(state, chainConfigs, contractMapping, config.ecosystem.name);
       chainConfigs.forEach(chainConfig => {
         state.chains[ChainId.toString(chainConfig.id)] = {
           id: chainConfig.id,
@@ -453,9 +418,9 @@ function toStorage(state, config) {
       });
       state.envioInfo = envioInfo;
       state.isInitialized = true;
-      return toInitialState(state, true);
+      return toInitialState(state, true, contractMapping);
     },
-    resumeInitialState: async () => toInitialState(state, false),
+    resumeInitialState: async () => toInitialState(state, false, config.contractMapping),
     loadOrThrow: async (filter, table) => handleLoad(state, table.tableName, filter),
     ensureQueryIndexes: async (param, param$1) => {},
     ensureSchemaIndexes: async param => {},
@@ -470,15 +435,14 @@ function toStorage(state, config) {
     },
     dumpEffectCache: async () => {},
     reset: async () => {
-      let clear = dict => {
-        Object.keys(dict).forEach(key => Utils.Dict.deleteInPlace(dict, key));
-      };
+      let clear = Utils.Dict.clearInPlace;
       clear(state.entities);
       clear(state.history);
       clear(state.chains);
       clear(state.effectCache);
       clear(state.cache);
       state.checkpoints = [];
+      AddressRows.Table.clear(state.addresses);
       state.envioInfo = undefined;
       state.isInitialized = false;
     },
@@ -539,7 +503,7 @@ function toStorage(state, config) {
     }),
     getRollbackProgressDiff: async rollbackTargetCheckpointId => getRollbackProgressDiff(state, rollbackTargetCheckpointId),
     getRollbackData: async (entityConfig, rollbackTargetCheckpointId) => getRollbackData(state, entityConfig, rollbackTargetCheckpointId),
-    writeBatch: async (batch, rollback, isInReorgThreshold, param, param$1, updatedEffectsCache, updatedEntities, chainMetaData, param$2) => writeBatch(state, batch, rollback, isInReorgThreshold, config, updatedEntities, updatedEffectsCache, chainMetaData),
+    writeBatch: async (batch, rollback, isInReorgThreshold, param, param$1, updatedEffectsCache, updatedEntities, registeredAddresses, chainMetaData, param$2) => writeBatch(state, batch, rollback, isInReorgThreshold, config, updatedEntities, registeredAddresses, updatedEffectsCache, chainMetaData),
     close: async () => {}
   };
 }
@@ -601,9 +565,8 @@ export {
   getEntityDict,
   getHistory,
   registerEntities,
-  seedIndexingAddresses,
-  toIndexingAddress,
-  getIndexingAddressesByChain,
+  seedConfigAddresses,
+  addressRowsByChain,
   toInitialChainStates,
   committedCheckpointId,
   reorgCheckpoints,

package/src/Metrics.res

@@ -12,6 +12,8 @@ type chainMetrics = {
   startBlock: int,
   endBlock: option<int>,
   numAddresses: int,
+  // Per-contract registration counts, in the chain's contract-id order.
+  addressesByContract: array<(string, int)>,
   isReady: bool,
   // Raw source height, unlike knownHeight which is clamped to endBlock.
   sourceBlockNumber: int,
@@ -694,11 +696,23 @@ let renderMetrics = (b: builder, metrics: t) => {
   )
   b->series(
     ~name="envio_indexing_addresses",
-    ~help="The number of addresses indexed on chain. Includes both static and dynamic addresses.",
+    ~help="The number of address registrations on chain, static and dynamic. An address shared by N contracts counts N times.",
     ~kind="gauge",
     ~entries=chains,
     ~value=m => m.numAddresses->Int.toFloat,
   )
+  b->series(
+    ~name="envio_indexing_contract_addresses",
+    ~help="The number of address registrations per contract on chain, static and dynamic. An address shared by N contracts counts N times.",
+    ~kind="gauge",
+    ~entries=metrics.chains->Array.flatMap(m =>
+      m.addressesByContract->Array.map(((contract, count)) => (
+        `{chainId="${m.chainId->ChainId.toString}",contract="${contract->escapeLabelValue}"}`,
+        count,
+      ))
+    ),
+    ~value=count => count->Int.toFloat,
+  )
 }
 
 let contentType = "text/plain; version=0.0.4; charset=utf-8"

package/src/Metrics.res.mjs

@@ -229,7 +229,11 @@ function renderMetrics(b, metrics) {
   series(b, "envio_storage_load_size", "Cumulative number of records loaded from storage during the indexing process.", "counter", storageLoads, s => s.size);
   series(b, "envio_storage_write_seconds", "Cumulative time spent writing batch data to storage.", "counter", storageWrites, s => s.seconds);
   series(b, "envio_storage_write_total", "Cumulative number of successful storage write operations during the indexing process.", "counter", storageWrites, s => s.count);
-  series(b, "envio_indexing_addresses", "The number of addresses indexed on chain. Includes both static and dynamic addresses.", "gauge", chains, m => m.numAddresses);
+  series(b, "envio_indexing_addresses", "The number of address registrations on chain, static and dynamic. An address shared by N contracts counts N times.", "gauge", chains, m => m.numAddresses);
+  series(b, "envio_indexing_contract_addresses", "The number of address registrations per contract on chain, static and dynamic. An address shared by N contracts counts N times.", "gauge", metrics.chains.flatMap(m => m.addressesByContract.map(param => [
+    `{chainId="` + ChainId.toString(m.chainId) + `",contract="` + escapeLabelValue(param[0]) + `"}`,
+    param[1]
+  ])), count => count);
 }
 
 function collect(metrics) {

package/src/Persistence.res

@@ -27,22 +27,25 @@ type initialChainState = {
   numEventsProcessed: float,
   firstEventBlockNumber: option<int>,
   timestampCaughtUpToHeadOrEndblock: option<Date.t>,
-  indexingAddresses: array<Internal.indexingAddress>,
+  // Every address the chain indexes, columnar — config-declared and dynamically
+  // registered alike. The chain's address store seeds straight from it.
+  addressRows: AddressRows.seedRows,
   sourceBlockNumber: int,
 }
 
 type initialState = {
   cleanRun: bool,
+  // On a resume this is what the database holds, not what the config would
+  // derive — the ids must never reshuffle under stored rows.
+  contractMapping: ContractMapping.t,
+  // Public config snapshot, restored with the address rows. None when
+  // envio_info or envio_contracts is missing.
+  envioInfo: option<JSON.t>,
   cache: dict<effectCacheRecord>,
   chains: array<initialChainState>,
   checkpointId: Internal.checkpointId,
   // Needed to keep reorg detection logic between restarts
   reorgCheckpoints: array<Internal.reorgCheckpoint>,
-  // Public config snapshot read from envio_info, used by `Persistence.init`
-  // to compat-check a resume against the running config. None when the
-  // schema pre-dates envio_info or the row is missing — `init` treats that
-  // as a version mismatch.
-  envioInfo: option<JSON.t>,
 }
 
 // Carries the already-resolved cache address (`table`) rather than an effect +
@@ -58,6 +61,9 @@ type updatedEffectCache = {
 type rollback = {
   targetCheckpointId: Internal.checkpointId,
   diffCheckpointId: Internal.checkpointId,
+  // The address registrations the rollback dropped, as the chains' address
+  // stores resolved them. Deleted by primary key in the same transaction.
+  rolledBackAddresses: array<AddressRows.key>,
   // Last valid block per chain affected by the rollback. Read by
   // `RollbackCommit.fire` once the diff is durably written.
   progressBlockNumberByChainId: dict<int>,
@@ -92,6 +98,7 @@ type storage = {
     ~chainConfigs: array<Config.chain>=?,
     ~entities: array<Internal.entityConfig>=?,
     ~enums: array<Table.enumConfig<Table.enum>>=?,
+    ~contractMapping: ContractMapping.t,
     ~envioInfo: JSON.t,
   ) => promise<initialState>,
   resumeInitialState: unit => promise<initialState>,
@@ -163,6 +170,8 @@ type storage = {
     ~allEntities: array<Internal.entityConfig>,
     ~updatedEffectsCache: array<updatedEffectCache>,
     ~updatedEntities: array<updatedEntity>,
+    // Addresses this batch registered, with the checkpoint that covers them.
+    ~registeredAddresses: array<AddressRows.staged>,
     // Chain metadata stale since the last write, persisted in the same
     // transaction so it never races the batch write.
     ~chainMetaData: option<dict<InternalTable.Chains.metaFields>>,
@@ -196,7 +205,7 @@ let make = (
   ~allEnums,
   ~storage,
 ) => {
-  let allEntities = userEntities->Array.concat([InternalTable.EnvioAddresses.entityConfig])
+  let allEntities = userEntities
   let allEnums =
     allEnums->Array.concat([EntityHistory.RowAction.config->Table.fromGenericEnumConfig])
   {
@@ -209,7 +218,15 @@ let make = (
 }
 
 let init = {
-  async (persistence, ~chainConfigs, ~envioInfo, ~resetCommand, ~runCommand, ~reset=false) => {
+  async (
+    persistence,
+    ~chainConfigs,
+    ~contractMapping,
+    ~envioInfo,
+    ~resetCommand,
+    ~runCommand,
+    ~reset=false,
+  ) => {
     try {
       let shouldRun = switch persistence.storageStatus {
       | Unknown => true
@@ -231,6 +248,7 @@ let init = {
             ~entities=persistence.allEntities,
             ~enums=persistence.allEnums,
             ~chainConfigs,
+            ~contractMapping,
             ~envioInfo,
           )
           Logging.info(`The indexer storage is ready. Starting indexing!`)
@@ -245,17 +263,10 @@ let init = {
         ) {
           Logging.info(`Found existing indexer storage. Resuming indexing state...`)
           let initialState = await persistence.storage.resumeInitialState()
-          // Compat-check the running config against what was stored on the
-          // last successful initialize. None means the schema pre-dates
-          // envio_info (or the row was wiped out-of-band) and we can't
-          // compare — treat it as a version mismatch.
           let changedPaths = switch initialState.envioInfo {
-          | None => ["envio info is missing — storage initialized by an older envio"]
+          | None => ["storage was initialized by an older envio version"]
           | Some(stored) => Config.diffPaths(~stored, ~current=envioInfo)
           }
-          // `storage.clickhouse` is serialized as a plain bool by the
-          // public config (see Rust `StorageConfig`), so probe for
-          // `Boolean(true)`, not an object.
           let hasClickhouse = switch envioInfo {
           | Object(d) =>
             switch d->Dict.get("storage") {
@@ -269,6 +280,9 @@ let init = {
           | _ => false
           }
           Config.throwIfIncompatible(changedPaths, ~resetCommand, ~runCommand, ~hasClickhouse)
+          if !(initialState.contractMapping->ContractMapping.isEqual(contractMapping)) {
+            Config.throwIfIncompatible(["contracts"], ~resetCommand, ~runCommand, ~hasClickhouse)
+          }
           persistence.storageStatus = Ready(initialState)
           let progress = Dict.make()
           initialState.chains->Array.forEach(c => {

package/src/Persistence.res.mjs

@@ -5,23 +5,23 @@ import * as Logging from "./Logging.res.mjs";
 import * as EntityHistory from "./db/EntityHistory.res.mjs";
 import * as ErrorHandling from "./ErrorHandling.res.mjs";
 import * as Stdlib_JsError from "@rescript/runtime/lib/es6/Stdlib_JsError.js";
+import * as ContractMapping from "./ContractMapping.res.mjs";
 import * as Primitive_exceptions from "@rescript/runtime/lib/es6/Primitive_exceptions.js";
 
 let StorageError = /* @__PURE__ */Primitive_exceptions.create("Persistence.StorageError");
 
 function make(userEntities, allEnums, storage) {
-  let allEntities = userEntities.concat([Config.EnvioAddresses.entityConfig]);
   let allEnums$1 = allEnums.concat([EntityHistory.RowAction.config]);
   return {
     userEntities: userEntities,
-    allEntities: allEntities,
+    allEntities: userEntities,
     allEnums: allEnums$1,
     storageStatus: "Unknown",
     storage: storage
   };
 }
 
-async function init(persistence, chainConfigs, envioInfo, resetCommand, runCommand, resetOpt) {
+async function init(persistence, chainConfigs, contractMapping, envioInfo, resetCommand, runCommand, resetOpt) {
   let reset = resetOpt !== undefined ? resetOpt : false;
   try {
     let promise = persistence.storageStatus;
@@ -49,7 +49,7 @@ async function init(persistence, chainConfigs, envioInfo, resetCommand, runComma
     };
     if (reset || !await persistence.storage.isInitialized()) {
       Logging.info(`Initializing the indexer storage...`);
-      let initialState = await persistence.storage.initialize(chainConfigs, persistence.allEntities, persistence.allEnums, envioInfo);
+      let initialState = await persistence.storage.initialize(chainConfigs, persistence.allEntities, persistence.allEnums, contractMapping, envioInfo);
       Logging.info(`The indexer storage is ready. Starting indexing!`);
       persistence.storageStatus = {
         TAG: "Ready",
@@ -63,7 +63,7 @@ async function init(persistence, chainConfigs, envioInfo, resetCommand, runComma
         Logging.info(`Found existing indexer storage. Resuming indexing state...`);
         let initialState$1 = await persistence.storage.resumeInitialState();
         let stored = initialState$1.envioInfo;
-        let changedPaths = stored !== undefined ? Config.diffPaths(stored, envioInfo) : ["envio info is missing — storage initialized by an older envio"];
+        let changedPaths = stored !== undefined ? Config.diffPaths(stored, envioInfo) : ["storage was initialized by an older envio version"];
         let hasClickhouse;
         if (typeof envioInfo === "object" && envioInfo !== null && !Array.isArray(envioInfo)) {
           let match$1 = envioInfo["storage"];
@@ -77,6 +77,9 @@ async function init(persistence, chainConfigs, envioInfo, resetCommand, runComma
           hasClickhouse = false;
         }
         Config.throwIfIncompatible(changedPaths, resetCommand, runCommand, hasClickhouse);
+        if (!ContractMapping.isEqual(initialState$1.contractMapping, contractMapping)) {
+          Config.throwIfIncompatible(["contracts"], resetCommand, runCommand, hasClickhouse);
+        }
         persistence.storageStatus = {
           TAG: "Ready",
           _0: initialState$1