envio 3.7.0 → 3.9.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "envio",
-  "version": "3.7.0",
+  "version": "3.9.0",
   "type": "module",
   "description": "A latency and sync speed optimized, developer friendly blockchain data indexer.",
   "bin": "./bin.mjs",
@@ -69,10 +69,10 @@
     "tsx": "4.21.0"
   },
   "optionalDependencies": {
-    "envio-linux-x64": "3.7.0",
-    "envio-linux-x64-musl": "3.7.0",
-    "envio-linux-arm64": "3.7.0",
-    "envio-darwin-x64": "3.7.0",
-    "envio-darwin-arm64": "3.7.0"
+    "envio-linux-x64": "3.9.0",
+    "envio-linux-x64-musl": "3.9.0",
+    "envio-linux-arm64": "3.9.0",
+    "envio-darwin-x64": "3.9.0",
+    "envio-darwin-arm64": "3.9.0"
   }
 }

package/src/AddressRows.res

@@ -0,0 +1,121 @@
+type key = {
+  chainId: ChainId.t,
+  address: NodeJs.Buffer.t,
+  contractId: int,
+}
+
+type row = {
+  chainId: ChainId.t,
+  address: NodeJs.Buffer.t,
+  contractId: int,
+  registrationBlock: int,
+}
+
+type staged = {
+  row: row,
+  checkpointId: Internal.checkpointId,
+}
+
+type seedRows = {
+  addresses: array<NodeJs.Buffer.t>,
+  contractIds: array<int>,
+  registrationBlocks: array<int>,
+}
+
+let emptySeedRows = (): seedRows => {
+  addresses: [],
+  contractIds: [],
+  registrationBlocks: [],
+}
+
+// Config addresses are registered by no event. Rollback never reaches them.
+let configRegistrationBlock = -1
+
+let keyOf = (row: row): key => {
+  chainId: row.chainId,
+  address: row.address,
+  contractId: row.contractId,
+}
+
+let packBuffers = (addresses: array<NodeJs.Buffer.t>) => (
+  NodeJs.Buffer.concat(addresses),
+  addresses->Array.map(address => address->NodeJs.Buffer.length),
+)
+
+let packKeys = (rows: array<row>) => packBuffers(rows->Array.map(row => row.address))
+
+let seedRowsOf = (rows: array<row>): seedRows => {
+  let addresses = []
+  let contractIds = []
+  let registrationBlocks = []
+  rows->Array.forEach(row => {
+    addresses->Array.push(row.address)->ignore
+    contractIds->Array.push(row.contractId)->ignore
+    registrationBlocks->Array.push(row.registrationBlock)->ignore
+  })
+  {addresses, contractIds, registrationBlocks}
+}
+
+// Rows arrive clustered per chain. Normalizing a chain id is a schema parse.
+let group = (rows: array<row>): dict<seedRows> => {
+  let rowsByChain: dict<array<row>> = Dict.make()
+  let runChainId = ref(None)
+  let runKey = ref("")
+  rows->Array.forEach(row => {
+    if runChainId.contents !== Some(row.chainId) {
+      runChainId := Some(row.chainId)
+      runKey := row.chainId->ChainId.normalizeOrThrow->ChainId.toString
+    }
+    rowsByChain->Utils.Dict.push(runKey.contents, row)
+  })
+  rowsByChain->Utils.Dict.mapValues(seedRowsOf)
+}
+
+module Table = {
+  type t = dict<row>
+
+  let dictKey = (key: key) =>
+    `${key.chainId->ChainId.toString}|${key.contractId->Int.toString}|${key.address->NodeJs.Buffer.toBase64}`
+
+  let make = (): t => Dict.make()
+
+  let clear = (table: t) => table->Utils.Dict.clearInPlace
+
+  let insert = (table: t, row: row) => {
+    let key = row->keyOf->dictKey
+    switch table->Utils.Dict.dangerouslyGetNonOption(key) {
+    | Some(_) => ()
+    | None => table->Dict.set(key, row)
+    }
+  }
+
+  let insertMany = (table: t, rows: array<row>) => rows->Array.forEach(row => table->insert(row))
+
+  let delete = (table: t, key: key) => table->Utils.Dict.deleteInPlace(key->dictKey)
+
+  let rows = (table: t): array<row> => table->Dict.valuesToArray
+
+  let groupByChain = (table: t) => table->rows->group
+}
+
+let render = (rows: array<row>, ~ecosystem: string, ~shouldChecksum: bool) => {
+  let (bytes, lengths) = rows->packKeys
+  Core.getAddon().renderAddresses(~ecosystem, ~shouldChecksum, ~bytes, ~lengths)
+}
+
+let renderOfContract = (
+  rows: seedRows,
+  ~ecosystem: string,
+  ~shouldChecksum: bool,
+  ~contractId: int,
+) => {
+  let (bytes, lengths) = packBuffers(rows.addresses)
+  Core.getAddon().renderContractAddresses(
+    ~ecosystem,
+    ~shouldChecksum,
+    ~bytes,
+    ~lengths,
+    ~contractIds=rows.contractIds,
+    ~contractId,
+  )
+}

package/src/AddressRows.res.mjs

@@ -0,0 +1,143 @@
+// Generated by ReScript, PLEASE EDIT WITH CARE
+
+import * as Core from "./Core.res.mjs";
+import * as Utils from "./Utils.res.mjs";
+import * as ChainId from "./ChainId.res.mjs";
+import * as Primitive_option from "@rescript/runtime/lib/es6/Primitive_option.js";
+
+function emptySeedRows() {
+  return {
+    addresses: [],
+    contractIds: [],
+    registrationBlocks: []
+  };
+}
+
+function keyOf(row) {
+  return {
+    chainId: row.chainId,
+    address: row.address,
+    contractId: row.contractId
+  };
+}
+
+function packBuffers(addresses) {
+  return [
+    Buffer.concat(addresses),
+    addresses.map(address => address.length)
+  ];
+}
+
+function packKeys(rows) {
+  return packBuffers(rows.map(row => row.address));
+}
+
+function seedRowsOf(rows) {
+  let addresses = [];
+  let contractIds = [];
+  let registrationBlocks = [];
+  rows.forEach(row => {
+    addresses.push(row.address);
+    contractIds.push(row.contractId);
+    registrationBlocks.push(row.registrationBlock);
+  });
+  return {
+    addresses: addresses,
+    contractIds: contractIds,
+    registrationBlocks: registrationBlocks
+  };
+}
+
+function group(rows) {
+  let rowsByChain = {};
+  let runChainId = {
+    contents: undefined
+  };
+  let runKey = {
+    contents: ""
+  };
+  rows.forEach(row => {
+    if (runChainId.contents !== Primitive_option.some(row.chainId)) {
+      runChainId.contents = Primitive_option.some(row.chainId);
+      runKey.contents = ChainId.toString(ChainId.normalizeOrThrow(row.chainId));
+    }
+    Utils.Dict.push(rowsByChain, runKey.contents, row);
+  });
+  return Utils.Dict.mapValues(rowsByChain, seedRowsOf);
+}
+
+function dictKey(key) {
+  return ChainId.toString(key.chainId) + `|` + key.contractId.toString() + `|` + key.address.toString("base64");
+}
+
+function make() {
+  return {};
+}
+
+function clear(table) {
+  Utils.Dict.clearInPlace(table);
+}
+
+function insert(table, row) {
+  let key = dictKey(keyOf(row));
+  let match = table[key];
+  if (match !== undefined) {
+    return;
+  } else {
+    table[key] = row;
+    return;
+  }
+}
+
+function insertMany(table, rows) {
+  rows.forEach(row => insert(table, row));
+}
+
+function $$delete(table, key) {
+  Utils.Dict.deleteInPlace(table, dictKey(key));
+}
+
+function rows(table) {
+  return Object.values(table);
+}
+
+function groupByChain(table) {
+  return group(Object.values(table));
+}
+
+let Table = {
+  dictKey: dictKey,
+  make: make,
+  clear: clear,
+  insert: insert,
+  insertMany: insertMany,
+  $$delete: $$delete,
+  rows: rows,
+  groupByChain: groupByChain
+};
+
+function render(rows, ecosystem, shouldChecksum) {
+  let match = packKeys(rows);
+  return Core.getAddon().renderAddresses(ecosystem, shouldChecksum, match[0], match[1]);
+}
+
+function renderOfContract(rows, ecosystem, shouldChecksum, contractId) {
+  let match = packBuffers(rows.addresses);
+  return Core.getAddon().renderContractAddresses(ecosystem, shouldChecksum, match[0], match[1], rows.contractIds, contractId);
+}
+
+let configRegistrationBlock = -1;
+
+export {
+  emptySeedRows,
+  configRegistrationBlock,
+  keyOf,
+  packBuffers,
+  packKeys,
+  seedRowsOf,
+  group,
+  Table,
+  render,
+  renderOfContract,
+}
+/* Core Not a pure module */

package/src/Batch.res

@@ -43,6 +43,7 @@ type t = {
   checkpointBlockNumbers: array<int>,
   checkpointBlockHashes: array<Null.t<string>>,
   checkpointEventsProcessed: array<int>,
+  registeredAddresses: array<AddressRows.staged>,
 }
 
 let getProgressedChainsById = {
@@ -326,6 +327,7 @@ let prepareBatch = (
     checkpointBlockNumbers,
     checkpointBlockHashes,
     checkpointEventsProcessed,
+    registeredAddresses: [],
   }
 }
 

package/src/Batch.res.mjs

@@ -185,7 +185,8 @@ function prepareBatch(checkpointIdBeforeBatch, chainsBeforeBatch, batchSizeTarge
     checkpointChainIds: checkpointChainIds,
     checkpointBlockNumbers: checkpointBlockNumbers,
     checkpointBlockHashes: checkpointBlockHashes,
-    checkpointEventsProcessed: checkpointEventsProcessed
+    checkpointEventsProcessed: checkpointEventsProcessed,
+    registeredAddresses: []
   };
 }
 

package/src/ChainState.res

@@ -62,20 +62,48 @@ type t = {
   mutable progressLatencyMs: option<int>,
 }
 
-let configAddresses = (chainConfig: Config.chain): array<Internal.indexingAddress> => {
+// The chain's config-declared addresses as storage rows: what a clean run
+// writes into `envio_addresses`. The keys are encoded by the Rust codec, so the
+// bytes a clean run writes are exactly the bytes a resume seeds from.
+let configStorageRows = (
+  chainConfig: Config.chain,
+  ~ecosystem: Ecosystem.name,
+  ~contractMapping: ContractMapping.t,
+): array<AddressRows.row> => {
   let addresses = []
+  let contractIds = []
   chainConfig.contracts->Array.forEach(contract => {
+    let contractId = contractMapping->ContractMapping.idOfOrThrow(contract.name)
     contract.addresses->Array.forEach(address => {
-      addresses->Array.push({
-        Internal.address,
-        contractName: contract.name,
-        registrationBlock: -1,
-      })
+      addresses->Array.push(address)->ignore
+      contractIds->Array.push(contractId)->ignore
     })
   })
-  addresses
+  Core.getAddon().encodeAddresses(
+    ~ecosystem=(ecosystem :> string),
+    ~addresses,
+  )->Array.mapWithIndex((address, idx) => {
+    AddressRows.chainId: chainConfig.id,
+    address,
+    contractId: contractIds->Array.getUnsafe(idx),
+    registrationBlock: AddressRows.configRegistrationBlock,
+  })
 }
 
+// Seeds an in-memory address table with every chain's config-declared
+// addresses, the twin of what `PgStorage.initialize` writes in one insert.
+let seedConfigAddresses = (
+  table: AddressRows.Table.t,
+  ~chainConfigs: array<Config.chain>,
+  ~ecosystem: Ecosystem.name,
+  ~contractMapping: ContractMapping.t,
+) =>
+  chainConfigs->Array.forEach(chainConfig =>
+    table->AddressRows.Table.insertMany(
+      chainConfig->configStorageRows(~ecosystem, ~contractMapping),
+    )
+  )
+
 let validateOnEventRegistrations = (
   ~chainId: ChainId.t,
   registrations: array<Internal.onEventRegistration>,
@@ -142,22 +170,12 @@ let make = (
   }
 }
 
-// Every chain's contracts, not just one chain's: `context.chain.X.add` accepts
-// any config contract, whichever chain declared it, so every chain's address
-// store has to hold the whole set.
-let configContractNames = (config: Config.t) => {
-  let names = Utils.Set.make()
-  config.chainMap
-  ->ChainMap.values
-  ->Array.forEach(chain =>
-    chain.contracts->Array.forEach(contract => names->Utils.Set.add(contract.name)->ignore)
-  )
-  names->Utils.Set.toArray
-}
-
 let makeInternal = (
   ~chainConfig: Config.chain,
-  ~indexingAddresses: array<Internal.indexingAddress>,
+  ~addressRows: AddressRows.seedRows,
+  // Passed rather than read off `config`, because a resume must use the mapping
+  // storage holds: the ids stored rows carry, not the ids this config derives.
+  ~contractMapping: ContractMapping.t,
   ~startBlock,
   ~endBlock,
   ~firstEventBlock=None,
@@ -199,16 +217,13 @@ let makeInternal = (
   let addressStore = AddressStore.make(
     ~ecosystem=config.ecosystem.name,
     ~shouldChecksum=!lowercaseAddresses,
-    ~contracts=AddressStore.contractsOf(
-      ~onEventRegistrations,
-      ~configContractNames=config->configContractNames,
-    ),
+    ~contracts=AddressStore.contractsOf(~onEventRegistrations, ~contractMapping),
   )
 
   let fetchState = FetchState.make(
     ~maxAddrInPartition=config.maxAddrInPartition,
     ~addressStore,
-    ~addresses=indexingAddresses,
+    ~addressRows,
     ~progressBlockNumber,
     ~startBlock,
     ~endBlock,
@@ -301,13 +316,15 @@ let makeInternal = (
         }),
       ]
     }
-  | Config.SimulateSourceConfig({items, endBlock}) => [
+  | Config.SimulateSourceConfig({items, endBlock, ?transactionStore, ?blockStore}) => [
       SimulateSource.make(
         ~items,
         ~endBlock,
         ~chainId,
         ~addressStore,
         ~ecosystem=config.ecosystem.name,
+        ~transactionStore,
+        ~blockStore,
       ),
     ]
   // For tests: use ready-to-use sources directly
@@ -357,7 +374,7 @@ let makeInternal = (
       ~chainReorgCheckpoints,
     ),
     ~committedProgressBlockNumber=progressBlockNumber,
-    ~perChainEntities=config.allEntities->EntityTables.perChain,
+    ~perChainEntities=config.userEntities->EntityTables.perChain,
     ~timestampCaughtUpToHeadOrEndblock,
     ~numEventsProcessed,
     ~transactionStore=TransactionStore.make(
@@ -371,33 +388,6 @@ let makeInternal = (
   )
 }
 
-let makeFromConfig = (
-  chainConfig: Config.chain,
-  ~config,
-  ~registrationsByChainId,
-  ~knownHeight,
-) => {
-  let logger = Logging.createChild(~params={"chainId": chainConfig.id})
-
-  makeInternal(
-    ~chainConfig,
-    ~config,
-    ~registrationsByChainId,
-    ~startBlock=chainConfig.startBlock,
-    ~endBlock=chainConfig.endBlock,
-    ~reorgCheckpoints=[],
-    ~maxReorgDepth=chainConfig.maxReorgDepth,
-    ~progressBlockNumber=-1,
-    ~timestampCaughtUpToHeadOrEndblock=None,
-    ~numEventsProcessed=0.,
-    ~logger,
-    ~indexingAddresses=configAddresses(chainConfig),
-    ~isInReorgThreshold=false,
-    ~isRealtime=false,
-    ~knownHeight,
-  )
-}
-
 /**
  * This function allows a chain state to be created from metadata, in particular this is useful for restarting an indexer and making sure it fetches blocks from the same place.
  */
@@ -408,6 +398,7 @@ let makeFromDbState = (
   ~isInReorgThreshold,
   ~isRealtime,
   ~config,
+  ~contractMapping,
   ~registrationsByChainId,
   ~reducedPollingInterval=?,
 ) => {
@@ -421,7 +412,8 @@ let makeFromDbState = (
       : resumedChainState.startBlock - 1
 
   makeInternal(
-    ~indexingAddresses=resumedChainState.indexingAddresses,
+    ~addressRows=resumedChainState.addressRows,
+    ~contractMapping,
     ~chainConfig,
     ~startBlock=resumedChainState.startBlock,
     ~endBlock=resumedChainState.endBlock,
@@ -505,7 +497,7 @@ let contractAddresses = (cs: t, ~contractName) =>
 
 // Hands over the dynamically registered addresses the database hasn't seen yet,
 // up to the block the caller is about to commit, each paired with the checkpoint
-// that owns its row. Destructive: the caller must persist them or take the
+// that registered it. Destructive: the caller must persist them or take the
 // indexer down. Throws with nothing consumed when a registration's block has no
 // checkpoint in the batch.
 let drainAddressesForWrite = (cs: t, ~toBlockInclusive, ~checkpointBlockNumbers) =>
@@ -852,6 +844,12 @@ let applyTransactionGroups = async (store: TransactionStore.t, g: transactionGro
       g.payloadGroups->Array.forEachWithIndex((payloads, i) => {
         let tx = txs->Array.getUnsafe(i)
         payloads->Array.forEach(payload => payload->Internal.setPayloadTransaction(tx))
+        switch tx
+        ->(Utils.magic: Internal.eventTransaction => dict<unknown>)
+        ->Dict.get("accountActivities") {
+        | Some(_) => payloads->Array.forEach(payload => Svm.attachAccountActivities(payload, tx))
+        | None => ()
+        }
       })
     } else {
       g.payloadGroups->Array.forEach(payloads =>
@@ -1019,6 +1017,9 @@ let toMetrics = (cs: t): Metrics.chainMetrics => {
   startBlock: cs.fetchState.startBlock,
   endBlock: cs.fetchState.endBlock,
   numAddresses: cs.addressStore->AddressStore.size,
+  addressesByContract: cs.addressStore
+  ->AddressStore.contractCounts
+  ->Array.map(({contractName, count}) => (contractName, count)),
   isReady: cs->isReady,
   sourceBlockNumber: cs.fetchState.knownHeight,
   progressBlockNumber: cs.committedProgressBlockNumber,
@@ -1209,13 +1210,30 @@ let markReady = (cs: t, ~readyAt) =>
 // with no diff entry still rewinds fetch state + stores to the target -
 // otherwise the stale block hash stays in the store and re-triggers the same
 // reorg.
+// Returns the address registrations the storage has to delete along with it —
+// the store is what decides which ones died, so the two halves of a rollback
+// can't disagree.
 let rollback = (
   cs: t,
   ~newProgressBlockNumber,
   ~eventsProcessedDiff,
   ~rollbackTargetBlockNumber,
   ~isReorgChain,
-) => {
+): array<AddressRows.key> => {
+  let rollbackTo = targetBlockNumber => {
+    let {fetchState, rolledBackAddresses} =
+      cs.fetchState->FetchState.rollback(
+        ~rolledBackAddressStore=cs.addressStore,
+        ~targetBlockNumber,
+      )
+    cs.fetchState = fetchState
+    rolledBackAddresses->Array.map(({address, contractId}): AddressRows.key => {
+      chainId: cs.chainConfig.id,
+      address,
+      contractId,
+    })
+  }
+
   switch newProgressBlockNumber {
   | Some(newProgressBlockNumber) =>
     let newTotalEventsProcessed =
@@ -1235,23 +1253,16 @@ let rollback = (
       )
     | None => ()
     }
-    cs.fetchState =
-      cs.fetchState->FetchState.rollback(
-        ~addressStore=cs.addressStore,
-        ~targetBlockNumber=newProgressBlockNumber,
-      )
+    let rolledBackAddresses = rollbackTo(newProgressBlockNumber)
     cs.transactionStore->TransactionStore.rollback(newProgressBlockNumber)
     cs.blockStore->BlockStore.rollback(newProgressBlockNumber)
     cs.committedProgressBlockNumber = newProgressBlockNumber
     cs.processingBlockNumber = newProgressBlockNumber
     cs.numEventsProcessed = newTotalEventsProcessed
+    rolledBackAddresses
   | None =>
     if isReorgChain {
-      cs.fetchState =
-        cs.fetchState->FetchState.rollback(
-          ~addressStore=cs.addressStore,
-          ~targetBlockNumber=rollbackTargetBlockNumber,
-        )
+      let rolledBackAddresses = rollbackTo(rollbackTargetBlockNumber)
       cs.transactionStore->TransactionStore.rollback(rollbackTargetBlockNumber)
       cs.blockStore->BlockStore.rollback(rollbackTargetBlockNumber)
       cs.committedProgressBlockNumber = Pervasives.min(
@@ -1259,6 +1270,9 @@ let rollback = (
         rollbackTargetBlockNumber,
       )
       cs.processingBlockNumber = Pervasives.min(cs.processingBlockNumber, rollbackTargetBlockNumber)
+      rolledBackAddresses
+    } else {
+      []
     }
   }
 }