envio 3.2.0 → 3.3.0-alpha.0

package/src/Writing.res

@@ -0,0 +1,242 @@
+// The write loop: drains the processed-batch queue and staged chain metadata to
+// storage off the processing path. A peer to ChainFetching/BatchProcessing/
+// Rollback, but data-driven — `schedule` kicks a single write fiber whenever
+// there's pending work, and processing waits on it through awaitCapacity/flush.
+// State mutations go through IndexerState's domain operations.
+
+// Max uncommitted entity/effect changes plus unwritten batch items before
+// processing must wait for the cycle to free capacity.
+let keepLatestChangesLimit = Env.inMemoryObjectsTarget
+
+let getChangesCount = (state: IndexerState.t) => {
+  let total = ref(0.)
+  state
+  ->IndexerState.allEntities
+  ->Array.forEach(entityConfig => {
+    total := total.contents +. (state->InMemoryStore.getInMemTable(~entityConfig)).changesCount
+  })
+  state
+  ->IndexerState.effects
+  ->Utils.Dict.forEach(inMemTable => {
+    total := total.contents +. inMemTable.changesCount
+  })
+  state
+  ->IndexerState.processedBatches
+  ->Array.forEach(batch => {
+    total := total.contents +. batch.totalBatchSize->Int.toFloat
+  })
+  total.contents
+}
+
+let waitForCommit = (state: IndexerState.t): promise<unit> =>
+  Promise.make((resolve, _) => {
+    state->IndexerState.addCommitWaiter(resolve)
+  })
+
+// Captures the cache:true outputs to persist. The dict is left intact — entries
+// stay warm and are reclaimed later by dropCommittedEffects once committed.
+let snapshotEffects = (state: IndexerState.t, ~cache): array<Persistence.updatedEffectCache> => {
+  let acc = []
+  state
+  ->IndexerState.effects
+  ->Utils.Dict.forEach(inMemTable => {
+    let {idsToStore, dict, effect, invalidationsCount} = inMemTable
+    switch idsToStore {
+    | [] => ()
+    | ids =>
+      let items = ids->Array.filterMap((id): option<Internal.effectCacheItem> =>
+        switch dict->Dict.getUnsafe(id) {
+        | Set({entity: output}) => Some({id, output})
+        | Delete(_) => None
+        }
+      )
+      let effectName = effect.name
+      let effectCacheRecord = switch cache->Utils.Dict.dangerouslyGetNonOption(effectName) {
+      | Some(c) => c
+      | None =>
+        let c: Persistence.effectCacheRecord = {effectName, count: 0}
+        cache->Dict.set(effectName, c)
+        c
+      }
+      let shouldInitialize = effectCacheRecord.count === 0
+      effectCacheRecord.count = effectCacheRecord.count + items->Array.length - invalidationsCount
+      Prometheus.EffectCacheCount.set(~count=effectCacheRecord.count, ~effectName)
+      acc->Array.push(({effect, items, shouldInitialize}: Persistence.updatedEffectCache))->ignore
+    }
+    inMemTable.idsToStore = []
+    inMemTable.invalidationsCount = 0
+  })
+  acc
+}
+
+let runOneWrite = async (state: IndexerState.t) => {
+  let persistence = state->IndexerState.persistence
+  let config = state->IndexerState.config
+  let cache = switch persistence.storageStatus {
+  | Unknown
+  | Initializing(_) =>
+    JsError.throwWithMessage(`Failed to access the indexer storage. The Persistence layer is not initialized.`)
+  | Ready({cache}) => cache
+  }
+
+  // Snapshot before the await: the batch write reads this later, in-transaction.
+  let chainMetaData = state->IndexerState.takeChainMetaSnapshot
+
+  switch state->IndexerState.processedBatches {
+  | [] =>
+    // Metadata only: a cheap upsert, still serialized by the single write loop.
+    switch chainMetaData {
+    | Some(chainsData) =>
+      await persistence.storage.setChainMeta(chainsData)->Utils.Promise.ignoreValue
+    | None => ()
+    }
+  | _ =>
+    let committedCheckpointId = state->IndexerState.committedCheckpointId
+    let batch = state->IndexerState.drainBatchRun
+    // The run's last checkpoint; entity changes above it stay queued for the next write.
+    let upToCheckpointId = switch batch.checkpointIds->Utils.Array.last {
+    | Some(checkpointId) => checkpointId
+    | None => committedCheckpointId
+    }
+
+    let rollback = state->IndexerState.takeRollback
+
+    let updatedEntities = persistence.allEntities->Array.filterMap(entityConfig => {
+      let table = state->InMemoryStore.getInMemTable(~entityConfig)
+      let changes =
+        table->InMemoryTable.Entity.snapshotChanges(~committedCheckpointId, ~upToCheckpointId)
+      if changes->Utils.Array.isEmpty {
+        None
+      } else {
+        Some(({entityConfig, changes}: Persistence.updatedEntity))
+      }
+    })
+    let updatedEffectsCache = snapshotEffects(state, ~cache)
+
+    await persistence.storage.writeBatch(
+      ~batch,
+      ~rollback,
+      ~isInReorgThreshold=batch.isInReorgThreshold,
+      ~config,
+      ~allEntities=persistence.allEntities,
+      ~updatedEntities,
+      ~updatedEffectsCache,
+      ~chainMetaData,
+    )
+
+    state->IndexerState.markCommitted(~upToCheckpointId)
+
+    switch rollback {
+    | Some({progressBlockNumberByChainId}) if RollbackCommit.callbacks->Utils.Array.notEmpty =>
+      await RollbackCommit.fire(~progressBlockNumberByChainId)
+    | _ => ()
+    }
+  }
+}
+
+let hasPendingWrite = (state: IndexerState.t) =>
+  state->IndexerState.processedBatches->Utils.Array.notEmpty || state->IndexerState.chainMetaDirty
+
+let runWriteLoop = async (state: IndexerState.t) => {
+  while state->hasPendingWrite && !(state->IndexerState.hasFailedWrite) {
+    try {
+      await runOneWrite(state)
+      state->IndexerState.wakeCommitWaiters
+    } catch {
+    | exn => state->IndexerState.recordWriteFailure(exn)
+    }
+  }
+  state->IndexerState.endWriteFiber
+  state->IndexerState.wakeCommitWaiters
+}
+
+// Kicks the single write fiber if there's pending work and one isn't running.
+let schedule = (state: IndexerState.t) =>
+  if (
+    state->IndexerState.writeFiber->Option.isNone &&
+    !(state->IndexerState.hasFailedWrite) &&
+    state->hasPendingWrite
+  ) {
+    state->IndexerState.beginWriteFiber(runWriteLoop(state))
+  }
+
+// Stages chain metadata and throttles a metadata-only write when no batches flow.
+let setChainMeta = (state: IndexerState.t, chainsData: dict<InternalTable.Chains.metaFields>) => {
+  state->IndexerState.stageChainMeta(chainsData)
+  if state->IndexerState.chainMetaDirty {
+    state
+    ->IndexerState.chainMetaThrottler
+    ->Throttler.schedule(() => {
+      state->schedule
+      Promise.resolve()
+    })
+  }
+}
+
+// Queues a processed batch and kicks the cycle. Returns immediately; the write
+// happens off the processing path.
+let commitBatch = (state: IndexerState.t, ~batch: Batch.t) => {
+  state->IndexerState.queueProcessedBatch(~batch)
+  state->schedule
+}
+
+// Drops committed entity and effect entries across all tables. With
+// keepLoadedFromDb, entries seeded from a db read are spared.
+let dropCommitted = (state: IndexerState.t, ~keepLoadedFromDb) => {
+  let committedCheckpointId = state->IndexerState.committedCheckpointId
+  state
+  ->IndexerState.allEntities
+  ->Array.forEach(entityConfig =>
+    state
+    ->InMemoryStore.getInMemTable(~entityConfig)
+    ->InMemoryTable.Entity.dropCommittedChanges(~committedCheckpointId, ~keepLoadedFromDb)
+  )
+  state
+  ->IndexerState.effects
+  ->Utils.Dict.forEach(inMemTable =>
+    inMemTable->InMemoryStore.dropCommittedEffects(~committedCheckpointId, ~keepLoadedFromDb)
+  )
+}
+
+// Blocks until the store holds fewer than keepLatestChangesLimit changes,
+// freeing committed changes first and awaiting commits as a last resort.
+let rec awaitCapacity = async (state: IndexerState.t) => {
+  // After a failed write nothing will free capacity, so bail instead of waiting
+  // on a commit that won't come (the error already went to onError).
+  if !(state->IndexerState.hasFailedWrite) && state->getChangesCount >= keepLatestChangesLimit {
+    // Drop committed writes first, sparing db-loaded entries (explicitly
+    // requested, so likelier to be read again).
+    state->dropCommitted(~keepLoadedFromDb=true)
+
+    // Still over: drop the db-loaded entries too.
+    if state->getChangesCount >= keepLatestChangesLimit {
+      state->dropCommitted(~keepLoadedFromDb=false)
+    }
+
+    // Still over: what's left is uncommitted. Only wait if a queued batch can
+    // free it; otherwise (e.g. a large rollback diff with no batch) waiting
+    // would deadlock, so let processing proceed.
+    if (
+      state->getChangesCount >= keepLatestChangesLimit &&
+        state->IndexerState.processedBatches->Utils.Array.notEmpty
+    ) {
+      state->schedule
+      await state->waitForCommit
+      await state->awaitCapacity
+    }
+  }
+}
+
+// Awaits until everything processed is persisted. On a failed write we stop
+// draining (onError already surfaced it) rather than throw.
+let rec flush = async (state: IndexerState.t) => {
+  if !(state->IndexerState.hasFailedWrite) {
+    state->schedule
+    switch state->IndexerState.writeFiber {
+    | Some(fiber) =>
+      await fiber
+      await state->flush
+    | None => ()
+    }
+  }
+}

package/src/Writing.res.mjs

@@ -0,0 +1,215 @@
+// Generated by ReScript, PLEASE EDIT WITH CARE
+
+import * as Env from "./Env.res.mjs";
+import * as Utils from "./Utils.res.mjs";
+import * as Throttler from "./Throttler.res.mjs";
+import * as Prometheus from "./Prometheus.res.mjs";
+import * as IndexerState from "./IndexerState.res.mjs";
+import * as Stdlib_Array from "@rescript/runtime/lib/es6/Stdlib_Array.js";
+import * as InMemoryStore from "./InMemoryStore.res.mjs";
+import * as InMemoryTable from "./InMemoryTable.res.mjs";
+import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js";
+import * as RollbackCommit from "./RollbackCommit.res.mjs";
+import * as Stdlib_JsError from "@rescript/runtime/lib/es6/Stdlib_JsError.js";
+import * as Primitive_exceptions from "@rescript/runtime/lib/es6/Primitive_exceptions.js";
+
+function getChangesCount(state) {
+  let total = {
+    contents: 0
+  };
+  IndexerState.allEntities(state).forEach(entityConfig => {
+    total.contents = total.contents + InMemoryStore.getInMemTable(state, entityConfig).changesCount;
+  });
+  Utils.Dict.forEach(IndexerState.effects(state), inMemTable => {
+    total.contents = total.contents + inMemTable.changesCount;
+  });
+  IndexerState.processedBatches(state).forEach(batch => {
+    total.contents = total.contents + batch.totalBatchSize;
+  });
+  return total.contents;
+}
+
+function waitForCommit(state) {
+  return new Promise((resolve, param) => IndexerState.addCommitWaiter(state, resolve));
+}
+
+function snapshotEffects(state, cache) {
+  let acc = [];
+  Utils.Dict.forEach(IndexerState.effects(state), inMemTable => {
+    let idsToStore = inMemTable.idsToStore;
+    let invalidationsCount = inMemTable.invalidationsCount;
+    let dict = inMemTable.dict;
+    let effect = inMemTable.effect;
+    if (idsToStore.length !== 0) {
+      let items = Stdlib_Array.filterMap(idsToStore, id => {
+        let match = dict[id];
+        if (match.type === "SET") {
+          return {
+            id: id,
+            output: match.entity
+          };
+        }
+      });
+      let effectName = effect.name;
+      let c = cache[effectName];
+      let effectCacheRecord;
+      if (c !== undefined) {
+        effectCacheRecord = c;
+      } else {
+        let c$1 = {
+          effectName: effectName,
+          count: 0
+        };
+        cache[effectName] = c$1;
+        effectCacheRecord = c$1;
+      }
+      let shouldInitialize = effectCacheRecord.count === 0;
+      effectCacheRecord.count = (effectCacheRecord.count + items.length | 0) - invalidationsCount | 0;
+      Prometheus.EffectCacheCount.set(effectCacheRecord.count, effectName);
+      acc.push({
+        effect: effect,
+        items: items,
+        shouldInitialize: shouldInitialize
+      });
+    }
+    inMemTable.idsToStore = [];
+    inMemTable.invalidationsCount = 0;
+  });
+  return acc;
+}
+
+async function runOneWrite(state) {
+  let persistence = IndexerState.persistence(state);
+  let config = IndexerState.config(state);
+  let match = persistence.storageStatus;
+  let cache;
+  cache = typeof match !== "object" || match.TAG === "Initializing" ? Stdlib_JsError.throwWithMessage(`Failed to access the indexer storage. The Persistence layer is not initialized.`) : match._0.cache;
+  let chainMetaData = IndexerState.takeChainMetaSnapshot(state);
+  let match$1 = IndexerState.processedBatches(state);
+  if (match$1.length === 0) {
+    if (chainMetaData !== undefined) {
+      return await persistence.storage.setChainMeta(chainMetaData);
+    } else {
+      return;
+    }
+  }
+  let committedCheckpointId = IndexerState.committedCheckpointId(state);
+  let batch = IndexerState.drainBatchRun(state);
+  let checkpointId = Utils.$$Array.last(batch.checkpointIds);
+  let upToCheckpointId = checkpointId !== undefined ? checkpointId : committedCheckpointId;
+  let rollback = IndexerState.takeRollback(state);
+  let updatedEntities = Stdlib_Array.filterMap(persistence.allEntities, entityConfig => {
+    let table = InMemoryStore.getInMemTable(state, entityConfig);
+    let changes = InMemoryTable.Entity.snapshotChanges(table, committedCheckpointId, upToCheckpointId);
+    if (Utils.$$Array.isEmpty(changes)) {
+      return;
+    } else {
+      return {
+        entityConfig: entityConfig,
+        changes: changes
+      };
+    }
+  });
+  let updatedEffectsCache = snapshotEffects(state, cache);
+  await persistence.storage.writeBatch(batch, rollback, batch.isInReorgThreshold, config, persistence.allEntities, updatedEffectsCache, updatedEntities, chainMetaData);
+  IndexerState.markCommitted(state, upToCheckpointId);
+  if (rollback !== undefined && Utils.$$Array.notEmpty(RollbackCommit.callbacks)) {
+    return await RollbackCommit.fire(rollback.progressBlockNumberByChainId);
+  }
+}
+
+function hasPendingWrite(state) {
+  if (Utils.$$Array.notEmpty(IndexerState.processedBatches(state))) {
+    return true;
+  } else {
+    return IndexerState.chainMetaDirty(state);
+  }
+}
+
+async function runWriteLoop(state) {
+  while (hasPendingWrite(state) && !IndexerState.hasFailedWrite(state)) {
+    try {
+      await runOneWrite(state);
+      IndexerState.wakeCommitWaiters(state);
+    } catch (raw_exn) {
+      let exn = Primitive_exceptions.internalToException(raw_exn);
+      IndexerState.recordWriteFailure(state, exn);
+    }
+  };
+  IndexerState.endWriteFiber(state);
+  return IndexerState.wakeCommitWaiters(state);
+}
+
+function schedule(state) {
+  if (Stdlib_Option.isNone(IndexerState.writeFiber(state)) && !IndexerState.hasFailedWrite(state) && hasPendingWrite(state)) {
+    return IndexerState.beginWriteFiber(state, runWriteLoop(state));
+  }
+}
+
+function setChainMeta(state, chainsData) {
+  IndexerState.stageChainMeta(state, chainsData);
+  if (IndexerState.chainMetaDirty(state)) {
+    return Throttler.schedule(IndexerState.chainMetaThrottler(state), () => {
+      schedule(state);
+      return Promise.resolve();
+    });
+  }
+}
+
+function commitBatch(state, batch) {
+  IndexerState.queueProcessedBatch(state, batch);
+  schedule(state);
+}
+
+function dropCommitted(state, keepLoadedFromDb) {
+  let committedCheckpointId = IndexerState.committedCheckpointId(state);
+  IndexerState.allEntities(state).forEach(entityConfig => InMemoryTable.Entity.dropCommittedChanges(InMemoryStore.getInMemTable(state, entityConfig), committedCheckpointId, keepLoadedFromDb));
+  Utils.Dict.forEach(IndexerState.effects(state), inMemTable => InMemoryStore.dropCommittedEffects(inMemTable, committedCheckpointId, keepLoadedFromDb));
+}
+
+async function awaitCapacity(state) {
+  if (!IndexerState.hasFailedWrite(state) && getChangesCount(state) >= Env.inMemoryObjectsTarget) {
+    dropCommitted(state, true);
+    if (getChangesCount(state) >= Env.inMemoryObjectsTarget) {
+      dropCommitted(state, false);
+    }
+    if (getChangesCount(state) >= Env.inMemoryObjectsTarget && Utils.$$Array.notEmpty(IndexerState.processedBatches(state))) {
+      schedule(state);
+      await new Promise((resolve, param) => IndexerState.addCommitWaiter(state, resolve));
+      return await awaitCapacity(state);
+    } else {
+      return;
+    }
+  }
+}
+
+async function flush(state) {
+  if (IndexerState.hasFailedWrite(state)) {
+    return;
+  }
+  schedule(state);
+  let fiber = IndexerState.writeFiber(state);
+  if (fiber !== undefined) {
+    await fiber;
+    return await flush(state);
+  }
+}
+
+let keepLatestChangesLimit = Env.inMemoryObjectsTarget;
+
+export {
+  keepLatestChangesLimit,
+  getChangesCount,
+  waitForCommit,
+  snapshotEffects,
+  runOneWrite,
+  hasPendingWrite,
+  runWriteLoop,
+  schedule,
+  setChainMeta,
+  commitBatch,
+  dropCommitted,
+  awaitCapacity,
+  flush,
+}
+/* Env Not a pure module */

package/src/db/InternalTable.res

@@ -571,33 +571,20 @@ GROUP BY "${(#chain_id: field :> string)}";`
 }
 
 module RawEvents = {
-  type t = {
-    @as("chain_id") chainId: int,
-    @as("event_id") eventId: bigint,
-    @as("event_name") eventName: string,
-    @as("contract_name") contractName: string,
-    @as("block_number") blockNumber: int,
-    @as("log_index") logIndex: int,
-    @as("src_address") srcAddress: Address.t,
-    @as("block_hash") blockHash: string,
-    @as("block_timestamp") blockTimestamp: int,
-    @as("block_fields") blockFields: JSON.t,
-    @as("transaction_fields") transactionFields: JSON.t,
-    params: JSON.t,
-  }
-
-  let schema = S.schema(s => {
-    chainId: s.matches(S.int),
-    eventId: s.matches(S.bigint),
-    eventName: s.matches(S.string),
-    contractName: s.matches(S.string),
-    blockNumber: s.matches(S.int),
-    logIndex: s.matches(S.int),
-    srcAddress: s.matches(Address.schema),
-    blockHash: s.matches(S.string),
-    blockTimestamp: s.matches(S.int),
-    blockFields: s.matches(S.json(~validate=false)),
-    transactionFields: s.matches(S.json(~validate=false)),
+  type t = Internal.rawEvent
+
+  let schema = S.schema((s): t => {
+    chain_id: s.matches(S.int),
+    event_id: s.matches(S.bigint),
+    event_name: s.matches(S.string),
+    contract_name: s.matches(S.string),
+    block_number: s.matches(S.int),
+    log_index: s.matches(S.int),
+    src_address: s.matches(Address.schema),
+    block_hash: s.matches(S.string),
+    block_timestamp: s.matches(S.int),
+    block_fields: s.matches(S.json(~validate=false)),
+    transaction_fields: s.matches(S.json(~validate=false)),
     params: s.matches(S.json(~validate=false)),
   })
 

package/src/sources/Evm.res

@@ -1,10 +1,25 @@
+// EVM's concrete item payload. Erased to `Internal.eventPayload` on the item
+// and recovered here via `toPayload`.
+type payload = {
+  contractName: string,
+  eventName: string,
+  params: Internal.eventParams,
+  chainId: int,
+  srcAddress: Address.t,
+  logIndex: int,
+  transaction: Internal.eventTransaction,
+  block: Internal.eventBlock,
+}
+external fromPayload: payload => Internal.eventPayload = "%identity"
+external toPayload: Internal.eventPayload => payload = "%identity"
+
 let cleanUpRawEventFieldsInPlace: JSON.t => unit = %raw(`fields => {
     delete fields.hash
     delete fields.number
     delete fields.timestamp
   }`)
 
-let ecosystem: Ecosystem.t = {
+let make = (~logger: Pino.t): Ecosystem.t => {
   name: Evm,
   blockFields: [
     "number",
@@ -90,4 +105,28 @@ let ecosystem: Ecosystem.t = {
   onEventBlockFilterSchema: S.object(s =>
     s.field("block", S.option(S.object(s2 => s2.field("number", S.unknown))))
   ),
+  logger,
+  toEvent: eventItem => eventItem.payload->Internal.payloadToEvent,
+  toEventLogger: eventItem =>
+    Logging.createChildFrom(
+      ~logger,
+      ~params={
+        "contract": eventItem.eventConfig.contractName,
+        "event": eventItem.eventConfig.name,
+        "chainId": eventItem.chain->ChainMap.Chain.toChainId,
+        "block": eventItem.blockNumber,
+        "logIndex": eventItem.logIndex,
+        "address": (eventItem.payload->toPayload).srcAddress,
+      },
+    ),
+  toRawEvent: eventItem => {
+    let payload = eventItem.payload->toPayload
+    eventItem->RawEvent.make(
+      ~block=payload.block,
+      ~transaction=payload.transaction,
+      ~params=payload.params,
+      ~srcAddress=payload.srcAddress,
+      ~cleanUpRawEventFieldsInPlace,
+    )
+  },
 }