envio 3.2.1 → 3.3.0-alpha.0

package/src/Writing.res.mjs

@@ -0,0 +1,215 @@
+// Generated by ReScript, PLEASE EDIT WITH CARE
+
+import * as Env from "./Env.res.mjs";
+import * as Utils from "./Utils.res.mjs";
+import * as Throttler from "./Throttler.res.mjs";
+import * as Prometheus from "./Prometheus.res.mjs";
+import * as IndexerState from "./IndexerState.res.mjs";
+import * as Stdlib_Array from "@rescript/runtime/lib/es6/Stdlib_Array.js";
+import * as InMemoryStore from "./InMemoryStore.res.mjs";
+import * as InMemoryTable from "./InMemoryTable.res.mjs";
+import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js";
+import * as RollbackCommit from "./RollbackCommit.res.mjs";
+import * as Stdlib_JsError from "@rescript/runtime/lib/es6/Stdlib_JsError.js";
+import * as Primitive_exceptions from "@rescript/runtime/lib/es6/Primitive_exceptions.js";
+
+function getChangesCount(state) {
+  let total = {
+    contents: 0
+  };
+  IndexerState.allEntities(state).forEach(entityConfig => {
+    total.contents = total.contents + InMemoryStore.getInMemTable(state, entityConfig).changesCount;
+  });
+  Utils.Dict.forEach(IndexerState.effects(state), inMemTable => {
+    total.contents = total.contents + inMemTable.changesCount;
+  });
+  IndexerState.processedBatches(state).forEach(batch => {
+    total.contents = total.contents + batch.totalBatchSize;
+  });
+  return total.contents;
+}
+
+function waitForCommit(state) {
+  return new Promise((resolve, param) => IndexerState.addCommitWaiter(state, resolve));
+}
+
+function snapshotEffects(state, cache) {
+  let acc = [];
+  Utils.Dict.forEach(IndexerState.effects(state), inMemTable => {
+    let idsToStore = inMemTable.idsToStore;
+    let invalidationsCount = inMemTable.invalidationsCount;
+    let dict = inMemTable.dict;
+    let effect = inMemTable.effect;
+    if (idsToStore.length !== 0) {
+      let items = Stdlib_Array.filterMap(idsToStore, id => {
+        let match = dict[id];
+        if (match.type === "SET") {
+          return {
+            id: id,
+            output: match.entity
+          };
+        }
+      });
+      let effectName = effect.name;
+      let c = cache[effectName];
+      let effectCacheRecord;
+      if (c !== undefined) {
+        effectCacheRecord = c;
+      } else {
+        let c$1 = {
+          effectName: effectName,
+          count: 0
+        };
+        cache[effectName] = c$1;
+        effectCacheRecord = c$1;
+      }
+      let shouldInitialize = effectCacheRecord.count === 0;
+      effectCacheRecord.count = (effectCacheRecord.count + items.length | 0) - invalidationsCount | 0;
+      Prometheus.EffectCacheCount.set(effectCacheRecord.count, effectName);
+      acc.push({
+        effect: effect,
+        items: items,
+        shouldInitialize: shouldInitialize
+      });
+    }
+    inMemTable.idsToStore = [];
+    inMemTable.invalidationsCount = 0;
+  });
+  return acc;
+}
+
+async function runOneWrite(state) {
+  let persistence = IndexerState.persistence(state);
+  let config = IndexerState.config(state);
+  let match = persistence.storageStatus;
+  let cache;
+  cache = typeof match !== "object" || match.TAG === "Initializing" ? Stdlib_JsError.throwWithMessage(`Failed to access the indexer storage. The Persistence layer is not initialized.`) : match._0.cache;
+  let chainMetaData = IndexerState.takeChainMetaSnapshot(state);
+  let match$1 = IndexerState.processedBatches(state);
+  if (match$1.length === 0) {
+    if (chainMetaData !== undefined) {
+      return await persistence.storage.setChainMeta(chainMetaData);
+    } else {
+      return;
+    }
+  }
+  let committedCheckpointId = IndexerState.committedCheckpointId(state);
+  let batch = IndexerState.drainBatchRun(state);
+  let checkpointId = Utils.$$Array.last(batch.checkpointIds);
+  let upToCheckpointId = checkpointId !== undefined ? checkpointId : committedCheckpointId;
+  let rollback = IndexerState.takeRollback(state);
+  let updatedEntities = Stdlib_Array.filterMap(persistence.allEntities, entityConfig => {
+    let table = InMemoryStore.getInMemTable(state, entityConfig);
+    let changes = InMemoryTable.Entity.snapshotChanges(table, committedCheckpointId, upToCheckpointId);
+    if (Utils.$$Array.isEmpty(changes)) {
+      return;
+    } else {
+      return {
+        entityConfig: entityConfig,
+        changes: changes
+      };
+    }
+  });
+  let updatedEffectsCache = snapshotEffects(state, cache);
+  await persistence.storage.writeBatch(batch, rollback, batch.isInReorgThreshold, config, persistence.allEntities, updatedEffectsCache, updatedEntities, chainMetaData);
+  IndexerState.markCommitted(state, upToCheckpointId);
+  if (rollback !== undefined && Utils.$$Array.notEmpty(RollbackCommit.callbacks)) {
+    return await RollbackCommit.fire(rollback.progressBlockNumberByChainId);
+  }
+}
+
+function hasPendingWrite(state) {
+  if (Utils.$$Array.notEmpty(IndexerState.processedBatches(state))) {
+    return true;
+  } else {
+    return IndexerState.chainMetaDirty(state);
+  }
+}
+
+async function runWriteLoop(state) {
+  while (hasPendingWrite(state) && !IndexerState.hasFailedWrite(state)) {
+    try {
+      await runOneWrite(state);
+      IndexerState.wakeCommitWaiters(state);
+    } catch (raw_exn) {
+      let exn = Primitive_exceptions.internalToException(raw_exn);
+      IndexerState.recordWriteFailure(state, exn);
+    }
+  };
+  IndexerState.endWriteFiber(state);
+  return IndexerState.wakeCommitWaiters(state);
+}
+
+function schedule(state) {
+  if (Stdlib_Option.isNone(IndexerState.writeFiber(state)) && !IndexerState.hasFailedWrite(state) && hasPendingWrite(state)) {
+    return IndexerState.beginWriteFiber(state, runWriteLoop(state));
+  }
+}
+
+function setChainMeta(state, chainsData) {
+  IndexerState.stageChainMeta(state, chainsData);
+  if (IndexerState.chainMetaDirty(state)) {
+    return Throttler.schedule(IndexerState.chainMetaThrottler(state), () => {
+      schedule(state);
+      return Promise.resolve();
+    });
+  }
+}
+
+function commitBatch(state, batch) {
+  IndexerState.queueProcessedBatch(state, batch);
+  schedule(state);
+}
+
+function dropCommitted(state, keepLoadedFromDb) {
+  let committedCheckpointId = IndexerState.committedCheckpointId(state);
+  IndexerState.allEntities(state).forEach(entityConfig => InMemoryTable.Entity.dropCommittedChanges(InMemoryStore.getInMemTable(state, entityConfig), committedCheckpointId, keepLoadedFromDb));
+  Utils.Dict.forEach(IndexerState.effects(state), inMemTable => InMemoryStore.dropCommittedEffects(inMemTable, committedCheckpointId, keepLoadedFromDb));
+}
+
+async function awaitCapacity(state) {
+  if (!IndexerState.hasFailedWrite(state) && getChangesCount(state) >= Env.inMemoryObjectsTarget) {
+    dropCommitted(state, true);
+    if (getChangesCount(state) >= Env.inMemoryObjectsTarget) {
+      dropCommitted(state, false);
+    }
+    if (getChangesCount(state) >= Env.inMemoryObjectsTarget && Utils.$$Array.notEmpty(IndexerState.processedBatches(state))) {
+      schedule(state);
+      await new Promise((resolve, param) => IndexerState.addCommitWaiter(state, resolve));
+      return await awaitCapacity(state);
+    } else {
+      return;
+    }
+  }
+}
+
+async function flush(state) {
+  if (IndexerState.hasFailedWrite(state)) {
+    return;
+  }
+  schedule(state);
+  let fiber = IndexerState.writeFiber(state);
+  if (fiber !== undefined) {
+    await fiber;
+    return await flush(state);
+  }
+}
+
+let keepLatestChangesLimit = Env.inMemoryObjectsTarget;
+
+export {
+  keepLatestChangesLimit,
+  getChangesCount,
+  waitForCommit,
+  snapshotEffects,
+  runOneWrite,
+  hasPendingWrite,
+  runWriteLoop,
+  schedule,
+  setChainMeta,
+  commitBatch,
+  dropCommitted,
+  awaitCapacity,
+  flush,
+}
+/* Env Not a pure module */

package/src/db/InternalTable.res

@@ -571,33 +571,20 @@ GROUP BY "${(#chain_id: field :> string)}";`
 }
 
 module RawEvents = {
-  type t = {
-    @as("chain_id") chainId: int,
-    @as("event_id") eventId: bigint,
-    @as("event_name") eventName: string,
-    @as("contract_name") contractName: string,
-    @as("block_number") blockNumber: int,
-    @as("log_index") logIndex: int,
-    @as("src_address") srcAddress: Address.t,
-    @as("block_hash") blockHash: string,
-    @as("block_timestamp") blockTimestamp: int,
-    @as("block_fields") blockFields: JSON.t,
-    @as("transaction_fields") transactionFields: JSON.t,
-    params: JSON.t,
-  }
-
-  let schema = S.schema(s => {
-    chainId: s.matches(S.int),
-    eventId: s.matches(S.bigint),
-    eventName: s.matches(S.string),
-    contractName: s.matches(S.string),
-    blockNumber: s.matches(S.int),
-    logIndex: s.matches(S.int),
-    srcAddress: s.matches(Address.schema),
-    blockHash: s.matches(S.string),
-    blockTimestamp: s.matches(S.int),
-    blockFields: s.matches(S.json(~validate=false)),
-    transactionFields: s.matches(S.json(~validate=false)),
+  type t = Internal.rawEvent
+
+  let schema = S.schema((s): t => {
+    chain_id: s.matches(S.int),
+    event_id: s.matches(S.bigint),
+    event_name: s.matches(S.string),
+    contract_name: s.matches(S.string),
+    block_number: s.matches(S.int),
+    log_index: s.matches(S.int),
+    src_address: s.matches(Address.schema),
+    block_hash: s.matches(S.string),
+    block_timestamp: s.matches(S.int),
+    block_fields: s.matches(S.json(~validate=false)),
+    transaction_fields: s.matches(S.json(~validate=false)),
     params: s.matches(S.json(~validate=false)),
   })
 

package/src/sources/Evm.res

@@ -1,10 +1,25 @@
+// EVM's concrete item payload. Erased to `Internal.eventPayload` on the item
+// and recovered here via `toPayload`.
+type payload = {
+  contractName: string,
+  eventName: string,
+  params: Internal.eventParams,
+  chainId: int,
+  srcAddress: Address.t,
+  logIndex: int,
+  transaction: Internal.eventTransaction,
+  block: Internal.eventBlock,
+}
+external fromPayload: payload => Internal.eventPayload = "%identity"
+external toPayload: Internal.eventPayload => payload = "%identity"
+
 let cleanUpRawEventFieldsInPlace: JSON.t => unit = %raw(`fields => {
     delete fields.hash
     delete fields.number
     delete fields.timestamp
   }`)
 
-let ecosystem: Ecosystem.t = {
+let make = (~logger: Pino.t): Ecosystem.t => {
   name: Evm,
   blockFields: [
     "number",
@@ -90,4 +105,28 @@ let ecosystem: Ecosystem.t = {
   onEventBlockFilterSchema: S.object(s =>
     s.field("block", S.option(S.object(s2 => s2.field("number", S.unknown))))
   ),
+  logger,
+  toEvent: eventItem => eventItem.payload->Internal.payloadToEvent,
+  toEventLogger: eventItem =>
+    Logging.createChildFrom(
+      ~logger,
+      ~params={
+        "contract": eventItem.eventConfig.contractName,
+        "event": eventItem.eventConfig.name,
+        "chainId": eventItem.chain->ChainMap.Chain.toChainId,
+        "block": eventItem.blockNumber,
+        "logIndex": eventItem.logIndex,
+        "address": (eventItem.payload->toPayload).srcAddress,
+      },
+    ),
+  toRawEvent: eventItem => {
+    let payload = eventItem.payload->toPayload
+    eventItem->RawEvent.make(
+      ~block=payload.block,
+      ~transaction=payload.transaction,
+      ~params=payload.params,
+      ~srcAddress=payload.srcAddress,
+      ~cleanUpRawEventFieldsInPlace,
+    )
+  },
 }

package/src/sources/Evm.res.mjs

@@ -1,5 +1,7 @@
 // Generated by ReScript, PLEASE EDIT WITH CARE
 
+import * as Logging from "../Logging.res.mjs";
+import * as RawEvent from "../RawEvent.res.mjs";
 import * as S$RescriptSchema from "rescript-schema/src/S.res.mjs";
 
 let cleanUpRawEventFieldsInPlace = (fields => {
@@ -8,91 +10,99 @@ let cleanUpRawEventFieldsInPlace = (fields => {
     delete fields.timestamp
   });
 
-let ecosystem_blockFields = [
-  "number",
-  "timestamp",
-  "hash",
-  "parentHash",
-  "nonce",
-  "sha3Uncles",
-  "logsBloom",
-  "transactionsRoot",
-  "stateRoot",
-  "receiptsRoot",
-  "miner",
-  "difficulty",
-  "totalDifficulty",
-  "extraData",
-  "size",
-  "gasLimit",
-  "gasUsed",
-  "uncles",
-  "baseFeePerGas",
-  "blobGasUsed",
-  "excessBlobGas",
-  "parentBeaconBlockRoot",
-  "withdrawalsRoot",
-  "l1BlockNumber",
-  "sendCount",
-  "sendRoot",
-  "mixHash"
-];
-
-let ecosystem_transactionFields = [
-  "transactionIndex",
-  "hash",
-  "from",
-  "to",
-  "gas",
-  "gasPrice",
-  "maxPriorityFeePerGas",
-  "maxFeePerGas",
-  "cumulativeGasUsed",
-  "effectiveGasPrice",
-  "gasUsed",
-  "input",
-  "nonce",
-  "value",
-  "v",
-  "r",
-  "s",
-  "contractAddress",
-  "logsBloom",
-  "root",
-  "status",
-  "yParity",
-  "chainId",
-  "maxFeePerBlobGas",
-  "blobVersionedHashes",
-  "type",
-  "l1Fee",
-  "l1GasPrice",
-  "l1GasUsed",
-  "l1FeeScalar",
-  "gasUsedForL1",
-  "accessList",
-  "authorizationList"
-];
-
-let ecosystem_onBlockFilterSchema = S$RescriptSchema.object(s => s.f("block", S$RescriptSchema.option(S$RescriptSchema.object(s2 => s2.f("number", S$RescriptSchema.unknown)))));
-
-let ecosystem_onEventBlockFilterSchema = S$RescriptSchema.object(s => s.f("block", S$RescriptSchema.option(S$RescriptSchema.object(s2 => s2.f("number", S$RescriptSchema.unknown)))));
-
-let ecosystem = {
-  name: "evm",
-  blockFields: ecosystem_blockFields,
-  transactionFields: ecosystem_transactionFields,
-  blockNumberName: "number",
-  blockTimestampName: "timestamp",
-  blockHashName: "hash",
-  cleanUpRawEventFieldsInPlace: cleanUpRawEventFieldsInPlace,
-  onBlockMethodName: "onBlock",
-  onBlockFilterSchema: ecosystem_onBlockFilterSchema,
-  onEventBlockFilterSchema: ecosystem_onEventBlockFilterSchema
-};
+function make(logger) {
+  return {
+    name: "evm",
+    blockFields: [
+      "number",
+      "timestamp",
+      "hash",
+      "parentHash",
+      "nonce",
+      "sha3Uncles",
+      "logsBloom",
+      "transactionsRoot",
+      "stateRoot",
+      "receiptsRoot",
+      "miner",
+      "difficulty",
+      "totalDifficulty",
+      "extraData",
+      "size",
+      "gasLimit",
+      "gasUsed",
+      "uncles",
+      "baseFeePerGas",
+      "blobGasUsed",
+      "excessBlobGas",
+      "parentBeaconBlockRoot",
+      "withdrawalsRoot",
+      "l1BlockNumber",
+      "sendCount",
+      "sendRoot",
+      "mixHash"
+    ],
+    transactionFields: [
+      "transactionIndex",
+      "hash",
+      "from",
+      "to",
+      "gas",
+      "gasPrice",
+      "maxPriorityFeePerGas",
+      "maxFeePerGas",
+      "cumulativeGasUsed",
+      "effectiveGasPrice",
+      "gasUsed",
+      "input",
+      "nonce",
+      "value",
+      "v",
+      "r",
+      "s",
+      "contractAddress",
+      "logsBloom",
+      "root",
+      "status",
+      "yParity",
+      "chainId",
+      "maxFeePerBlobGas",
+      "blobVersionedHashes",
+      "type",
+      "l1Fee",
+      "l1GasPrice",
+      "l1GasUsed",
+      "l1FeeScalar",
+      "gasUsedForL1",
+      "accessList",
+      "authorizationList"
+    ],
+    blockNumberName: "number",
+    blockTimestampName: "timestamp",
+    blockHashName: "hash",
+    cleanUpRawEventFieldsInPlace: cleanUpRawEventFieldsInPlace,
+    onBlockMethodName: "onBlock",
+    onBlockFilterSchema: S$RescriptSchema.object(s => s.f("block", S$RescriptSchema.option(S$RescriptSchema.object(s2 => s2.f("number", S$RescriptSchema.unknown))))),
+    onEventBlockFilterSchema: S$RescriptSchema.object(s => s.f("block", S$RescriptSchema.option(S$RescriptSchema.object(s2 => s2.f("number", S$RescriptSchema.unknown))))),
+    logger: logger,
+    toEvent: eventItem => eventItem.payload,
+    toEventLogger: eventItem => Logging.createChildFrom(logger, {
+      contract: eventItem.eventConfig.contractName,
+      event: eventItem.eventConfig.name,
+      chainId: eventItem.chain,
+      block: eventItem.blockNumber,
+      logIndex: eventItem.logIndex,
+      address: eventItem.payload.srcAddress
+    }),
+    toRawEvent: eventItem => {
+      let payload = eventItem.payload;
+      return RawEvent.make(eventItem, payload.block, payload.transaction, payload.params, payload.srcAddress, cleanUpRawEventFieldsInPlace);
+    }
+  };
+}
 
 export {
   cleanUpRawEventFieldsInPlace,
-  ecosystem,
+  make,
 }
-/* ecosystem Not a pure module */
+/* Logging Not a pure module */

package/src/sources/Fuel.res

@@ -1,10 +1,25 @@
+// Fuel's concrete item payload. Erased to `Internal.eventPayload` on the item
+// and recovered here via `toPayload`.
+type payload = {
+  contractName: string,
+  eventName: string,
+  params: Internal.eventParams,
+  chainId: int,
+  srcAddress: Address.t,
+  logIndex: int,
+  transaction: Internal.eventTransaction,
+  block: Internal.eventBlock,
+}
+external fromPayload: payload => Internal.eventPayload = "%identity"
+external toPayload: Internal.eventPayload => payload = "%identity"
+
 let cleanUpRawEventFieldsInPlace: JSON.t => unit = %raw(`fields => {
     delete fields.id
     delete fields.height
     delete fields.time
   }`)
 
-let ecosystem: Ecosystem.t = {
+let make = (~logger: Pino.t): Ecosystem.t => {
   name: Fuel,
   blockFields: ["id", "height", "time"],
   transactionFields: ["id"],
@@ -25,4 +40,28 @@ let ecosystem: Ecosystem.t = {
   onEventBlockFilterSchema: S.object(s =>
     s.field("block", S.option(S.object(s2 => s2.field("height", S.unknown))))
   ),
+  logger,
+  toEvent: eventItem => eventItem.payload->Internal.payloadToEvent,
+  toEventLogger: eventItem =>
+    Logging.createChildFrom(
+      ~logger,
+      ~params={
+        "contract": eventItem.eventConfig.contractName,
+        "event": eventItem.eventConfig.name,
+        "chainId": eventItem.chain->ChainMap.Chain.toChainId,
+        "block": eventItem.blockNumber,
+        "logIndex": eventItem.logIndex,
+        "address": (eventItem.payload->toPayload).srcAddress,
+      },
+    ),
+  toRawEvent: eventItem => {
+    let payload = eventItem.payload->toPayload
+    eventItem->RawEvent.make(
+      ~block=payload.block,
+      ~transaction=payload.transaction,
+      ~params=payload.params,
+      ~srcAddress=payload.srcAddress,
+      ~cleanUpRawEventFieldsInPlace,
+    )
+  },
 }

package/src/sources/Fuel.res.mjs

@@ -1,5 +1,7 @@
 // Generated by ReScript, PLEASE EDIT WITH CARE
 
+import * as Logging from "../Logging.res.mjs";
+import * as RawEvent from "../RawEvent.res.mjs";
 import * as S$RescriptSchema from "rescript-schema/src/S.res.mjs";
 
 let cleanUpRawEventFieldsInPlace = (fields => {
@@ -8,33 +10,41 @@ let cleanUpRawEventFieldsInPlace = (fields => {
     delete fields.time
   });
 
-let ecosystem_blockFields = [
-  "id",
-  "height",
-  "time"
-];
-
-let ecosystem_transactionFields = ["id"];
-
-let ecosystem_onBlockFilterSchema = S$RescriptSchema.object(s => s.f("block", S$RescriptSchema.option(S$RescriptSchema.object(s2 => s2.f("height", S$RescriptSchema.unknown)))));
-
-let ecosystem_onEventBlockFilterSchema = S$RescriptSchema.object(s => s.f("block", S$RescriptSchema.option(S$RescriptSchema.object(s2 => s2.f("height", S$RescriptSchema.unknown)))));
-
-let ecosystem = {
-  name: "fuel",
-  blockFields: ecosystem_blockFields,
-  transactionFields: ecosystem_transactionFields,
-  blockNumberName: "height",
-  blockTimestampName: "time",
-  blockHashName: "id",
-  cleanUpRawEventFieldsInPlace: cleanUpRawEventFieldsInPlace,
-  onBlockMethodName: "onBlock",
-  onBlockFilterSchema: ecosystem_onBlockFilterSchema,
-  onEventBlockFilterSchema: ecosystem_onEventBlockFilterSchema
-};
+function make(logger) {
+  return {
+    name: "fuel",
+    blockFields: [
+      "id",
+      "height",
+      "time"
+    ],
+    transactionFields: ["id"],
+    blockNumberName: "height",
+    blockTimestampName: "time",
+    blockHashName: "id",
+    cleanUpRawEventFieldsInPlace: cleanUpRawEventFieldsInPlace,
+    onBlockMethodName: "onBlock",
+    onBlockFilterSchema: S$RescriptSchema.object(s => s.f("block", S$RescriptSchema.option(S$RescriptSchema.object(s2 => s2.f("height", S$RescriptSchema.unknown))))),
+    onEventBlockFilterSchema: S$RescriptSchema.object(s => s.f("block", S$RescriptSchema.option(S$RescriptSchema.object(s2 => s2.f("height", S$RescriptSchema.unknown))))),
+    logger: logger,
+    toEvent: eventItem => eventItem.payload,
+    toEventLogger: eventItem => Logging.createChildFrom(logger, {
+      contract: eventItem.eventConfig.contractName,
+      event: eventItem.eventConfig.name,
+      chainId: eventItem.chain,
+      block: eventItem.blockNumber,
+      logIndex: eventItem.logIndex,
+      address: eventItem.payload.srcAddress
+    }),
+    toRawEvent: eventItem => {
+      let payload = eventItem.payload;
+      return RawEvent.make(eventItem, payload.block, payload.transaction, payload.params, payload.srcAddress, cleanUpRawEventFieldsInPlace);
+    }
+  };
+}
 
 export {
   cleanUpRawEventFieldsInPlace,
-  ecosystem,
+  make,
 }
-/* ecosystem Not a pure module */
+/* Logging Not a pure module */