envio 3.2.1 → 3.3.0-alpha.1

package/src/Prometheus.res.mjs

@@ -400,39 +400,28 @@ let IndexingAddresses = {
   set: set$4
 };
 
-let gauge$5 = makeOrThrow$1("envio_indexing_max_concurrency", "The maximum number of concurrent queries to the chain data-source.", chainIdLabelsSchema);
+let gauge$5 = makeOrThrow$1("envio_indexing_concurrency", "The number of executing concurrent queries to the chain data-source.", chainIdLabelsSchema);
 
-function set$5(maxConcurrency, chainId) {
-  handleInt$1(gauge$5, chainId, maxConcurrency);
+function set$5(concurrency, chainId) {
+  handleInt$1(gauge$5, chainId, concurrency);
 }
 
-let IndexingMaxConcurrency = {
+let IndexingConcurrency = {
   gauge: gauge$5,
   set: set$5
 };
 
-let gauge$6 = makeOrThrow$1("envio_indexing_concurrency", "The number of executing concurrent queries to the chain data-source.", chainIdLabelsSchema);
+let gauge$6 = makeOrThrow$1("envio_indexing_partitions", "The number of partitions used to split fetching logic by addresses and block ranges.", chainIdLabelsSchema);
 
-function set$6(concurrency, chainId) {
-  handleInt$1(gauge$6, chainId, concurrency);
+function set$6(partitionsCount, chainId) {
+  handleInt$1(gauge$6, chainId, partitionsCount);
 }
 
-let IndexingConcurrency = {
+let IndexingPartitions = {
   gauge: gauge$6,
   set: set$6
 };
 
-let gauge$7 = makeOrThrow$1("envio_indexing_partitions", "The number of partitions used to split fetching logic by addresses and block ranges.", chainIdLabelsSchema);
-
-function set$7(partitionsCount, chainId) {
-  handleInt$1(gauge$7, chainId, partitionsCount);
-}
-
-let IndexingPartitions = {
-  gauge: gauge$7,
-  set: set$7
-};
-
 let counter = makeOrThrow("envio_indexing_idle_seconds", "The time the indexer source syncing has been idle. A high value may indicate the source sync is a bottleneck.", chainIdLabelsSchema);
 
 let IndexingIdleTime = {
@@ -451,51 +440,51 @@ let IndexingQueryTime = {
   counter: counter$2
 };
 
-let gauge$8 = makeOrThrow$1("envio_indexing_buffer_size", "The current number of items in the indexing buffer.", chainIdLabelsSchema);
+let gauge$7 = makeOrThrow$1("envio_indexing_buffer_size", "The current number of items in the indexing buffer.", chainIdLabelsSchema);
 
-function set$8(bufferSize, chainId) {
-  handleInt$1(gauge$8, chainId, bufferSize);
+function set$7(bufferSize, chainId) {
+  handleInt$1(gauge$7, chainId, bufferSize);
 }
 
 let IndexingBufferSize = {
-  gauge: gauge$8,
-  set: set$8
+  gauge: gauge$7,
+  set: set$7
 };
 
-let gauge$9 = PromClient.Gauge.makeGauge({
+let gauge$8 = PromClient.Gauge.makeGauge({
   name: "envio_indexing_target_buffer_size",
-  help: "The target buffer size per chain for indexing. The actual number of items in the queue may exceed this value, but the indexer always tries to keep the buffer filled up to this target."
+  help: "The indexer-wide target buffer size shared across all chains. The actual number of items in the queue may exceed this value, but the indexer always tries to keep the buffer filled up to this target."
 });
 
-function set$9(targetBufferSize) {
-  gauge$9.set(targetBufferSize);
+function set$8(targetBufferSize) {
+  gauge$8.set(targetBufferSize);
 }
 
 let IndexingTargetBufferSize = {
-  gauge: gauge$9,
-  set: set$9
+  gauge: gauge$8,
+  set: set$8
 };
 
-let gauge$10 = makeOrThrow$1("envio_indexing_buffer_block", "The highest block number that has been fully fetched by the indexer.", chainIdLabelsSchema);
+let gauge$9 = makeOrThrow$1("envio_indexing_buffer_block", "The highest block number that has been fully fetched by the indexer.", chainIdLabelsSchema);
 
-function set$10(blockNumber, chainId) {
-  handleInt$1(gauge$10, chainId, blockNumber);
+function set$9(blockNumber, chainId) {
+  handleInt$1(gauge$9, chainId, blockNumber);
 }
 
 let IndexingBufferBlockNumber = {
-  gauge: gauge$10,
-  set: set$10
+  gauge: gauge$9,
+  set: set$9
 };
 
-let gauge$11 = makeOrThrow$1("envio_indexing_end_block", "The block number to stop indexing at. (inclusive)", chainIdLabelsSchema);
+let gauge$10 = makeOrThrow$1("envio_indexing_end_block", "The block number to stop indexing at. (inclusive)", chainIdLabelsSchema);
 
-function set$11(endBlock, chainId) {
-  handleInt$1(gauge$11, chainId, endBlock);
+function set$10(endBlock, chainId) {
+  handleInt$1(gauge$10, chainId, endBlock);
 }
 
 let IndexingEndBlock = {
-  gauge: gauge$11,
-  set: set$11
+  gauge: gauge$10,
+  set: set$10
 };
 
 let sourceLabelsSchema = S$RescriptSchema.schema(s => ({
@@ -536,18 +525,18 @@ let SourceRequestCount = {
   addSeconds: addSeconds
 };
 
-let gauge$12 = makeOrThrow$1("envio_source_known_height", "The latest known block number reported by the source. This value may lag behind the actual chain height, as it is updated only when queried.", sourceLabelsSchema);
+let gauge$11 = makeOrThrow$1("envio_source_known_height", "The latest known block number reported by the source. This value may lag behind the actual chain height, as it is updated only when queried.", sourceLabelsSchema);
 
-function set$12(sourceName, chainId, blockNumber) {
-  handleInt$1(gauge$12, {
+function set$11(sourceName, chainId, blockNumber) {
+  handleInt$1(gauge$11, {
     source: sourceName,
     chainId: chainId
   }, blockNumber);
 }
 
 let SourceHeight = {
-  gauge: gauge$12,
-  set: set$12
+  gauge: gauge$11,
+  set: set$11
 };
 
 let counter$4 = makeOrThrow("envio_reorg_detected_total", "Total number of reorgs detected", chainIdLabelsSchema);
@@ -561,43 +550,43 @@ let ReorgCount = {
   increment: increment$5
 };
 
-let gauge$13 = makeOrThrow$1("envio_reorg_detected_block", "The block number where reorg was detected the last time. This doesn't mean that the block was reorged, this is simply where we found block hash to be different.", chainIdLabelsSchema);
+let gauge$12 = makeOrThrow$1("envio_reorg_detected_block", "The block number where reorg was detected the last time. This doesn't mean that the block was reorged, this is simply where we found block hash to be different.", chainIdLabelsSchema);
 
-function set$13(blockNumber, chain) {
-  handleInt$1(gauge$13, chain, blockNumber);
+function set$12(blockNumber, chain) {
+  handleInt$1(gauge$12, chain, blockNumber);
 }
 
 let ReorgDetectionBlockNumber = {
-  gauge: gauge$13,
-  set: set$13
+  gauge: gauge$12,
+  set: set$12
 };
 
-let gauge$14 = PromClient.Gauge.makeGauge({
+let gauge$13 = PromClient.Gauge.makeGauge({
   name: "envio_reorg_threshold",
   help: "Whether indexing is currently within the reorg threshold"
 });
 
-function set$14(isInReorgThreshold) {
-  gauge$14.set(isInReorgThreshold ? 1 : 0);
+function set$13(isInReorgThreshold) {
+  gauge$13.set(isInReorgThreshold ? 1 : 0);
 }
 
 let ReorgThreshold = {
-  gauge: gauge$14,
-  set: set$14
+  gauge: gauge$13,
+  set: set$13
 };
 
-let gauge$15 = PromClient.Gauge.makeGauge({
+let gauge$14 = PromClient.Gauge.makeGauge({
   name: "envio_rollback_enabled",
   help: "Whether rollback on reorg is enabled"
 });
 
-function set$15(enabled) {
-  gauge$15.set(enabled ? 1 : 0);
+function set$14(enabled) {
+  gauge$14.set(enabled ? 1 : 0);
 }
 
 let RollbackEnabled = {
-  gauge: gauge$15,
-  set: set$15
+  gauge: gauge$14,
+  set: set$14
 };
 
 let timeCounter$3 = PromClient.Counter.makeCounter({
@@ -646,62 +635,62 @@ let RollbackHistoryPrune = {
   increment: increment$7
 };
 
-let gauge$16 = makeOrThrow$1("envio_rollback_target_block", "The block number reorg was rollbacked to the last time.", chainIdLabelsSchema);
+let gauge$15 = makeOrThrow$1("envio_rollback_target_block", "The block number reorg was rollbacked to the last time.", chainIdLabelsSchema);
 
-function set$16(blockNumber, chain) {
-  handleInt$1(gauge$16, chain, blockNumber);
+function set$15(blockNumber, chain) {
+  handleInt$1(gauge$15, chain, blockNumber);
 }
 
 let RollbackTargetBlockNumber = {
-  gauge: gauge$16,
-  set: set$16
+  gauge: gauge$15,
+  set: set$15
 };
 
-let gauge$17 = PromClient.Gauge.makeGauge({
+let gauge$16 = PromClient.Gauge.makeGauge({
   name: "envio_processing_max_batch_size",
   help: "The maximum number of items to process in a single batch."
 });
 
-function set$17(maxBatchSize) {
-  gauge$17.set(maxBatchSize);
+function set$16(maxBatchSize) {
+  gauge$16.set(maxBatchSize);
 }
 
 let ProcessingMaxBatchSize = {
-  gauge: gauge$17,
-  set: set$17
+  gauge: gauge$16,
+  set: set$16
 };
 
-let gauge$18 = makeOrThrow$1("envio_progress_block", "The block number of the latest block processed and stored in the database.", chainIdLabelsSchema);
+let gauge$17 = makeOrThrow$1("envio_progress_block", "The block number of the latest block processed and stored in the database.", chainIdLabelsSchema);
 
-function set$18(blockNumber, chainId) {
-  handleInt$1(gauge$18, chainId, blockNumber);
+function set$17(blockNumber, chainId) {
+  handleInt$1(gauge$17, chainId, blockNumber);
 }
 
 let ProgressBlockNumber = {
-  gauge: gauge$18,
-  set: set$18
+  gauge: gauge$17,
+  set: set$17
 };
 
-let gauge$19 = makeOrThrow$1("envio_progress_events", "The number of events processed and reflected in the database.", chainIdLabelsSchema);
+let gauge$18 = makeOrThrow$1("envio_progress_events", "The number of events processed and reflected in the database.", chainIdLabelsSchema);
 
-function set$19(processedCount, chainId) {
-  handleFloat$1(gauge$19, chainId, processedCount);
+function set$18(processedCount, chainId) {
+  handleFloat$1(gauge$18, chainId, processedCount);
 }
 
 let ProgressEventsCount = {
-  gauge: gauge$19,
-  set: set$19
+  gauge: gauge$18,
+  set: set$18
 };
 
-let gauge$20 = makeOrThrow$1("envio_progress_latency", "The latency in milliseconds between the latest processed event creation and the time it was written to storage.", chainIdLabelsSchema);
+let gauge$19 = makeOrThrow$1("envio_progress_latency", "The latency in milliseconds between the latest processed event creation and the time it was written to storage.", chainIdLabelsSchema);
 
-function set$20(latencyMs, chainId) {
-  handleInt$1(gauge$20, chainId, latencyMs);
+function set$19(latencyMs, chainId) {
+  handleInt$1(gauge$19, chainId, latencyMs);
 }
 
 let ProgressLatency = {
-  gauge: gauge$20,
-  set: set$20
+  gauge: gauge$19,
+  set: set$19
 };
 
 let effectLabelsSchema = S$RescriptSchema.object(s => s.f("effect", S$RescriptSchema.string));
@@ -721,15 +710,15 @@ let EffectCalls = {
   activeCallsCount: activeCallsCount
 };
 
-let gauge$21 = makeOrThrow$1("envio_effect_cache", "The number of items in the effect cache.", effectLabelsSchema);
+let gauge$20 = makeOrThrow$1("envio_effect_cache", "The number of items in the effect cache.", effectLabelsSchema);
 
-function set$21(count, effectName) {
-  handleInt$1(gauge$21, effectName, count);
+function set$20(count, effectName) {
+  handleInt$1(gauge$20, effectName, count);
 }
 
 let EffectCacheCount = {
-  gauge: gauge$21,
-  set: set$21
+  gauge: gauge$20,
+  set: set$20
 };
 
 let counter$7 = makeOrThrow("envio_effect_cache_invalidations", "The number of effect cache invalidations.", effectLabelsSchema);
@@ -743,18 +732,18 @@ let EffectCacheInvalidationsCount = {
   increment: increment$8
 };
 
-let gauge$22 = makeOrThrow$1("envio_effect_queue", "The number of effect calls waiting in the rate limit queue.", effectLabelsSchema);
+let gauge$21 = makeOrThrow$1("envio_effect_queue", "The number of effect calls waiting in the rate limit queue.", effectLabelsSchema);
 
 let timeCounter$6 = makeOrThrow("envio_effect_queue_wait_seconds", "The time spent waiting in the rate limit queue.", effectLabelsSchema);
 
-function set$22(count, effectName) {
-  handleInt$1(gauge$22, effectName, count);
+function set$21(count, effectName) {
+  handleInt$1(gauge$21, effectName, count);
 }
 
 let EffectQueueCount = {
-  gauge: gauge$22,
+  gauge: gauge$21,
   timeCounter: timeCounter$6,
-  set: set$22
+  set: set$21
 };
 
 let loadLabelsSchema = S$RescriptSchema.schema(s => ({
@@ -858,7 +847,6 @@ export {
   Info,
   ProcessStartTimeSeconds,
   IndexingAddresses,
-  IndexingMaxConcurrency,
   IndexingConcurrency,
   IndexingPartitions,
   IndexingIdleTime,

package/src/PruneStaleHistory.res

@@ -0,0 +1,45 @@
+// Throttled prune of stale entity-history rows below the safe checkpoint.
+
+let runPrune = async (state: IndexerState.t) => {
+  switch state->IndexerState.getSafeCheckpointId {
+  | None => ()
+  | Some(safeCheckpointId) =>
+    let persistence = state->IndexerState.persistence
+    await persistence.storage.pruneStaleCheckpoints(~safeCheckpointId)
+
+    for idx in 0 to persistence.allEntities->Array.length - 1 {
+      if idx !== 0 {
+        // Add some delay between entities
+        // To unblock the pg client if it's needed for something else
+        await Utils.delay(1000)
+      }
+      let entityConfig = persistence.allEntities->Array.getUnsafe(idx)
+      let timeRef = Hrtime.makeTimer()
+      try {
+        let () = await persistence.storage.pruneStaleEntityHistory(
+          ~entityName=entityConfig.name,
+          ~entityIndex=entityConfig.index,
+          ~safeCheckpointId,
+        )
+      } catch {
+      | exn =>
+        exn->ErrorHandling.mkLogAndRaise(
+          ~msg=`Failed to prune stale entity history`,
+          ~logger=Logging.createChild(
+            ~params={
+              "entityName": entityConfig.name,
+              "safeCheckpointId": safeCheckpointId,
+            },
+          ),
+        )
+      }
+      Prometheus.RollbackHistoryPrune.increment(
+        ~timeSeconds=Hrtime.timeSince(timeRef)->Hrtime.toSecondsFloat,
+        ~entityName=entityConfig.name,
+      )
+    }
+  }
+}
+
+let schedule = (state: IndexerState.t) =>
+  state->IndexerState.pruneStaleEntityHistoryThrottler->Throttler.schedule(() => runPrune(state))

package/src/PruneStaleHistory.res.mjs

@@ -0,0 +1,46 @@
+// Generated by ReScript, PLEASE EDIT WITH CARE
+
+import * as Utils from "./Utils.res.mjs";
+import * as Hrtime from "./bindings/Hrtime.res.mjs";
+import * as Logging from "./Logging.res.mjs";
+import * as Throttler from "./Throttler.res.mjs";
+import * as Prometheus from "./Prometheus.res.mjs";
+import * as IndexerState from "./IndexerState.res.mjs";
+import * as ErrorHandling from "./ErrorHandling.res.mjs";
+import * as Primitive_exceptions from "@rescript/runtime/lib/es6/Primitive_exceptions.js";
+
+async function runPrune(state) {
+  let safeCheckpointId = IndexerState.getSafeCheckpointId(state);
+  if (safeCheckpointId === undefined) {
+    return;
+  }
+  let persistence = IndexerState.persistence(state);
+  await persistence.storage.pruneStaleCheckpoints(safeCheckpointId);
+  for (let idx = 0, idx_finish = persistence.allEntities.length; idx < idx_finish; ++idx) {
+    if (idx !== 0) {
+      await Utils.delay(1000);
+    }
+    let entityConfig = persistence.allEntities[idx];
+    let timeRef = Hrtime.makeTimer();
+    try {
+      await persistence.storage.pruneStaleEntityHistory(entityConfig.name, entityConfig.index, safeCheckpointId);
+    } catch (raw_exn) {
+      let exn = Primitive_exceptions.internalToException(raw_exn);
+      ErrorHandling.mkLogAndRaise(Logging.createChild({
+        entityName: entityConfig.name,
+        safeCheckpointId: safeCheckpointId
+      }), `Failed to prune stale entity history`, exn);
+    }
+    Prometheus.RollbackHistoryPrune.increment(Hrtime.toSecondsFloat(Hrtime.timeSince(timeRef)), entityConfig.name);
+  }
+}
+
+function schedule(state) {
+  Throttler.schedule(IndexerState.pruneStaleEntityHistoryThrottler(state), () => runPrune(state));
+}
+
+export {
+  runPrune,
+  schedule,
+}
+/* Utils Not a pure module */

package/src/RawEvent.res

@@ -0,0 +1,73 @@
+let convertFieldsToJson = (fields: option<dict<unknown>>) => {
+  switch fields {
+  | None => %raw(`{}`)
+  | Some(fields) =>
+    // Convert bigint fields to string. There are no fields with nested
+    // bigints, so iterating only the top level is safe.
+    fields
+    ->Utils.Dict.mapValues(value =>
+      typeof(value) === #bigint
+        ? value
+          ->(Utils.magic: unknown => bigint)
+          ->BigInt.toString
+          ->(Utils.magic: string => unknown)
+        : value
+    )
+    ->(Utils.magic: dict<unknown> => JSON.t)
+  }
+}
+
+// Block and transaction are passed already extracted from the ecosystem's
+// concrete payload (EVM or Fuel) — `RawEvent` stays payload-shape-agnostic and
+// only needs them as opaque field bags to serialise.
+let make = (
+  eventItem: Internal.eventItem,
+  ~block,
+  ~transaction,
+  ~params: Internal.eventParams,
+  ~srcAddress: Address.t,
+  ~cleanUpRawEventFieldsInPlace: JSON.t => unit,
+): Internal.rawEvent => {
+  let {eventConfig, chain, blockNumber, blockHash, timestamp: blockTimestamp, logIndex} = eventItem
+  let chainId = chain->ChainMap.Chain.toChainId
+  let eventId = EventUtils.packEventIndex(~logIndex, ~blockNumber)
+  let blockFields =
+    block
+    ->(Utils.magic: 'block => option<dict<unknown>>)
+    ->convertFieldsToJson
+  let transactionFields =
+    transaction
+    ->(Utils.magic: 'transaction => option<dict<unknown>>)
+    ->convertFieldsToJson
+
+  blockFields->cleanUpRawEventFieldsInPlace
+
+  // Serialize to unknown, because serializing to Js.Json.t fails for Bytes Fuel type, since it has unknown schema
+  let params =
+    params
+    ->S.reverseConvertOrThrow(eventConfig.paramsRawEventSchema)
+    ->(Utils.magic: unknown => JSON.t)
+  let params = if params === %raw(`null`) {
+    // Should probably make the params field nullable
+    // But this is currently needed to make events
+    // with empty params work
+    %raw(`"null"`)
+  } else {
+    params
+  }
+
+  {
+    chain_id: chainId,
+    event_id: eventId,
+    event_name: eventConfig.name,
+    contract_name: eventConfig.contractName,
+    block_number: blockNumber,
+    log_index: logIndex,
+    src_address: srcAddress,
+    block_hash: blockHash,
+    block_timestamp: blockTimestamp,
+    block_fields: blockFields,
+    transaction_fields: transactionFields,
+    params,
+  }
+}

package/src/RawEvent.res.mjs

@@ -0,0 +1,51 @@
+// Generated by ReScript, PLEASE EDIT WITH CARE
+
+import * as Utils from "./Utils.res.mjs";
+import * as EventUtils from "./EventUtils.res.mjs";
+import * as S$RescriptSchema from "rescript-schema/src/S.res.mjs";
+
+function convertFieldsToJson(fields) {
+  if (fields !== undefined) {
+    return Utils.Dict.mapValues(fields, value => {
+      if (typeof value === "bigint") {
+        return value.toString();
+      } else {
+        return value;
+      }
+    });
+  } else {
+    return {};
+  }
+}
+
+function make(eventItem, block, transaction, params, srcAddress, cleanUpRawEventFieldsInPlace) {
+  let logIndex = eventItem.logIndex;
+  let blockNumber = eventItem.blockNumber;
+  let eventConfig = eventItem.eventConfig;
+  let eventId = EventUtils.packEventIndex(blockNumber, logIndex);
+  let blockFields = convertFieldsToJson(block);
+  let transactionFields = convertFieldsToJson(transaction);
+  cleanUpRawEventFieldsInPlace(blockFields);
+  let params$1 = S$RescriptSchema.reverseConvertOrThrow(params, eventConfig.paramsRawEventSchema);
+  let params$2 = params$1 === null ? "null" : params$1;
+  return {
+    chain_id: eventItem.chain,
+    event_id: eventId,
+    event_name: eventConfig.name,
+    contract_name: eventConfig.contractName,
+    block_number: blockNumber,
+    log_index: logIndex,
+    src_address: srcAddress,
+    block_hash: eventItem.blockHash,
+    block_timestamp: eventItem.timestamp,
+    block_fields: blockFields,
+    transaction_fields: transactionFields,
+    params: params$2
+  };
+}
+
+export {
+  convertFieldsToJson,
+  make,
+}
+/* Utils Not a pure module */