envio 2.32.1 → 3.0.0-alpha-main-clickhouse-sink

package/src/Platform.res.js

@@ -0,0 +1,170 @@
+// Generated by ReScript, PLEASE EDIT WITH CARE
+'use strict';
+
+
+var cleanUpRawEventFieldsInPlace = (fields => {
+    delete fields.hash
+    delete fields.number
+    delete fields.timestamp
+  });
+
+var Evm = {
+  cleanUpRawEventFieldsInPlace: cleanUpRawEventFieldsInPlace
+};
+
+var evm_blockFields = [
+  "number",
+  "timestamp",
+  "hash",
+  "parentHash",
+  "nonce",
+  "sha3Uncles",
+  "logsBloom",
+  "transactionsRoot",
+  "stateRoot",
+  "receiptsRoot",
+  "miner",
+  "difficulty",
+  "totalDifficulty",
+  "extraData",
+  "size",
+  "gasLimit",
+  "gasUsed",
+  "uncles",
+  "baseFeePerGas",
+  "blobGasUsed",
+  "excessBlobGas",
+  "parentBeaconBlockRoot",
+  "withdrawalsRoot",
+  "l1BlockNumber",
+  "sendCount",
+  "sendRoot",
+  "mixHash"
+];
+
+var evm_transactionFields = [
+  "transactionIndex",
+  "hash",
+  "from",
+  "to",
+  "gas",
+  "gasPrice",
+  "maxPriorityFeePerGas",
+  "maxFeePerGas",
+  "cumulativeGasUsed",
+  "effectiveGasPrice",
+  "gasUsed",
+  "input",
+  "nonce",
+  "value",
+  "v",
+  "r",
+  "s",
+  "contractAddress",
+  "logsBloom",
+  "root",
+  "status",
+  "yParity",
+  "chainId",
+  "maxFeePerBlobGas",
+  "blobVersionedHashes",
+  "kind",
+  "l1Fee",
+  "l1GasPrice",
+  "l1GasUsed",
+  "l1FeeScalar",
+  "gasUsedForL1",
+  "accessList",
+  "authorizationList"
+];
+
+function evm_getNumber(prim) {
+  return prim.number;
+}
+
+function evm_getTimestamp(prim) {
+  return prim.timestamp;
+}
+
+function evm_getId(prim) {
+  return prim.hash;
+}
+
+var evm = {
+  name: "evm",
+  blockFields: evm_blockFields,
+  transactionFields: evm_transactionFields,
+  blockNumberName: "number",
+  blockTimestampName: "timestamp",
+  blockHashName: "hash",
+  getNumber: evm_getNumber,
+  getTimestamp: evm_getTimestamp,
+  getId: evm_getId,
+  cleanUpRawEventFieldsInPlace: cleanUpRawEventFieldsInPlace
+};
+
+var cleanUpRawEventFieldsInPlace$1 = (fields => {
+    delete fields.id
+    delete fields.height
+    delete fields.time
+  });
+
+var Fuel = {
+  cleanUpRawEventFieldsInPlace: cleanUpRawEventFieldsInPlace$1
+};
+
+var fuel_blockFields = [
+  "id",
+  "height",
+  "time"
+];
+
+var fuel_transactionFields = ["id"];
+
+function fuel_getNumber(prim) {
+  return prim.height;
+}
+
+function fuel_getTimestamp(prim) {
+  return prim.time;
+}
+
+function fuel_getId(prim) {
+  return prim.id;
+}
+
+var fuel = {
+  name: "fuel",
+  blockFields: fuel_blockFields,
+  transactionFields: fuel_transactionFields,
+  blockNumberName: "height",
+  blockTimestampName: "time",
+  blockHashName: "id",
+  getNumber: fuel_getNumber,
+  getTimestamp: fuel_getTimestamp,
+  getId: fuel_getId,
+  cleanUpRawEventFieldsInPlace: cleanUpRawEventFieldsInPlace$1
+};
+
+function fromName(name) {
+  if (name === "evm") {
+    return evm;
+  } else {
+    return fuel;
+  }
+}
+
+function makeBlockEvent(blockNumber, chainId, platform) {
+  var blockEvent = {};
+  blockEvent["chainId"] = chainId;
+  blockEvent[platform.blockNumberName] = blockNumber;
+  return blockEvent;
+}
+
+exports.Evm = Evm;
+exports.evm = evm;
+exports.Fuel = Fuel;
+exports.fuel = fuel;
+exports.fromName = fromName;
+exports.makeBlockEvent = makeBlockEvent;
+/* No side effect */

package/src/Prometheus.res

@@ -16,6 +16,18 @@ let executeBatchDurationCounter = PromClient.Counter.makeCounter({
   "labelNames": [],
 })
 
+let storageWriteTimeCounter = PromClient.Counter.makeCounter({
+  "name": "envio_storage_write_time",
+  "help": "Cumulative time spent writing batches to storage in milliseconds",
+  "labelNames": [],
+})
+
+let storageWriteCounter = PromClient.Counter.makeCounter({
+  "name": "envio_storage_write_count",
+  "help": "Total number of batch writes to storage",
+  "labelNames": [],
+})
+
 let allChainsSyncedToHead = PromClient.Gauge.makeGauge({
   "name": "hyperindex_synced_to_head",
   "help": "All chains fully synced",
@@ -213,6 +225,14 @@ let incrementExecuteBatchDurationCounter = (~duration) => {
   executeBatchDurationCounter->PromClient.Counter.incMany(duration)
 }
 
+let incrementStorageWriteTimeCounter = (~duration) => {
+  storageWriteTimeCounter->PromClient.Counter.incMany(duration)
+}
+
+let incrementStorageWriteCounter = () => {
+  storageWriteCounter->PromClient.Counter.inc
+}
+
 let setSourceChainHeight = (~blockNumber, ~chainId) => {
   sourceChainHeight
   ->PromClient.Gauge.labels({"chainId": chainId})
@@ -739,3 +759,24 @@ module StorageLoad = {
     sizeCounter->SafeCounter.handleInt(~labels={operation}, ~value=size)
   }
 }
+
+module SinkWrite = {
+  let sinkLabelsSchema = S.object(s => s.field("sink", S.string))
+
+  let timeCounter = SafeCounter.makeOrThrow(
+    ~name="envio_sink_write_time",
+    ~help="Processing time taken to write data to sink. (milliseconds)",
+    ~labelSchema=sinkLabelsSchema,
+  )
+
+  let counter = SafeCounter.makeOrThrow(
+    ~name="envio_sink_write_count",
+    ~help="Cumulative number of successful sink write operations during the indexing process.",
+    ~labelSchema=sinkLabelsSchema,
+  )
+
+  let increment = (~sinkName, ~timeMillis) => {
+    timeCounter->SafeCounter.handleInt(~labels={sinkName}, ~value=timeMillis)
+    counter->SafeCounter.increment(~labels={sinkName})
+  }
+}

package/src/Prometheus.res.js

@@ -27,6 +27,18 @@ var executeBatchDurationCounter = new PromClient.Counter({
       labelNames: []
     });
 
+var storageWriteTimeCounter = new PromClient.Counter({
+      name: "envio_storage_write_time",
+      help: "Cumulative time spent writing batches to storage in milliseconds",
+      labelNames: []
+    });
+
+var storageWriteCounter = new PromClient.Counter({
+      name: "envio_storage_write_count",
+      help: "Total number of batch writes to storage",
+      labelNames: []
+    });
+
 var allChainsSyncedToHead = new PromClient.Gauge({
       name: "hyperindex_synced_to_head",
       help: "All chains fully synced",
@@ -314,6 +326,14 @@ function incrementExecuteBatchDurationCounter(duration) {
   executeBatchDurationCounter.inc(duration);
 }
 
+function incrementStorageWriteTimeCounter(duration) {
+  storageWriteTimeCounter.inc(duration);
+}
+
+function incrementStorageWriteCounter() {
+  storageWriteCounter.inc();
+}
+
 function setSourceChainHeight(blockNumber, chainId) {
   sourceChainHeight.labels({
           chainId: chainId
@@ -808,9 +828,31 @@ var StorageLoad = {
   endOperation: endOperation
 };
 
+var sinkLabelsSchema = S$RescriptSchema.object(function (s) {
+      return s.f("sink", S$RescriptSchema.string);
+    });
+
+var timeCounter$5 = makeOrThrow("envio_sink_write_time", "Processing time taken to write data to sink. (milliseconds)", sinkLabelsSchema);
+
+var counter$8 = makeOrThrow("envio_sink_write_count", "Cumulative number of successful sink write operations during the indexing process.", sinkLabelsSchema);
+
+function increment$7(sinkName, timeMillis) {
+  handleInt(timeCounter$5, sinkName, timeMillis);
+  increment(counter$8, sinkName);
+}
+
+var SinkWrite = {
+  sinkLabelsSchema: sinkLabelsSchema,
+  timeCounter: timeCounter$5,
+  counter: counter$8,
+  increment: increment$7
+};
+
 exports.loadEntitiesDurationCounter = loadEntitiesDurationCounter;
 exports.eventRouterDurationCounter = eventRouterDurationCounter;
 exports.executeBatchDurationCounter = executeBatchDurationCounter;
+exports.storageWriteTimeCounter = storageWriteTimeCounter;
+exports.storageWriteCounter = storageWriteCounter;
 exports.allChainsSyncedToHead = allChainsSyncedToHead;
 exports.sourceChainHeight = sourceChainHeight;
 exports.Labels = Labels;
@@ -823,6 +865,8 @@ exports.BenchmarkSummaryData = BenchmarkSummaryData;
 exports.incrementLoadEntityDurationCounter = incrementLoadEntityDurationCounter;
 exports.incrementEventRouterDurationCounter = incrementEventRouterDurationCounter;
 exports.incrementExecuteBatchDurationCounter = incrementExecuteBatchDurationCounter;
+exports.incrementStorageWriteTimeCounter = incrementStorageWriteTimeCounter;
+exports.incrementStorageWriteCounter = incrementStorageWriteCounter;
 exports.setSourceChainHeight = setSourceChainHeight;
 exports.setAllChainsSyncedToHead = setAllChainsSyncedToHead;
 exports.BenchmarkCounters = BenchmarkCounters;
@@ -860,4 +904,5 @@ exports.EffectCacheCount = EffectCacheCount;
 exports.EffectCacheInvalidationsCount = EffectCacheInvalidationsCount;
 exports.EffectQueueCount = EffectQueueCount;
 exports.StorageLoad = StorageLoad;
+exports.SinkWrite = SinkWrite;
 /* loadEntitiesDurationCounter Not a pure module */

package/src/SafeCheckpointTracking.res

@@ -4,7 +4,7 @@
 // The safe checkpoint id can be used to optimize checkpoints traverse logic and
 // make pruning operation super cheap.
 type t = {
-  checkpointIds: array<int>,
+  checkpointIds: array<float>,
   checkpointBlockNumbers: array<int>,
   maxReorgDepth: int,
 }
@@ -37,9 +37,10 @@ let getSafeCheckpointId = (safeCheckpointTracking: t, ~sourceBlockNumber: int) =
   let safeBlockNumber = sourceBlockNumber - safeCheckpointTracking.maxReorgDepth
 
   switch safeCheckpointTracking.checkpointIds {
-  | [] => 0
+  | [] => 0.
   | _
-    if safeCheckpointTracking.checkpointBlockNumbers->Belt.Array.getUnsafe(0) > safeBlockNumber => 0
+    if safeCheckpointTracking.checkpointBlockNumbers->Belt.Array.getUnsafe(0) >
+      safeBlockNumber => 0.
   | [checkpointId] => checkpointId
   | _ => {
       let trackingCheckpointsCount = safeCheckpointTracking.checkpointIds->Array.length
@@ -70,7 +71,7 @@ let updateOnNewBatch = (
   safeCheckpointTracking: t,
   ~sourceBlockNumber: int,
   ~chainId: int,
-  ~batchCheckpointIds: array<int>,
+  ~batchCheckpointIds: array<float>,
   ~batchCheckpointBlockNumbers: array<int>,
   ~batchCheckpointChainIds: array<int>,
 ) => {

package/src/Sink.res

@@ -0,0 +1,47 @@
+type t = {
+  name: string,
+  initialize: (
+    ~chainConfigs: array<Config.chain>=?,
+    ~entities: array<Internal.entityConfig>=?,
+    ~enums: array<Table.enumConfig<Table.enum>>=?,
+  ) => promise<unit>,
+  resume: (~checkpointId: float) => promise<unit>,
+  writeBatch: (
+    ~batch: Batch.t,
+    ~updatedEntities: array<Persistence.updatedEntity>,
+  ) => promise<unit>,
+}
+
+let makeClickHouse = (~host, ~database, ~username, ~password): t => {
+  let client = ClickHouse.createClient({
+    url: host,
+    username,
+    password,
+  })
+
+  // Don't assign it to client immediately,
+  // since it will fail if the database doesn't exist
+  // Call USE database instead
+  let database = switch database {
+  | Some(database) => database
+  | None => "envio_sink"
+  }
+
+  {
+    name: "ClickHouse",
+    initialize: (~chainConfigs as _=[], ~entities=[], ~enums=[]) => {
+      ClickHouse.initialize(client, ~database, ~entities, ~enums)
+    },
+    resume: (~checkpointId) => {
+      ClickHouse.resume(client, ~database, ~checkpointId)
+    },
+    writeBatch: async (~batch, ~updatedEntities) => {
+      await Promise.all(
+        updatedEntities->Belt.Array.map(({entityConfig, updates}) => {
+          ClickHouse.setUpdatesOrThrow(client, ~updates, ~entityConfig, ~database)
+        }),
+      )->Promise.ignoreValue
+      await ClickHouse.setCheckpointsOrThrow(client, ~batch, ~database)
+    },
+  }
+}

package/src/Sink.res.js

@@ -0,0 +1,36 @@
+// Generated by ReScript, PLEASE EDIT WITH CARE
+'use strict';
+
+var Belt_Array = require("rescript/lib/js/belt_Array.js");
+var ClickHouse = require("./bindings/ClickHouse.res.js");
+var Client = require("@clickhouse/client");
+
+function makeClickHouse(host, database, username, password) {
+  var client = Client.createClient({
+        url: host,
+        username: username,
+        password: password
+      });
+  var database$1 = database !== undefined ? database : "envio_sink";
+  return {
+          name: "ClickHouse",
+          initialize: (function ($staropt$star, $staropt$star$1, $staropt$star$2) {
+              $staropt$star !== undefined;
+              var entities = $staropt$star$1 !== undefined ? $staropt$star$1 : [];
+              var enums = $staropt$star$2 !== undefined ? $staropt$star$2 : [];
+              return ClickHouse.initialize(client, database$1, entities, enums);
+            }),
+          resume: (function (checkpointId) {
+              return ClickHouse.resume(client, database$1, checkpointId);
+            }),
+          writeBatch: (async function (batch, updatedEntities) {
+              await Promise.all(Belt_Array.map(updatedEntities, (function (param) {
+                          return ClickHouse.setUpdatesOrThrow(client, param.updates, param.entityConfig, database$1);
+                        })));
+              return await ClickHouse.setCheckpointsOrThrow(client, batch, database$1);
+            })
+        };
+}
+
+exports.makeClickHouse = makeClickHouse;
+/* ClickHouse Not a pure module */

package/src/Utils.res

@@ -199,6 +199,8 @@ module UnsafeIntOperators = {
 }
 
 module Array = {
+  let immutableEmpty: array<unknown> = []
+
   @send
   external forEachAsync: (array<'a>, 'a => promise<unit>) => unit = "forEach"
 

package/src/Utils.res.js

@@ -199,6 +199,8 @@ var $$Math = {
 
 var UnsafeIntOperators = {};
 
+var immutableEmpty = [];
+
 function mergeSorted(f, xs, ys) {
   if (xs.length === 0) {
     return ys;
@@ -345,6 +347,7 @@ function interleave(arr, separator) {
 }
 
 var $$Array$1 = {
+  immutableEmpty: immutableEmpty,
   mergeSorted: mergeSorted,
   clearInPlace: clearInPlace,
   setIndexImmutable: setIndexImmutable,