envio 2.32.1 → 3.0.0-alpha-main-clickhouse-sink

package/index.d.ts

@@ -7,6 +7,7 @@ export type {
   rateLimitDuration as RateLimitDuration,
   rateLimit as RateLimit,
   blockEvent as BlockEvent,
+  fuelBlockEvent as FuelBlockEvent,
   onBlockArgs as OnBlockArgs,
   onBlockOptions as OnBlockOptions,
 } from "./src/Envio.gen.ts";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "envio",
-  "version": "v2.32.1",
+  "version": "v3.0.0-alpha-main-clickhouse-sink",
   "description": "A latency and sync speed optimized, developer friendly blockchain data indexer.",
   "bin": "./bin.js",
   "main": "./index.js",
@@ -25,12 +25,13 @@
   },
   "homepage": "https://envio.dev",
   "optionalDependencies": {
-    "envio-linux-x64": "v2.32.1",
-    "envio-linux-arm64": "v2.32.1",
-    "envio-darwin-x64": "v2.32.1",
-    "envio-darwin-arm64": "v2.32.1"
+    "envio-linux-x64": "v3.0.0-alpha-main-clickhouse-sink",
+    "envio-linux-arm64": "v3.0.0-alpha-main-clickhouse-sink",
+    "envio-darwin-x64": "v3.0.0-alpha-main-clickhouse-sink",
+    "envio-darwin-arm64": "v3.0.0-alpha-main-clickhouse-sink"
   },
   "dependencies": {
+    "@clickhouse/client": "1.12.1",
     "@envio-dev/hypersync-client": "0.6.6",
     "@envio-dev/hyperfuel-client": "1.2.2",
     "rescript": "11.1.3",

package/src/Batch.res

@@ -24,7 +24,7 @@ type t = {
   items: array<Internal.item>,
   progressedChainsById: dict<chainAfterBatch>,
   // Unnest-like checkpoint fields:
-  checkpointIds: array<int>,
+  checkpointIds: array<float>,
   checkpointChainIds: array<int>,
   checkpointBlockNumbers: array<int>,
   checkpointBlockHashes: array<Js.Null.t<string>>,
@@ -196,7 +196,7 @@ let addReorgCheckpoints = (
     for blockNumber in fromBlockExclusive + 1 to toBlockExclusive - 1 {
       switch reorgDetection->ReorgDetection.getHashByBlockNumber(~blockNumber) {
       | Js.Null.Value(hash) =>
-        let checkpointId = prevCheckpointId.contents + 1
+        let checkpointId = prevCheckpointId.contents +. 1.
         prevCheckpointId := checkpointId
 
         mutCheckpointIds->Js.Array2.push(checkpointId)->ignore
@@ -277,7 +277,7 @@ let prepareOrderedBatch = (
               ~mutCheckpointEventsProcessed=checkpointEventsProcessed,
             )
 
-          let checkpointId = prevCheckpointId.contents + 1
+          let checkpointId = prevCheckpointId.contents +. 1.
 
           items
           ->Js.Array2.push(item0)
@@ -421,7 +421,7 @@ let prepareUnorderedBatch = (
               ~mutCheckpointEventsProcessed=checkpointEventsProcessed,
             )
 
-          let checkpointId = prevCheckpointId.contents + 1
+          let checkpointId = prevCheckpointId.contents +. 1.
 
           checkpointIds->Js.Array2.push(checkpointId)->ignore
           checkpointChainIds->Js.Array2.push(fetchState.chainId)->ignore

package/src/Change.res

@@ -0,0 +1,9 @@
+@tag("type")
+type t<'entity> =
+  | @as("SET") Set({entityId: string, entity: 'entity, checkpointId: float})
+  | @as("DELETE") Delete({entityId: string, checkpointId: float})
+
+@get
+external getEntityId: t<'entity> => string = "entityId"
+@get
+external getCheckpointId: t<'entity> => float = "checkpointId"

package/src/Change.res.js

@@ -0,0 +1,2 @@
+// Generated by ReScript, PLEASE EDIT WITH CARE
+/* This output is empty. Its source's type definitions, externals and/or unused code got optimized away. */

package/src/Config.res

@@ -1,7 +1,5 @@
 open Belt
 
-type ecosystem = | @as("evm") Evm | @as("fuel") Fuel
-
 type sourceSyncOptions = {
   initialBlockInterval?: int,
   backoffMultiplicative?: float,
@@ -47,7 +45,7 @@ type t = {
   multichain: multichain,
   chainMap: ChainMap.t<chain>,
   defaultChain: option<chain>,
-  ecosystem: ecosystem,
+  platform: Platform.t,
   enableRawEvents: bool,
   preloadHandlers: bool,
   maxAddrInPartition: int,
@@ -62,7 +60,7 @@ let make = (
   ~chains: array<chain>=[],
   ~enableRawEvents=false,
   ~preloadHandlers=false,
-  ~ecosystem=Evm,
+  ~ecosystem: Platform.name=Platform.Evm,
   ~batchSize=5000,
   ~lowercaseAddresses=false,
   ~multichain=Unordered,
@@ -92,6 +90,8 @@ let make = (
     })
   })
 
+  let platform = Platform.fromName(ecosystem)
+
   {
     shouldRollbackOnReorg,
     shouldSaveFullHistory,
@@ -99,7 +99,7 @@ let make = (
     chainMap,
     defaultChain: chains->Array.get(0),
     enableRawEvents,
-    ecosystem,
+    platform,
     maxAddrInPartition,
     preloadHandlers,
     batchSize,

package/src/Config.res.js

@@ -4,6 +4,7 @@
 var Utils = require("./Utils.res.js");
 var Js_exn = require("rescript/lib/js/js_exn.js");
 var ChainMap = require("./ChainMap.res.js");
+var Platform = require("./Platform.res.js");
 var Belt_Array = require("rescript/lib/js/belt_Array.js");
 
 function make(shouldRollbackOnReorgOpt, shouldSaveFullHistoryOpt, chainsOpt, enableRawEventsOpt, preloadHandlersOpt, ecosystemOpt, batchSizeOpt, lowercaseAddressesOpt, multichainOpt, shouldUseHypersyncClientDecoderOpt, maxAddrInPartitionOpt) {
@@ -34,13 +35,14 @@ function make(shouldRollbackOnReorgOpt, shouldSaveFullHistoryOpt, chainsOpt, ena
                   addContractNameToContractNameMapping[addKey] = contract.name;
                 }));
         }));
+  var platform = Platform.fromName(ecosystem);
   return {
           shouldRollbackOnReorg: shouldRollbackOnReorg,
           shouldSaveFullHistory: shouldSaveFullHistory,
           multichain: multichain,
           chainMap: chainMap,
           defaultChain: Belt_Array.get(chains, 0),
-          ecosystem: ecosystem,
+          platform: platform,
           enableRawEvents: enableRawEvents,
           preloadHandlers: preloadHandlers,
           maxAddrInPartition: maxAddrInPartition,

package/src/Envio.gen.ts

@@ -11,11 +11,11 @@ import type {Logger as $$logger} from './Types.ts';
 
 import type {S_t as RescriptSchema_S_t} from 'rescript-schema/RescriptSchema.gen';
 
-import type {blockEvent as Internal_blockEvent} from './Internal.gen';
+export type blockEvent = { readonly number: number; readonly chainId: number };
 
-export type blockEvent = Internal_blockEvent;
+export type fuelBlockEvent = { readonly height: number; readonly chainId: number };
 
-export type onBlockArgs<context> = { readonly block: blockEvent; readonly context: context };
+export type onBlockArgs<block,context> = { readonly block: block; readonly context: context };
 
 export type onBlockOptions<chain> = {
   readonly name: string; 

package/src/Envio.res

@@ -2,12 +2,23 @@
 // Should be an entry point after we get rid of the generated project.
 // Don't forget to keep index.d.ts in sync with this file.
 
+// EVM block event with 'number' field for backward compatibility
 @genType
-type blockEvent = Internal.blockEvent
+type blockEvent = {
+  number: int,
+  chainId: int,
+}
+
+// Fuel block event with 'height' field
+@genType
+type fuelBlockEvent = {
+  height: int,
+  chainId: int,
+}
 
 @genType
-type onBlockArgs<'context> = {
-  block: blockEvent,
+type onBlockArgs<'block, 'context> = {
+  block: 'block,
   context: 'context,
 }
 

package/src/EventRegister.res

@@ -4,7 +4,7 @@ type registrations = {
 }
 
 type activeRegistration = {
-  ecosystem: Config.ecosystem,
+  platform: Platform.t,
   multichain: Config.multichain,
   preloadHandlers: bool,
   registrations: registrations,
@@ -36,9 +36,9 @@ let withRegistration = (fn: activeRegistration => unit) => {
   }
 }
 
-let startRegistration = (~ecosystem, ~multichain, ~preloadHandlers) => {
+let startRegistration = (~platform, ~multichain, ~preloadHandlers) => {
   let r = {
-    ecosystem,
+    platform,
     multichain,
     preloadHandlers,
     registrations: {
@@ -80,14 +80,6 @@ let onBlockOptionsSchema = S.schema(s =>
 
 let onBlock = (rawOptions: unknown, handler: Internal.onBlockArgs => promise<unit>) => {
   withRegistration(registration => {
-    // There's no big reason for this. It's just more work
-    switch registration.ecosystem {
-    | Evm => ()
-    | Fuel =>
-      Js.Exn.raiseError(
-        "Block Handlers are not supported for non-EVM ecosystems. Please reach out to the Envio team if you need this feature.",
-      )
-    }
     // We need to get timestamp for ordered multichain mode
     switch registration.multichain {
     | Unordered => ()

package/src/EventRegister.res.js

@@ -28,9 +28,9 @@ function withRegistration(fn) {
   }
 }
 
-function startRegistration(ecosystem, multichain, preloadHandlers) {
+function startRegistration(platform, multichain, preloadHandlers) {
   var r = {
-    ecosystem: ecosystem,
+    platform: platform,
     multichain: multichain,
     preloadHandlers: preloadHandlers,
     registrations: {
@@ -71,12 +71,8 @@ var onBlockOptionsSchema = S$RescriptSchema.schema(function (s) {
 
 function onBlock(rawOptions, handler) {
   withRegistration(function (registration) {
-        var match = registration.ecosystem;
-        if (match !== "evm") {
-          Js_exn.raiseError("Block Handlers are not supported for non-EVM ecosystems. Please reach out to the Envio team if you need this feature.");
-        }
-        var match$1 = registration.multichain;
-        if (match$1 === "ordered") {
+        var match = registration.multichain;
+        if (match === "ordered") {
           Js_exn.raiseError("Block Handlers are not supported for ordered multichain mode. Please reach out to the Envio team if you need this feature or enable unordered multichain mode with `unordered_multichain_mode: true` in your config.");
         }
         if (registration.preloadHandlers) {

package/src/EventRegister.resi

@@ -4,7 +4,7 @@ type registrations = {
 }
 
 let startRegistration: (
-  ~ecosystem: Config.ecosystem,
+  ~platform: Platform.t,
   ~multichain: Config.multichain,
   ~preloadHandlers: bool,
 ) => unit

package/src/InMemoryStore.gen.ts

@@ -0,0 +1,6 @@
+/* TypeScript file generated from InMemoryStore.res by genType. */
+
+/* eslint-disable */
+/* tslint:disable */
+
+export type rawEventsKey = { readonly chainId: number; readonly eventId: string };

package/src/InMemoryStore.res

@@ -0,0 +1,149 @@
+@genType
+type rawEventsKey = {
+  chainId: int,
+  eventId: string,
+}
+
+let hashRawEventsKey = (key: rawEventsKey) =>
+  EventUtils.getEventIdKeyString(~chainId=key.chainId, ~eventId=key.eventId)
+
+module EntityTables = {
+  type t = dict<InMemoryTable.Entity.t<Internal.entity>>
+  exception UndefinedEntity({entityName: string})
+  let make = (entities: array<Internal.entityConfig>): t => {
+    let init = Js.Dict.empty()
+    entities->Belt.Array.forEach(entityConfig => {
+      init->Js.Dict.set((entityConfig.name :> string), InMemoryTable.Entity.make())
+    })
+    init
+  }
+
+  let get = (type entity, self: t, ~entityName: string) => {
+    switch self->Utils.Dict.dangerouslyGetNonOption(entityName) {
+    | Some(table) =>
+      table->(
+        Utils.magic: InMemoryTable.Entity.t<Internal.entity> => InMemoryTable.Entity.t<entity>
+      )
+
+    | None =>
+      UndefinedEntity({entityName: entityName})->ErrorHandling.mkLogAndRaise(
+        ~msg="Unexpected, entity InMemoryTable is undefined",
+      )
+    }
+  }
+
+  let clone = (self: t) => {
+    self
+    ->Js.Dict.entries
+    ->Belt.Array.map(((k, v)) => (k, v->InMemoryTable.Entity.clone))
+    ->Js.Dict.fromArray
+  }
+}
+
+type effectCacheInMemTable = {
+  idsToStore: array<string>,
+  mutable invalidationsCount: int,
+  dict: dict<Internal.effectOutput>,
+  effect: Internal.effect,
+}
+
+type t = {
+  rawEvents: InMemoryTable.t<rawEventsKey, InternalTable.RawEvents.t>,
+  entities: dict<InMemoryTable.Entity.t<Internal.entity>>,
+  effects: dict<effectCacheInMemTable>,
+  rollbackTargetCheckpointId: option<Internal.checkpointId>,
+}
+
+let make = (~entities: array<Internal.entityConfig>, ~rollbackTargetCheckpointId=?): t => {
+  rawEvents: InMemoryTable.make(~hash=hashRawEventsKey),
+  entities: EntityTables.make(entities),
+  effects: Js.Dict.empty(),
+  rollbackTargetCheckpointId,
+}
+
+let clone = (self: t) => {
+  rawEvents: self.rawEvents->InMemoryTable.clone,
+  entities: self.entities->EntityTables.clone,
+  effects: Js.Dict.map(table => {
+    idsToStore: table.idsToStore->Array.copy,
+    invalidationsCount: table.invalidationsCount,
+    dict: table.dict->Utils.Dict.shallowCopy,
+    effect: table.effect,
+  }, self.effects),
+  rollbackTargetCheckpointId: self.rollbackTargetCheckpointId,
+}
+
+let getEffectInMemTable = (inMemoryStore: t, ~effect: Internal.effect) => {
+  let key = effect.name
+  switch inMemoryStore.effects->Utils.Dict.dangerouslyGetNonOption(key) {
+  | Some(table) => table
+  | None =>
+    let table = {
+      idsToStore: [],
+      dict: Js.Dict.empty(),
+      invalidationsCount: 0,
+      effect,
+    }
+    inMemoryStore.effects->Js.Dict.set(key, table)
+    table
+  }
+}
+
+let getInMemTable = (
+  inMemoryStore: t,
+  ~entityConfig: Internal.entityConfig,
+): InMemoryTable.Entity.t<Internal.entity> => {
+  inMemoryStore.entities->EntityTables.get(~entityName=entityConfig.name)
+}
+
+let isRollingBack = (inMemoryStore: t) => inMemoryStore.rollbackTargetCheckpointId !== None
+
+let setBatchDcs = (inMemoryStore: t, ~batch: Batch.t, ~shouldSaveHistory) => {
+  let inMemTable =
+    inMemoryStore->getInMemTable(~entityConfig=InternalTable.DynamicContractRegistry.config)
+
+  let itemIdx = ref(0)
+
+  for checkpoint in 0 to batch.checkpointIds->Array.length - 1 {
+    let checkpointId = batch.checkpointIds->Js.Array2.unsafe_get(checkpoint)
+    let chainId = batch.checkpointChainIds->Js.Array2.unsafe_get(checkpoint)
+    let checkpointEventsProcessed =
+      batch.checkpointEventsProcessed->Js.Array2.unsafe_get(checkpoint)
+
+    for idx in 0 to checkpointEventsProcessed - 1 {
+      let item = batch.items->Js.Array2.unsafe_get(itemIdx.contents + idx)
+      switch item->Internal.getItemDcs {
+      | None => ()
+      | Some(dcs) =>
+        // Currently only events support contract registration, so we can cast to event item
+        let eventItem = item->Internal.castUnsafeEventItem
+        for dcIdx in 0 to dcs->Array.length - 1 {
+          let dc = dcs->Js.Array2.unsafe_get(dcIdx)
+          let entity: InternalTable.DynamicContractRegistry.t = {
+            id: InternalTable.DynamicContractRegistry.makeId(~chainId, ~contractAddress=dc.address),
+            chainId,
+            contractAddress: dc.address,
+            contractName: dc.contractName,
+            registeringEventBlockNumber: eventItem.blockNumber,
+            registeringEventLogIndex: eventItem.logIndex,
+            registeringEventBlockTimestamp: eventItem.timestamp,
+            registeringEventContractName: eventItem.eventConfig.contractName,
+            registeringEventName: eventItem.eventConfig.name,
+            registeringEventSrcAddress: eventItem.event.srcAddress,
+          }
+
+          inMemTable->InMemoryTable.Entity.set(
+            Set({
+              entityId: entity.id,
+              checkpointId,
+              entity: entity->InternalTable.DynamicContractRegistry.castToInternal,
+            }),
+            ~shouldSaveHistory,
+          )
+        }
+      }
+    }
+
+    itemIdx := itemIdx.contents + checkpointEventsProcessed
+  }
+}

package/src/InMemoryStore.res.js

@@ -0,0 +1,161 @@
+// Generated by ReScript, PLEASE EDIT WITH CARE
+'use strict';
+
+var $$Array = require("rescript/lib/js/array.js");
+var Utils = require("./Utils.res.js");
+var Js_dict = require("rescript/lib/js/js_dict.js");
+var Belt_Array = require("rescript/lib/js/belt_Array.js");
+var EventUtils = require("./EventUtils.res.js");
+var ErrorHandling = require("./ErrorHandling.res.js");
+var InMemoryTable = require("./InMemoryTable.res.js");
+var InternalTable = require("./db/InternalTable.res.js");
+var Caml_exceptions = require("rescript/lib/js/caml_exceptions.js");
+
+function hashRawEventsKey(key) {
+  return EventUtils.getEventIdKeyString(key.chainId, key.eventId);
+}
+
+var UndefinedEntity = /* @__PURE__ */Caml_exceptions.create("InMemoryStore.EntityTables.UndefinedEntity");
+
+function make(entities) {
+  var init = {};
+  Belt_Array.forEach(entities, (function (entityConfig) {
+          init[entityConfig.name] = InMemoryTable.Entity.make();
+        }));
+  return init;
+}
+
+function get(self, entityName) {
+  var table = self[entityName];
+  if (table !== undefined) {
+    return table;
+  } else {
+    return ErrorHandling.mkLogAndRaise(undefined, "Unexpected, entity InMemoryTable is undefined", {
+                RE_EXN_ID: UndefinedEntity,
+                entityName: entityName
+              });
+  }
+}
+
+function clone(self) {
+  return Js_dict.fromArray(Belt_Array.map(Js_dict.entries(self), (function (param) {
+                    return [
+                            param[0],
+                            InMemoryTable.Entity.clone(param[1])
+                          ];
+                  })));
+}
+
+var EntityTables = {
+  UndefinedEntity: UndefinedEntity,
+  make: make,
+  get: get,
+  clone: clone
+};
+
+function make$1(entities, rollbackTargetCheckpointId) {
+  return {
+          rawEvents: InMemoryTable.make(hashRawEventsKey),
+          entities: make(entities),
+          effects: {},
+          rollbackTargetCheckpointId: rollbackTargetCheckpointId
+        };
+}
+
+function clone$1(self) {
+  return {
+          rawEvents: InMemoryTable.clone(self.rawEvents),
+          entities: clone(self.entities),
+          effects: Js_dict.map((function (table) {
+                  return {
+                          idsToStore: $$Array.copy(table.idsToStore),
+                          invalidationsCount: table.invalidationsCount,
+                          dict: Utils.Dict.shallowCopy(table.dict),
+                          effect: table.effect
+                        };
+                }), self.effects),
+          rollbackTargetCheckpointId: self.rollbackTargetCheckpointId
+        };
+}
+
+function getEffectInMemTable(inMemoryStore, effect) {
+  var key = effect.name;
+  var table = inMemoryStore.effects[key];
+  if (table !== undefined) {
+    return table;
+  }
+  var table$1 = {
+    idsToStore: [],
+    invalidationsCount: 0,
+    dict: {},
+    effect: effect
+  };
+  inMemoryStore.effects[key] = table$1;
+  return table$1;
+}
+
+function getInMemTable(inMemoryStore, entityConfig) {
+  return get(inMemoryStore.entities, entityConfig.name);
+}
+
+function isRollingBack(inMemoryStore) {
+  return inMemoryStore.rollbackTargetCheckpointId !== undefined;
+}
+
+function setBatchDcs(inMemoryStore, batch, shouldSaveHistory) {
+  var inMemTable = getInMemTable(inMemoryStore, InternalTable.DynamicContractRegistry.config);
+  var itemIdx = 0;
+  for(var checkpoint = 0 ,checkpoint_finish = batch.checkpointIds.length; checkpoint < checkpoint_finish; ++checkpoint){
+    var checkpointId = batch.checkpointIds[checkpoint];
+    var chainId = batch.checkpointChainIds[checkpoint];
+    var checkpointEventsProcessed = batch.checkpointEventsProcessed[checkpoint];
+    for(var idx = 0; idx < checkpointEventsProcessed; ++idx){
+      var item = batch.items[itemIdx + idx | 0];
+      var dcs = item.dcs;
+      if (dcs !== undefined) {
+        for(var dcIdx = 0 ,dcIdx_finish = dcs.length; dcIdx < dcIdx_finish; ++dcIdx){
+          var dc = dcs[dcIdx];
+          var entity_id = InternalTable.DynamicContractRegistry.makeId(chainId, dc.address);
+          var entity_registering_event_block_number = item.blockNumber;
+          var entity_registering_event_log_index = item.logIndex;
+          var entity_registering_event_block_timestamp = item.timestamp;
+          var entity_registering_event_contract_name = item.eventConfig.contractName;
+          var entity_registering_event_name = item.eventConfig.name;
+          var entity_registering_event_src_address = item.event.srcAddress;
+          var entity_contract_address = dc.address;
+          var entity_contract_name = dc.contractName;
+          var entity = {
+            id: entity_id,
+            chain_id: chainId,
+            registering_event_block_number: entity_registering_event_block_number,
+            registering_event_log_index: entity_registering_event_log_index,
+            registering_event_block_timestamp: entity_registering_event_block_timestamp,
+            registering_event_contract_name: entity_registering_event_contract_name,
+            registering_event_name: entity_registering_event_name,
+            registering_event_src_address: entity_registering_event_src_address,
+            contract_address: entity_contract_address,
+            contract_name: entity_contract_name
+          };
+          InMemoryTable.Entity.set(inMemTable, {
+                type: "SET",
+                entityId: entity_id,
+                entity: entity,
+                checkpointId: checkpointId
+              }, shouldSaveHistory, undefined);
+        }
+      }
+      
+    }
+    itemIdx = itemIdx + checkpointEventsProcessed | 0;
+  }
+}
+
+exports.hashRawEventsKey = hashRawEventsKey;
+exports.EntityTables = EntityTables;
+exports.make = make$1;
+exports.clone = clone$1;
+exports.getEffectInMemTable = getEffectInMemTable;
+exports.getInMemTable = getInMemTable;
+exports.isRollingBack = isRollingBack;
+exports.setBatchDcs = setBatchDcs;
+/* Utils Not a pure module */