envio 2.32.2 → 3.0.0-alpha-main-clickhouse-sink

package/index.d.ts

@@ -7,6 +7,7 @@ export type {
   rateLimitDuration as RateLimitDuration,
   rateLimit as RateLimit,
   blockEvent as BlockEvent,
+  fuelBlockEvent as FuelBlockEvent,
   onBlockArgs as OnBlockArgs,
   onBlockOptions as OnBlockOptions,
 } from "./src/Envio.gen.ts";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "envio",
-  "version": "v2.32.2",
+  "version": "v3.0.0-alpha-main-clickhouse-sink",
   "description": "A latency and sync speed optimized, developer friendly blockchain data indexer.",
   "bin": "./bin.js",
   "main": "./index.js",
@@ -25,12 +25,13 @@
   },
   "homepage": "https://envio.dev",
   "optionalDependencies": {
-    "envio-linux-x64": "v2.32.2",
-    "envio-linux-arm64": "v2.32.2",
-    "envio-darwin-x64": "v2.32.2",
-    "envio-darwin-arm64": "v2.32.2"
+    "envio-linux-x64": "v3.0.0-alpha-main-clickhouse-sink",
+    "envio-linux-arm64": "v3.0.0-alpha-main-clickhouse-sink",
+    "envio-darwin-x64": "v3.0.0-alpha-main-clickhouse-sink",
+    "envio-darwin-arm64": "v3.0.0-alpha-main-clickhouse-sink"
   },
   "dependencies": {
+    "@clickhouse/client": "1.12.1",
     "@envio-dev/hypersync-client": "0.6.6",
     "@envio-dev/hyperfuel-client": "1.2.2",
     "rescript": "11.1.3",

package/src/Batch.res

@@ -24,7 +24,7 @@ type t = {
   items: array<Internal.item>,
   progressedChainsById: dict<chainAfterBatch>,
   // Unnest-like checkpoint fields:
-  checkpointIds: array<int>,
+  checkpointIds: array<float>,
   checkpointChainIds: array<int>,
   checkpointBlockNumbers: array<int>,
   checkpointBlockHashes: array<Js.Null.t<string>>,
@@ -196,7 +196,7 @@ let addReorgCheckpoints = (
     for blockNumber in fromBlockExclusive + 1 to toBlockExclusive - 1 {
       switch reorgDetection->ReorgDetection.getHashByBlockNumber(~blockNumber) {
       | Js.Null.Value(hash) =>
-        let checkpointId = prevCheckpointId.contents + 1
+        let checkpointId = prevCheckpointId.contents +. 1.
         prevCheckpointId := checkpointId
 
         mutCheckpointIds->Js.Array2.push(checkpointId)->ignore
@@ -277,7 +277,7 @@ let prepareOrderedBatch = (
               ~mutCheckpointEventsProcessed=checkpointEventsProcessed,
             )
 
-          let checkpointId = prevCheckpointId.contents + 1
+          let checkpointId = prevCheckpointId.contents +. 1.
 
           items
           ->Js.Array2.push(item0)
@@ -421,7 +421,7 @@ let prepareUnorderedBatch = (
               ~mutCheckpointEventsProcessed=checkpointEventsProcessed,
             )
 
-          let checkpointId = prevCheckpointId.contents + 1
+          let checkpointId = prevCheckpointId.contents +. 1.
 
           checkpointIds->Js.Array2.push(checkpointId)->ignore
           checkpointChainIds->Js.Array2.push(fetchState.chainId)->ignore

package/src/Change.res

@@ -0,0 +1,9 @@
+@tag("type")
+type t<'entity> =
+  | @as("SET") Set({entityId: string, entity: 'entity, checkpointId: float})
+  | @as("DELETE") Delete({entityId: string, checkpointId: float})
+
+@get
+external getEntityId: t<'entity> => string = "entityId"
+@get
+external getCheckpointId: t<'entity> => float = "checkpointId"

package/src/Change.res.js

@@ -0,0 +1,2 @@
+// Generated by ReScript, PLEASE EDIT WITH CARE
+/* This output is empty. Its source's type definitions, externals and/or unused code got optimized away. */

package/src/Config.res

@@ -1,7 +1,5 @@
 open Belt
 
-type ecosystem = | @as("evm") Evm | @as("fuel") Fuel
-
 type sourceSyncOptions = {
   initialBlockInterval?: int,
   backoffMultiplicative?: float,
@@ -47,7 +45,7 @@ type t = {
   multichain: multichain,
   chainMap: ChainMap.t<chain>,
   defaultChain: option<chain>,
-  ecosystem: ecosystem,
+  platform: Platform.t,
   enableRawEvents: bool,
   preloadHandlers: bool,
   maxAddrInPartition: int,
@@ -62,7 +60,7 @@ let make = (
   ~chains: array<chain>=[],
   ~enableRawEvents=false,
   ~preloadHandlers=false,
-  ~ecosystem=Evm,
+  ~ecosystem: Platform.name=Platform.Evm,
   ~batchSize=5000,
   ~lowercaseAddresses=false,
   ~multichain=Unordered,
@@ -92,6 +90,8 @@ let make = (
     })
   })
 
+  let platform = Platform.fromName(ecosystem)
+
   {
     shouldRollbackOnReorg,
     shouldSaveFullHistory,
@@ -99,7 +99,7 @@ let make = (
     chainMap,
     defaultChain: chains->Array.get(0),
     enableRawEvents,
-    ecosystem,
+    platform,
     maxAddrInPartition,
     preloadHandlers,
     batchSize,

package/src/Config.res.js

@@ -4,6 +4,7 @@
 var Utils = require("./Utils.res.js");
 var Js_exn = require("rescript/lib/js/js_exn.js");
 var ChainMap = require("./ChainMap.res.js");
+var Platform = require("./Platform.res.js");
 var Belt_Array = require("rescript/lib/js/belt_Array.js");
 
 function make(shouldRollbackOnReorgOpt, shouldSaveFullHistoryOpt, chainsOpt, enableRawEventsOpt, preloadHandlersOpt, ecosystemOpt, batchSizeOpt, lowercaseAddressesOpt, multichainOpt, shouldUseHypersyncClientDecoderOpt, maxAddrInPartitionOpt) {
@@ -34,13 +35,14 @@ function make(shouldRollbackOnReorgOpt, shouldSaveFullHistoryOpt, chainsOpt, ena
                   addContractNameToContractNameMapping[addKey] = contract.name;
                 }));
         }));
+  var platform = Platform.fromName(ecosystem);
   return {
           shouldRollbackOnReorg: shouldRollbackOnReorg,
           shouldSaveFullHistory: shouldSaveFullHistory,
           multichain: multichain,
           chainMap: chainMap,
           defaultChain: Belt_Array.get(chains, 0),
-          ecosystem: ecosystem,
+          platform: platform,
           enableRawEvents: enableRawEvents,
           preloadHandlers: preloadHandlers,
           maxAddrInPartition: maxAddrInPartition,

package/src/Envio.gen.ts

@@ -11,11 +11,11 @@ import type {Logger as $$logger} from './Types.ts';
 
 import type {S_t as RescriptSchema_S_t} from 'rescript-schema/RescriptSchema.gen';
 
-import type {blockEvent as Internal_blockEvent} from './Internal.gen';
+export type blockEvent = { readonly number: number; readonly chainId: number };
 
-export type blockEvent = Internal_blockEvent;
+export type fuelBlockEvent = { readonly height: number; readonly chainId: number };
 
-export type onBlockArgs<context> = { readonly block: blockEvent; readonly context: context };
+export type onBlockArgs<block,context> = { readonly block: block; readonly context: context };
 
 export type onBlockOptions<chain> = {
   readonly name: string; 

package/src/Envio.res

@@ -2,12 +2,23 @@
 // Should be an entry point after we get rid of the generated project.
 // Don't forget to keep index.d.ts in sync with this file.
 
+// EVM block event with 'number' field for backward compatibility
 @genType
-type blockEvent = Internal.blockEvent
+type blockEvent = {
+  number: int,
+  chainId: int,
+}
+
+// Fuel block event with 'height' field
+@genType
+type fuelBlockEvent = {
+  height: int,
+  chainId: int,
+}
 
 @genType
-type onBlockArgs<'context> = {
-  block: blockEvent,
+type onBlockArgs<'block, 'context> = {
+  block: 'block,
   context: 'context,
 }
 

package/src/EventRegister.res

@@ -4,7 +4,7 @@ type registrations = {
 }
 
 type activeRegistration = {
-  ecosystem: Config.ecosystem,
+  platform: Platform.t,
   multichain: Config.multichain,
   preloadHandlers: bool,
   registrations: registrations,
@@ -36,9 +36,9 @@ let withRegistration = (fn: activeRegistration => unit) => {
   }
 }
 
-let startRegistration = (~ecosystem, ~multichain, ~preloadHandlers) => {
+let startRegistration = (~platform, ~multichain, ~preloadHandlers) => {
   let r = {
-    ecosystem,
+    platform,
     multichain,
     preloadHandlers,
     registrations: {
@@ -80,14 +80,6 @@ let onBlockOptionsSchema = S.schema(s =>
 
 let onBlock = (rawOptions: unknown, handler: Internal.onBlockArgs => promise<unit>) => {
   withRegistration(registration => {
-    // There's no big reason for this. It's just more work
-    switch registration.ecosystem {
-    | Evm => ()
-    | Fuel =>
-      Js.Exn.raiseError(
-        "Block Handlers are not supported for non-EVM ecosystems. Please reach out to the Envio team if you need this feature.",
-      )
-    }
     // We need to get timestamp for ordered multichain mode
     switch registration.multichain {
     | Unordered => ()

package/src/EventRegister.res.js

@@ -28,9 +28,9 @@ function withRegistration(fn) {
   }
 }
 
-function startRegistration(ecosystem, multichain, preloadHandlers) {
+function startRegistration(platform, multichain, preloadHandlers) {
   var r = {
-    ecosystem: ecosystem,
+    platform: platform,
     multichain: multichain,
     preloadHandlers: preloadHandlers,
     registrations: {
@@ -71,12 +71,8 @@ var onBlockOptionsSchema = S$RescriptSchema.schema(function (s) {
 
 function onBlock(rawOptions, handler) {
   withRegistration(function (registration) {
-        var match = registration.ecosystem;
-        if (match !== "evm") {
-          Js_exn.raiseError("Block Handlers are not supported for non-EVM ecosystems. Please reach out to the Envio team if you need this feature.");
-        }
-        var match$1 = registration.multichain;
-        if (match$1 === "ordered") {
+        var match = registration.multichain;
+        if (match === "ordered") {
           Js_exn.raiseError("Block Handlers are not supported for ordered multichain mode. Please reach out to the Envio team if you need this feature or enable unordered multichain mode with `unordered_multichain_mode: true` in your config.");
         }
         if (registration.preloadHandlers) {

package/src/EventRegister.resi

@@ -4,7 +4,7 @@ type registrations = {
 }
 
 let startRegistration: (
-  ~ecosystem: Config.ecosystem,
+  ~platform: Platform.t,
   ~multichain: Config.multichain,
   ~preloadHandlers: bool,
 ) => unit

package/src/InMemoryStore.res

@@ -22,9 +22,7 @@ module EntityTables = {
     switch self->Utils.Dict.dangerouslyGetNonOption(entityName) {
     | Some(table) =>
       table->(
-        Utils.magic: InMemoryTable.Entity.t<Internal.entity> => InMemoryTable.Entity.t<
-          entity,
-        >
+        Utils.magic: InMemoryTable.Entity.t<Internal.entity> => InMemoryTable.Entity.t<entity>
       )
 
     | None =>
@@ -53,13 +51,10 @@ type t = {
   rawEvents: InMemoryTable.t<rawEventsKey, InternalTable.RawEvents.t>,
   entities: dict<InMemoryTable.Entity.t<Internal.entity>>,
   effects: dict<effectCacheInMemTable>,
-  rollbackTargetCheckpointId: option<int>,
+  rollbackTargetCheckpointId: option<Internal.checkpointId>,
 }
 
-let make = (
-  ~entities: array<Internal.entityConfig>,
-  ~rollbackTargetCheckpointId=?,
-): t => {
+let make = (~entities: array<Internal.entityConfig>, ~rollbackTargetCheckpointId=?): t => {
   rawEvents: InMemoryTable.make(~hash=hashRawEventsKey),
   entities: EntityTables.make(entities),
   effects: Js.Dict.empty(),
@@ -105,9 +100,7 @@ let isRollingBack = (inMemoryStore: t) => inMemoryStore.rollbackTargetCheckpoint
 
 let setBatchDcs = (inMemoryStore: t, ~batch: Batch.t, ~shouldSaveHistory) => {
   let inMemTable =
-    inMemoryStore->getInMemTable(
-      ~entityConfig=InternalTable.DynamicContractRegistry.config,
-    )
+    inMemoryStore->getInMemTable(~entityConfig=InternalTable.DynamicContractRegistry.config)
 
   let itemIdx = ref(0)
 
@@ -140,11 +133,11 @@ let setBatchDcs = (inMemoryStore: t, ~batch: Batch.t, ~shouldSaveHistory) => {
           }
 
           inMemTable->InMemoryTable.Entity.set(
-            {
+            Set({
               entityId: entity.id,
               checkpointId,
-              entityUpdateAction: Set(entity->InternalTable.DynamicContractRegistry.castToInternal),
-            },
+              entity: entity->InternalTable.DynamicContractRegistry.castToInternal,
+            }),
             ~shouldSaveHistory,
           )
         }
@@ -154,4 +147,3 @@ let setBatchDcs = (inMemoryStore: t, ~batch: Batch.t, ~shouldSaveHistory) => {
     itemIdx := itemIdx.contents + checkpointEventsProcessed
   }
 }
-

package/src/InMemoryStore.res.js

@@ -137,11 +137,9 @@ function setBatchDcs(inMemoryStore, batch, shouldSaveHistory) {
             contract_name: entity_contract_name
           };
           InMemoryTable.Entity.set(inMemTable, {
+                type: "SET",
                 entityId: entity_id,
-                entityUpdateAction: {
-                  TAG: "Set",
-                  _0: entity
-                },
+                entity: entity,
                 checkpointId: checkpointId
               }, shouldSaveHistory, undefined);
         }

package/src/InMemoryTable.res

@@ -39,7 +39,8 @@ module Entity = {
   type indexFieldNameToIndices = t<TableIndices.Index.t, indicesSerializedToValue>
 
   type entityWithIndices<'entity> = {
-    entityRow: Internal.inMemoryStoreRowEntity<'entity>,
+    latest: option<'entity>,
+    status: Internal.inMemoryStoreEntityStatus<'entity>,
     entityIndices: Utils.Set.t<TableIndices.Index.t>,
   }
   type t<'entity> = {
@@ -147,18 +148,20 @@ module Entity = {
     //or if allowOverWriteEntity is true (used for mockDb in test helpers)
     if shouldWriteEntity {
       let entityIndices = Utils.Set.make()
-      let initialStoreRow: Internal.inMemoryStoreRowEntity<'entity> = switch entity {
+      switch entity {
       | Some(entity) =>
         //update table indices in the case where there
         //is an already set entity
         inMemTable->updateIndices(~entity, ~entityIndices)
-        InitialReadFromDb(AlreadySet(entity))
-
-      | None => InitialReadFromDb(NotSet)
+      | None => ()
       }
       inMemTable.table.dict->Js.Dict.set(
         key->inMemTable.table.hash,
-        {entityRow: initialStoreRow, entityIndices},
+        {
+          latest: entity,
+          status: Loaded,
+          entityIndices,
+        },
       )
     }
   }
@@ -166,56 +169,64 @@ module Entity = {
   let setRow = set
   let set = (
     inMemTable: t<'entity>,
-    entityUpdate: EntityHistory.entityUpdate<'entity>,
+    change: Change.t<'entity>,
     ~shouldSaveHistory,
     ~containsRollbackDiffChange=false,
   ) => {
     //New entity row with only the latest update
     @inline
-    let newEntityRow = () => Internal.Updated({
-      latest: entityUpdate,
-      history: shouldSaveHistory ? [entityUpdate] : [],
+    let newStatus = () => Internal.Updated({
+      latestChange: change,
+      history: shouldSaveHistory
+        ? [change]
+        : Utils.Array.immutableEmpty->(Utils.magic: array<unknown> => array<Change.t<'entity>>),
       containsRollbackDiffChange,
     })
+    let latest = switch change {
+    | Set({entity}) => Some(entity)
+    | Delete(_) => None
+    }
 
-    let {entityRow, entityIndices} = switch inMemTable.table->get(entityUpdate.entityId) {
-    | None => {entityRow: newEntityRow(), entityIndices: Utils.Set.make()}
-    | Some({entityRow: InitialReadFromDb(_), entityIndices}) => {
-        entityRow: newEntityRow(),
+    let updatedEntityRecord = switch inMemTable.table->get(change->Change.getEntityId) {
+    | None => {latest, status: newStatus(), entityIndices: Utils.Set.make()}
+    | Some({status: Loaded, entityIndices}) => {
+        latest,
+        status: newStatus(),
         entityIndices,
       }
-    | Some({entityRow: Updated(previous_values), entityIndices}) =>
-      let entityRow = Internal.Updated({
-        latest: entityUpdate,
+    | Some({status: Updated(previous_values), entityIndices}) =>
+      let newStatus = Internal.Updated({
+        latestChange: change,
         history: switch shouldSaveHistory {
         // This prevents two db actions in the same event on the same entity from being recorded to the history table.
-        | true if previous_values.latest.checkpointId === entityUpdate.checkpointId =>
+        | true
+          if previous_values.latestChange->Change.getCheckpointId ===
+            change->Change.getCheckpointId =>
           previous_values.history->Utils.Array.setIndexImmutable(
             previous_values.history->Array.length - 1,
-            entityUpdate,
+            change,
           )
-        | true => [...previous_values.history, entityUpdate]
+        | true => [...previous_values.history, change]
         | false => previous_values.history
         },
         containsRollbackDiffChange: previous_values.containsRollbackDiffChange,
       })
-      {entityRow, entityIndices}
+      {latest, status: newStatus, entityIndices}
     }
 
-    switch entityUpdate.entityUpdateAction {
-    | Set(entity) => inMemTable->updateIndices(~entity, ~entityIndices)
-    | Delete => inMemTable->deleteEntityFromIndices(~entityId=entityUpdate.entityId, ~entityIndices)
+    switch change {
+    | Set({entity}) =>
+      inMemTable->updateIndices(~entity, ~entityIndices=updatedEntityRecord.entityIndices)
+    | Delete({entityId}) =>
+      inMemTable->deleteEntityFromIndices(
+        ~entityId,
+        ~entityIndices=updatedEntityRecord.entityIndices,
+      )
     }
-    inMemTable.table->setRow(entityUpdate.entityId, {entityRow, entityIndices})
+    inMemTable.table->setRow(change->Change.getEntityId, updatedEntityRecord)
   }
 
-  let rowToEntity = row =>
-    switch row.entityRow {
-    | Internal.Updated({latest: {entityUpdateAction: Set(entity)}}) => Some(entity)
-    | Updated({latest: {entityUpdateAction: Delete}}) => None
-    | InitialReadFromDb(AlreadySet(entity)) => Some(entity)
-    | InitialReadFromDb(NotSet) => None
-    }
+  let rowToEntity = row => row.latest
 
   let getRow = get
 
@@ -330,10 +341,15 @@ module Entity = {
       }
     }
 
-  let rows = (inMemTable: t<'entity>) => {
+  let updates = (inMemTable: t<'entity>) => {
     inMemTable.table
     ->values
-    ->Array.map(v => v.entityRow)
+    ->Array.keepMapU(v =>
+      switch v.status {
+      | Updated(update) => Some(update)
+      | Loaded => None
+      }
+    )
   }
 
   let values = (inMemTable: t<'entity>) => {
@@ -353,4 +369,3 @@ module Entity = {
     },
   }
 }
-