envio 2.28.0-alpha.3 → 2.28.0

package/src/Throttler.res.js

@@ -2,6 +2,7 @@
 'use strict';
 
 var Pino = require("./bindings/Pino.res.js");
+var Utils = require("./Utils.res.js");
 var Caml_js_exceptions = require("rescript/lib/js/caml_js_exceptions.js");
 
 function make(intervalMillis, logger) {
@@ -38,7 +39,7 @@ async function startInternal(throttler) {
     }
     catch (raw_exn){
       var exn = Caml_js_exceptions.internalToOCamlException(raw_exn);
-      throttler.logger.error(Pino.createPinoMessageWithError("Scheduled action failed in throttler", exn));
+      throttler.logger.error(Pino.createPinoMessageWithError("Scheduled action failed in throttler", Utils.prettifyExn(exn)));
     }
     throttler.isRunning = false;
     return await startInternal(throttler);

package/src/Utils.res

@@ -130,6 +130,13 @@ module Dict = {
   let shallowCopy: dict<'a> => dict<'a> = %raw(`(dict) => ({...dict})`)
 
   let size = dict => dict->Js.Dict.keys->Js.Array2.length
+
+  @set_index
+  external setByInt: (dict<'a>, int, 'a) => unit = ""
+
+  let incrementByInt: (dict<int>, int) => unit = %raw(`(dict, key) => {
+    dict[key]++
+  }`)
 }
 
 module Math = {

package/src/Utils.res.js

@@ -130,6 +130,10 @@ function size(dict) {
   return Object.keys(dict).length;
 }
 
+var incrementByInt = ((dict, key) => {
+    dict[key]++
+  });
+
 var Dict = {
   has: has,
   push: push,
@@ -141,7 +145,8 @@ var Dict = {
   deleteInPlace: deleteInPlace,
   updateImmutable: updateImmutable,
   shallowCopy: shallowCopy,
-  size: size
+  size: size,
+  incrementByInt: incrementByInt
 };
 
 function minOptInt(a, b) {

package/src/bindings/Postgres.res

@@ -78,7 +78,7 @@ type poolConfig = {
   // types?: array<'a>, // Array of custom types, see more below (default: [])
   onnotice?: string => unit, // Default console.log, set false to silence NOTICE (default: fn)
   onParameter?: (string, string) => unit, // (key, value) when server param change (default: fn)
-  debug?: 'a. 'a => unit, //(connection, query, params, types) => unit, // Is called with (connection, query, params, types) (default: fn)
+  debug?: (~connection: unknown, ~query: unknown, ~params: unknown, ~types: unknown) => unit, // Is called with (connection, query, params, types)
   socket?: unit => unit, // fn returning custom socket to use (default: fn)
   transform?: transformConfig,
   connection?: connectionConfig,

package/src/db/InternalTable.gen.ts

@@ -7,13 +7,6 @@ import type {Json_t as Js_Json_t} from '../../src/Js.shim';
 
 import type {t as Address_t} from '../../src/Address.gen';
 
-export type EventSyncState_t = {
-  readonly chain_id: number; 
-  readonly block_number: number; 
-  readonly log_index: number; 
-  readonly block_timestamp: number
-};
-
 export type RawEvents_t = {
   readonly chain_id: number; 
   readonly event_id: bigint; 

package/src/db/InternalTable.res

@@ -5,51 +5,15 @@ let isPrimaryKey = true
 let isNullable = true
 let isIndex = true
 
-module EventSyncState = {
-  //Used unsafely in DbFunctions.res so just enforcing the naming here
-  let blockTimestampFieldName = "block_timestamp"
-  let blockNumberFieldName = "block_number"
-  let logIndexFieldName = "log_index"
-  let isPreRegisteringDynamicContractsFieldName = "is_pre_registering_dynamic_contracts"
-
-  // @genType Used for Test DB
-  @genType
-  type t = {
-    @as("chain_id") chainId: int,
-    @as("block_number") blockNumber: int,
-    @as("log_index") logIndex: int,
-    @as("block_timestamp") blockTimestamp: int,
-  }
-
-  let table = mkTable(
-    "event_sync_state",
-    ~fields=[
-      mkField("chain_id", Integer, ~fieldSchema=S.int, ~isPrimaryKey),
-      mkField(blockNumberFieldName, Integer, ~fieldSchema=S.int),
-      mkField(logIndexFieldName, Integer, ~fieldSchema=S.int),
-      mkField(blockTimestampFieldName, Integer, ~fieldSchema=S.int),
-      // Keep it in case Hosted Service relies on it to prevent a breaking changes
-      mkField(
-        isPreRegisteringDynamicContractsFieldName,
-        Boolean,
-        ~default="false",
-        ~fieldSchema=S.bool,
-      ),
-    ],
-  )
-
-  //We need to update values here not delet the rows, since restarting without a row
-  //has a different behaviour to restarting with an initialised row with zero values
-  let resetCurrentCurrentSyncStateQuery = (~pgSchema) =>
-    `UPDATE "${pgSchema}"."${table.tableName}"
-    SET ${blockNumberFieldName} = 0, 
-        ${logIndexFieldName} = 0, 
-        ${blockTimestampFieldName} = 0, 
-        ${isPreRegisteringDynamicContractsFieldName} = false;`
-}
-
 module Chains = {
+  type progressFields = [
+    | #progress_block
+    | #_progress_log_index
+    | #events_processed
+  ]
+
   type field = [
+    | progressFields
     | #id
     | #start_block
     | #end_block
@@ -57,10 +21,8 @@ module Chains = {
     | #first_event_block
     | #buffer_block
     | #ready_at
-    | #events_processed
-    | #_is_hyper_sync
-    | #_latest_processed_block
     | #_num_batches_fetched
+    | #_is_hyper_sync
   ]
 
   let fields: array<field> = [
@@ -70,28 +32,34 @@ module Chains = {
     #source_block,
     #first_event_block,
     #buffer_block,
+    #progress_block,
     #ready_at,
     #events_processed,
+    #_progress_log_index,
     #_is_hyper_sync,
-    #_latest_processed_block,
     #_num_batches_fetched,
   ]
 
-  type t = {
-    @as("id") id: int,
-    @as("start_block") startBlock: int,
-    @as("end_block") endBlock: Js.null<int>,
-    @as("source_block") blockHeight: int,
+  type metaFields = {
     @as("first_event_block") firstEventBlockNumber: Js.null<int>,
     @as("buffer_block") latestFetchedBlockNumber: int,
+    @as("source_block") blockHeight: int,
     @as("ready_at")
     timestampCaughtUpToHeadOrEndblock: Js.null<Js.Date.t>,
-    @as("events_processed") numEventsProcessed: int,
-    @as("_latest_processed_block") latestProcessedBlock: Js.null<int>,
     @as("_is_hyper_sync") isHyperSync: bool,
     @as("_num_batches_fetched") numBatchesFetched: int,
   }
 
+  type t = {
+    @as("id") id: int,
+    @as("start_block") startBlock: int,
+    @as("end_block") endBlock: Js.null<int>,
+    @as("progress_block") progressBlockNumber: int,
+    @as("_progress_log_index") progressNextBlockLogIndex: Js.null<int>,
+    @as("events_processed") numEventsProcessed: int,
+    ...metaFields,
+  }
+
   let table = mkTable(
     "envio_chains",
     ~fields=[
@@ -118,15 +86,20 @@ module Chains = {
         ~fieldSchema=S.null(Utils.Schema.dbDate),
         ~isNullable,
       ),
-      mkField((#events_processed: field :> string), Integer, ~fieldSchema=S.int), // TODO: In the future it should reference a table with sources
+      mkField((#events_processed: field :> string), Integer, ~fieldSchema=S.int),
+      // TODO: In the future it should reference a table with sources
       mkField((#_is_hyper_sync: field :> string), Boolean, ~fieldSchema=S.bool),
-      // TODO: Make the data more public facing
+      // Fully processed block number
+      mkField((#progress_block: field :> string), Integer, ~fieldSchema=S.int),
+      // Optional log index of the next block after progress block
+      // To correctly resume indexing when we processed half of the block.
       mkField(
-        (#_latest_processed_block: field :> string),
+        (#_progress_log_index: field :> string),
         Integer,
         ~fieldSchema=S.null(S.int),
         ~isNullable,
       ),
+      // TODO: Should deprecate after changing the ETA calculation logic
       mkField((#_num_batches_fetched: field :> string), Integer, ~fieldSchema=S.int),
     ],
   )
@@ -140,7 +113,8 @@ module Chains = {
       firstEventBlockNumber: Js.Null.empty,
       latestFetchedBlockNumber: -1,
       timestampCaughtUpToHeadOrEndblock: Js.Null.empty,
-      latestProcessedBlock: Js.Null.empty,
+      progressBlockNumber: -1,
+      progressNextBlockLogIndex: Js.Null.empty,
       isHyperSync: false,
       numEventsProcessed: 0,
       numBatchesFetched: 0,
@@ -178,21 +152,19 @@ VALUES ${valuesRows->Js.Array2.joinWith(",\n       ")};`,
     }
   }
 
-  // Fields that should be updated on conflict (excluding static config fields)
-  let updateFields: array<field> = [
+  // Fields that can be updated outside of the batch transaction
+  let metaFields: array<field> = [
     #source_block,
-    #first_event_block,
     #buffer_block,
+    #first_event_block,
     #ready_at,
-    #events_processed,
     #_is_hyper_sync,
-    #_latest_processed_block,
     #_num_batches_fetched,
   ]
 
-  let makeSingleUpdateQuery = (~pgSchema) => {
+  let makeMetaFieldsUpdateQuery = (~pgSchema) => {
     // Generate SET clauses with parameter placeholders
-    let setClauses = Belt.Array.mapWithIndex(updateFields, (index, field) => {
+    let setClauses = Belt.Array.mapWithIndex(metaFields, (index, field) => {
       let fieldName = (field :> string)
       let paramIndex = index + 2 // +2 because $1 is for id in WHERE clause
       `"${fieldName}" = $${Belt.Int.toString(paramIndex)}`
@@ -203,27 +175,80 @@ SET ${setClauses->Js.Array2.joinWith(",\n    ")}
 WHERE "id" = $1;`
   }
 
-  let setValues = (sql, ~pgSchema, ~chainsData: array<t>) => {
-    let query = makeSingleUpdateQuery(~pgSchema)
+  let progressFields: array<progressFields> = [
+    #progress_block,
+    #_progress_log_index,
+    #events_processed,
+  ]
+
+  let makeProgressFieldsUpdateQuery = (~pgSchema) => {
+    let setClauses = Belt.Array.mapWithIndex(progressFields, (index, field) => {
+      let fieldName = (field :> string)
+      let paramIndex = index + 2 // +2 because $1 is for id in WHERE clause
+      `"${fieldName}" = $${Belt.Int.toString(paramIndex)}`
+    })
+
+    `UPDATE "${pgSchema}"."${table.tableName}"
+SET ${setClauses->Js.Array2.joinWith(",\n    ")}
+WHERE "id" = $1;`
+  }
+
+  let setMeta = (sql, ~pgSchema, ~chainsData: dict<metaFields>) => {
+    let query = makeMetaFieldsUpdateQuery(~pgSchema)
+
+    let promises = []
 
-    let promises = chainsData->Belt.Array.map(chain => {
+    chainsData->Utils.Dict.forEachWithKey((chainId, data) => {
       let params = []
 
       // Push id first (for WHERE clause)
-      let idValue = chain->(Utils.magic: t => dict<unknown>)->Js.Dict.get("id")
-      params->Js.Array2.push(idValue)->ignore
+      params->Js.Array2.push(chainId->(Utils.magic: string => unknown))->ignore
 
       // Then push all updateable field values (for SET clause)
-      updateFields->Js.Array2.forEach(field => {
-        let value = chain->(Utils.magic: t => dict<unknown>)->Js.Dict.get((field :> string))
+      metaFields->Js.Array2.forEach(field => {
+        let value =
+          data->(Utils.magic: metaFields => dict<unknown>)->Js.Dict.unsafeGet((field :> string))
         params->Js.Array2.push(value)->ignore
       })
 
-      sql->Postgres.preparedUnsafe(query, params->Obj.magic)
+      promises->Js.Array2.push(sql->Postgres.preparedUnsafe(query, params->Obj.magic))->ignore
     })
 
     Promise.all(promises)
   }
+
+  let setProgressedChains = (sql, ~pgSchema, ~progressedChains: array<Batch.progressedChain>) => {
+    let query = makeProgressFieldsUpdateQuery(~pgSchema)
+
+    let promises = []
+
+    progressedChains->Js.Array2.forEach(data => {
+      let params = []
+
+      // Push id first (for WHERE clause)
+      params->Js.Array2.push(data.chainId->(Utils.magic: int => unknown))->ignore
+
+      // Then push all updateable field values (for SET clause)
+      progressFields->Js.Array2.forEach(field => {
+        params
+        ->Js.Array2.push(
+          switch field {
+          | #progress_block => data.progressBlockNumber->(Utils.magic: int => unknown)
+          | #_progress_log_index =>
+            data.progressNextBlockLogIndex
+            ->Js.Null.fromOption
+            ->(Utils.magic: Js.null<int> => unknown)
+          | #events_processed => data.totalEventsProcessed->(Utils.magic: int => unknown)
+          },
+        )
+        ->ignore
+      })
+
+      promises->Js.Array2.push(sql->Postgres.preparedUnsafe(query, params->Obj.magic))->ignore
+    })
+
+    Promise.all(promises)->Promise.ignoreValue
+  }
 }
 
 module PersistedState = {
@@ -336,10 +361,12 @@ module Views = {
        "${(#id: Chains.field :> string)}" AS "chainId",
        "${(#start_block: Chains.field :> string)}" AS "startBlock", 
        "${(#end_block: Chains.field :> string)}" AS "endBlock",
+       "${(#progress_block: Chains.field :> string)}" AS "progressBlock",
        "${(#buffer_block: Chains.field :> string)}" AS "bufferBlock",
-       "${(#ready_at: Chains.field :> string)}" AS "readyAt",
        "${(#first_event_block: Chains.field :> string)}" AS "firstEventBlock",
        "${(#events_processed: Chains.field :> string)}" AS "eventsProcessed",
+       "${(#source_block: Chains.field :> string)}" AS "sourceBlock",
+       "${(#ready_at: Chains.field :> string)}" AS "readyAt",
        ("${(#ready_at: Chains.field :> string)}" IS NOT NULL) AS "isReady"
      FROM "${pgSchema}"."${Chains.table.tableName}"
      ORDER BY "${(#id: Chains.field :> string)}";`
@@ -354,7 +381,7 @@ module Views = {
        "${(#first_event_block: Chains.field :> string)}" AS "first_event_block_number",
        "${(#_is_hyper_sync: Chains.field :> string)}" AS "is_hyper_sync",
        "${(#buffer_block: Chains.field :> string)}" AS "latest_fetched_block_number",
-       "${(#_latest_processed_block: Chains.field :> string)}" AS "latest_processed_block",
+       "${(#progress_block: Chains.field :> string)}" AS "latest_processed_block",
        "${(#_num_batches_fetched: Chains.field :> string)}" AS "num_batches_fetched",
        "${(#events_processed: Chains.field :> string)}" AS "num_events_processed",
        "${(#start_block: Chains.field :> string)}" AS "start_block",
@@ -427,4 +454,42 @@ module DynamicContractRegistry = {
     table,
     entityHistory,
   }->Internal.fromGenericEntityConfig
+
+  let makeCleanUpOnRestartQuery = (~pgSchema, ~chains: array<Chains.t>) => {
+    let query = ref(``)
+
+    chains->Js.Array2.forEach(chain => {
+      if query.contents !== "" {
+        query := query.contents ++ "\n"
+      }
+      query :=
+        query.contents ++
+        `DELETE FROM "${pgSchema}"."${table.tableName}"
+WHERE chain_id = ${chain.id->Belt.Int.toString}${switch chain {
+          | {progressBlockNumber, progressNextBlockLogIndex: Value(progressNextBlockLogIndex)} =>
+            ` AND (
+  registering_event_block_number > ${(progressBlockNumber + 1)->Belt.Int.toString}
+  OR registering_event_block_number = ${(progressBlockNumber + 1)->Belt.Int.toString}
+  AND registering_event_log_index > ${progressNextBlockLogIndex->Belt.Int.toString}
+)`
+          | {progressBlockNumber: -1, progressNextBlockLogIndex: Null} => ``
+          | {progressBlockNumber, progressNextBlockLogIndex: Null} =>
+            ` AND registering_event_block_number > ${progressBlockNumber->Belt.Int.toString}`
+          }};
+DELETE FROM "${pgSchema}"."${table.tableName}_history"
+WHERE entity_history_chain_id = ${chain.id->Belt.Int.toString}${switch chain {
+          | {progressBlockNumber, progressNextBlockLogIndex: Value(progressNextBlockLogIndex)} =>
+            ` AND (
+  entity_history_block_number > ${(progressBlockNumber + 1)->Belt.Int.toString}
+  OR entity_history_block_number = ${(progressBlockNumber + 1)->Belt.Int.toString}
+  AND entity_history_log_index > ${progressNextBlockLogIndex->Belt.Int.toString}
+)`
+          | {progressBlockNumber: -1, progressNextBlockLogIndex: Null} => ``
+          | {progressBlockNumber, progressNextBlockLogIndex: Null} =>
+            ` AND entity_history_block_number > ${progressBlockNumber->Belt.Int.toString}`
+          }};`
+    })
+
+    query.contents
+  }
 }