envio 2.30.2 → 2.31.0-alpha.1

package/src/db/InternalTable.res

@@ -5,6 +5,72 @@ let isPrimaryKey = true
 let isNullable = true
 let isIndex = true
 
+module DynamicContractRegistry = {
+  let name = "dynamic_contract_registry"
+
+  let makeId = (~chainId, ~contractAddress) => {
+    chainId->Belt.Int.toString ++ "-" ++ contractAddress->Address.toString
+  }
+
+  // @genType Used for Test DB
+  @genType
+  type t = {
+    id: string,
+    @as("chain_id") chainId: int,
+    @as("registering_event_block_number") registeringEventBlockNumber: int,
+    @as("registering_event_log_index") registeringEventLogIndex: int,
+    @as("registering_event_block_timestamp") registeringEventBlockTimestamp: int,
+    @as("registering_event_contract_name") registeringEventContractName: string,
+    @as("registering_event_name") registeringEventName: string,
+    @as("registering_event_src_address") registeringEventSrcAddress: Address.t,
+    @as("contract_address") contractAddress: Address.t,
+    @as("contract_name") contractName: string,
+  }
+
+  let schema = S.schema(s => {
+    id: s.matches(S.string),
+    chainId: s.matches(S.int),
+    registeringEventBlockNumber: s.matches(S.int),
+    registeringEventLogIndex: s.matches(S.int),
+    registeringEventContractName: s.matches(S.string),
+    registeringEventName: s.matches(S.string),
+    registeringEventSrcAddress: s.matches(Address.schema),
+    registeringEventBlockTimestamp: s.matches(S.int),
+    contractAddress: s.matches(Address.schema),
+    contractName: s.matches(S.string),
+  })
+
+  let rowsSchema = S.array(schema)
+
+  let table = mkTable(
+    name,
+    ~fields=[
+      mkField("id", Text, ~isPrimaryKey, ~fieldSchema=S.string),
+      mkField("chain_id", Integer, ~fieldSchema=S.int),
+      mkField("registering_event_block_number", Integer, ~fieldSchema=S.int),
+      mkField("registering_event_log_index", Integer, ~fieldSchema=S.int),
+      mkField("registering_event_block_timestamp", Integer, ~fieldSchema=S.int),
+      mkField("registering_event_contract_name", Text, ~fieldSchema=S.string),
+      mkField("registering_event_name", Text, ~fieldSchema=S.string),
+      mkField("registering_event_src_address", Text, ~fieldSchema=Address.schema),
+      mkField("contract_address", Text, ~fieldSchema=Address.schema),
+      mkField("contract_name", Text, ~fieldSchema=S.string),
+    ],
+  )
+
+  let entityHistory = table->EntityHistory.fromTable(~schema)
+
+  external castToInternal: t => Internal.entity = "%identity"
+
+  let config = {
+    name,
+    schema,
+    rowsSchema,
+    table,
+    entityHistory,
+  }->Internal.fromGenericEntityConfig
+}
+
 module Chains = {
   type progressFields = [
     | #progress_block
@@ -16,6 +82,7 @@ module Chains = {
     | #id
     | #start_block
     | #end_block
+    | #max_reorg_depth
     | #source_block
     | #first_event_block
     | #buffer_block
@@ -28,6 +95,7 @@ module Chains = {
     #id,
     #start_block,
     #end_block,
+    #max_reorg_depth,
     #source_block,
     #first_event_block,
     #buffer_block,
@@ -52,6 +120,7 @@ module Chains = {
     @as("id") id: int,
     @as("start_block") startBlock: int,
     @as("end_block") endBlock: Js.null<int>,
+    @as("max_reorg_depth") maxReorgDepth: int,
     @as("progress_block") progressBlockNumber: int,
     @as("events_processed") numEventsProcessed: int,
     ...metaFields,
@@ -64,6 +133,7 @@ module Chains = {
       // Values populated from config
       mkField((#start_block: field :> string), Integer, ~fieldSchema=S.int),
       mkField((#end_block: field :> string), Integer, ~fieldSchema=S.null(S.int), ~isNullable),
+      mkField((#max_reorg_depth: field :> string), Integer, ~fieldSchema=S.int),
       // Block number of the latest block that was fetched from the source
       mkField((#buffer_block: field :> string), Integer, ~fieldSchema=S.int),
       // Block number of the currently active source
@@ -98,6 +168,7 @@ module Chains = {
       id: chainConfig.id,
       startBlock: chainConfig.startBlock,
       endBlock: chainConfig.endBlock->Js.Null.fromOption,
+      maxReorgDepth: chainConfig.maxReorgDepth,
       blockHeight: 0,
       firstEventBlockNumber: Js.Null.empty,
       latestFetchedBlockNumber: -1,
@@ -160,7 +231,51 @@ VALUES ${valuesRows->Js.Array2.joinWith(",\n       ")};`,
 
     `UPDATE "${pgSchema}"."${table.tableName}"
 SET ${setClauses->Js.Array2.joinWith(",\n    ")}
-WHERE "id" = $1;`
+WHERE "${(#id: field :> string)}" = $1;`
+  }
+
+  type rawInitialState = {
+    id: int,
+    startBlock: int,
+    endBlock: Js.Null.t<int>,
+    maxReorgDepth: int,
+    firstEventBlockNumber: Js.Null.t<int>,
+    timestampCaughtUpToHeadOrEndblock: Js.Null.t<Js.Date.t>,
+    numEventsProcessed: int,
+    progressBlockNumber: int,
+    dynamicContracts: array<Internal.indexingContract>,
+  }
+
+  // FIXME: Using registering_event_block_number for startBlock
+  // seems incorrect, since there might be a custom start block
+  // for the contract.
+  // TODO: Write a repro test where it might break something and fix
+  let makeGetInitialStateQuery = (~pgSchema) => {
+    `SELECT "${(#id: field :> string)}" as "id",
+"${(#start_block: field :> string)}" as "startBlock",
+"${(#end_block: field :> string)}" as "endBlock",
+"${(#max_reorg_depth: field :> string)}" as "maxReorgDepth",
+"${(#first_event_block: field :> string)}" as "firstEventBlockNumber",
+"${(#ready_at: field :> string)}" as "timestampCaughtUpToHeadOrEndblock",
+"${(#events_processed: field :> string)}" as "numEventsProcessed",
+"${(#progress_block: field :> string)}" as "progressBlockNumber",
+(
+  SELECT COALESCE(json_agg(json_build_object(
+    'address', "contract_address",
+    'contractName', "contract_name",
+    'startBlock', "registering_event_block_number",
+    'registrationBlock', "registering_event_block_number"
+  )), '[]'::json)
+  FROM "${pgSchema}"."${DynamicContractRegistry.table.tableName}"
+  WHERE "chain_id" = chains."${(#id: field :> string)}"
+) as "dynamicContracts"
+FROM "${pgSchema}"."${table.tableName}" as chains;`
+  }
+
+  let getInitialState = (sql, ~pgSchema) => {
+    sql
+    ->Postgres.unsafe(makeGetInitialStateQuery(~pgSchema))
+    ->(Utils.magic: promise<array<unknown>> => promise<array<rawInitialState>>)
   }
 
   let progressFields: array<progressFields> = [#progress_block, #events_processed]
@@ -182,7 +297,7 @@ WHERE "id" = $1;`
 
     let promises = []
 
-    chainsData->Utils.Dict.forEachWithKey((chainId, data) => {
+    chainsData->Utils.Dict.forEachWithKey((data, chainId) => {
       let params = []
 
       // Push id first (for WHERE clause)
@@ -201,7 +316,13 @@ WHERE "id" = $1;`
     Promise.all(promises)
   }
 
-  let setProgressedChains = (sql, ~pgSchema, ~progressedChains: array<Batch.progressedChain>) => {
+  type progressedChain = {
+    chainId: int,
+    progressBlockNumber: int,
+    totalEventsProcessed: int,
+  }
+
+  let setProgressedChains = (sql, ~pgSchema, ~progressedChains: array<progressedChain>) => {
     let query = makeProgressFieldsUpdateQuery(~pgSchema)
 
     let promises = []
@@ -254,21 +375,175 @@ module PersistedState = {
   )
 }
 
-module EndOfBlockRangeScannedData = {
+module Checkpoints = {
+  type field = [
+    | #id
+    | #chain_id
+    | #block_number
+    | #block_hash
+    | #events_processed
+  ]
+
   type t = {
-    chain_id: int,
-    block_number: int,
-    block_hash: string,
+    id: int,
+    @as("chain_id")
+    chainId: int,
+    @as("block_number")
+    blockNumber: int,
+    @as("block_hash")
+    blockHash: Js.null<string>,
+    @as("events_processed")
+    eventsProcessed: int,
   }
 
+  let initialCheckpointId = 0
+
   let table = mkTable(
-    "end_of_block_range_scanned_data",
+    "envio_checkpoints",
     ~fields=[
-      mkField("chain_id", Integer, ~fieldSchema=S.int, ~isPrimaryKey),
-      mkField("block_number", Integer, ~fieldSchema=S.int, ~isPrimaryKey),
-      mkField("block_hash", Text, ~fieldSchema=S.string),
+      mkField((#id: field :> string), Integer, ~fieldSchema=S.int, ~isPrimaryKey),
+      mkField((#chain_id: field :> string), Integer, ~fieldSchema=S.int),
+      mkField((#block_number: field :> string), Integer, ~fieldSchema=S.int),
+      mkField((#block_hash: field :> string), Text, ~fieldSchema=S.null(S.string), ~isNullable),
+      mkField((#events_processed: field :> string), Integer, ~fieldSchema=S.int),
     ],
   )
+
+  let makeGetReorgCheckpointsQuery = (~pgSchema): string => {
+    // Use CTE to pre-filter chains and compute safe_block once per chain
+    // This is faster because:
+    // 1. Chains table is small, so filtering it first is cheap
+    // 2. safe_block is computed once per chain, not per checkpoint
+    // 3. Query planner can materialize the small CTE result before joining
+    `WITH reorg_chains AS (
+  SELECT 
+    "${(#id: Chains.field :> string)}" as id,
+    "${(#source_block: Chains.field :> string)}" - "${(#max_reorg_depth: Chains.field :> string)}" AS safe_block
+  FROM "${pgSchema}"."${Chains.table.tableName}"
+  WHERE "${(#max_reorg_depth: Chains.field :> string)}" > 0
+    AND "${(#progress_block: Chains.field :> string)}" > "${(#source_block: Chains.field :> string)}" - "${(#max_reorg_depth: Chains.field :> string)}"
+)
+SELECT 
+  cp."${(#id: field :> string)}", 
+  cp."${(#chain_id: field :> string)}", 
+  cp."${(#block_number: field :> string)}", 
+  cp."${(#block_hash: field :> string)}"
+FROM "${pgSchema}"."${table.tableName}" cp
+INNER JOIN reorg_chains rc 
+  ON cp."${(#chain_id: field :> string)}" = rc.id
+WHERE cp."${(#block_hash: field :> string)}" IS NOT NULL
+  AND cp."${(#block_number: field :> string)}" >= rc.safe_block;` // Include safe_block checkpoint to use it for safe checkpoint tracking
+  }
+
+  let makeCommitedCheckpointIdQuery = (~pgSchema) => {
+    `SELECT COALESCE(MAX(${(#id: field :> string)}), ${initialCheckpointId->Belt.Int.toString}) AS id FROM "${pgSchema}"."${table.tableName}";`
+  }
+
+  let makeInsertCheckpointQuery = (~pgSchema) => {
+    `INSERT INTO "${pgSchema}"."${table.tableName}" ("${(#id: field :> string)}", "${(#chain_id: field :> string)}", "${(#block_number: field :> string)}", "${(#block_hash: field :> string)}", "${(#events_processed: field :> string)}")
+SELECT * FROM unnest($1::${(Integer :> string)}[],$2::${(Integer :> string)}[],$3::${(Integer :> string)}[],$4::${(Text :> string)}[],$5::${(Integer :> string)}[]);`
+  }
+
+  let insert = (
+    sql,
+    ~pgSchema,
+    ~checkpointIds,
+    ~checkpointChainIds,
+    ~checkpointBlockNumbers,
+    ~checkpointBlockHashes,
+    ~checkpointEventsProcessed,
+  ) => {
+    let query = makeInsertCheckpointQuery(~pgSchema)
+
+    sql
+    ->Postgres.preparedUnsafe(
+      query,
+      (
+        checkpointIds,
+        checkpointChainIds,
+        checkpointBlockNumbers,
+        checkpointBlockHashes,
+        checkpointEventsProcessed,
+      )->(
+        Utils.magic: (
+          (array<int>, array<int>, array<int>, array<Js.Null.t<string>>, array<int>)
+        ) => unknown
+      ),
+    )
+    ->Promise.ignoreValue
+  }
+
+  let rollback = (sql, ~pgSchema, ~rollbackTargetCheckpointId: int) => {
+    sql
+    ->Postgres.preparedUnsafe(
+      `DELETE FROM "${pgSchema}"."${table.tableName}" WHERE "${(#id: field :> string)}" > $1;`,
+      [rollbackTargetCheckpointId]->Utils.magic,
+    )
+    ->Promise.ignoreValue
+  }
+
+  let makePruneStaleCheckpointsQuery = (~pgSchema) => {
+    `DELETE FROM "${pgSchema}"."${table.tableName}" WHERE "${(#id: field :> string)}" < $1;`
+  }
+
+  let pruneStaleCheckpoints = (sql, ~pgSchema, ~safeCheckpointId: int) => {
+    sql
+    ->Postgres.preparedUnsafe(
+      makePruneStaleCheckpointsQuery(~pgSchema),
+      [safeCheckpointId]->Obj.magic,
+    )
+    ->Promise.ignoreValue
+  }
+
+  let makeGetRollbackTargetCheckpointQuery = (~pgSchema) => {
+    `SELECT "${(#id: field :> string)}" FROM "${pgSchema}"."${table.tableName}"
+WHERE 
+  "${(#chain_id: field :> string)}" = $1 AND
+  "${(#block_number: field :> string)}" <= $2
+ORDER BY "${(#id: field :> string)}" DESC
+LIMIT 1;`
+  }
+
+  let getRollbackTargetCheckpoint = (
+    sql,
+    ~pgSchema,
+    ~reorgChainId: int,
+    ~lastKnownValidBlockNumber: int,
+  ) => {
+    sql
+    ->Postgres.preparedUnsafe(
+      makeGetRollbackTargetCheckpointQuery(~pgSchema),
+      (reorgChainId, lastKnownValidBlockNumber)->Obj.magic,
+    )
+    ->(Utils.magic: promise<unknown> => promise<array<{"id": int}>>)
+  }
+
+  let makeGetRollbackProgressDiffQuery = (~pgSchema) => {
+    `SELECT 
+  "${(#chain_id: field :> string)}",
+  SUM("${(#events_processed: field :> string)}") as events_processed_diff,
+  MIN("${(#block_number: field :> string)}") - 1 as new_progress_block_number
+FROM "${pgSchema}"."${table.tableName}"
+WHERE "${(#id: field :> string)}" > $1
+GROUP BY "${(#chain_id: field :> string)}";`
+  }
+
+  let getRollbackProgressDiff = (sql, ~pgSchema, ~rollbackTargetCheckpointId: int) => {
+    sql
+    ->Postgres.preparedUnsafe(
+      makeGetRollbackProgressDiffQuery(~pgSchema),
+      [rollbackTargetCheckpointId]->Obj.magic,
+    )
+    ->(
+      Utils.magic: promise<unknown> => promise<
+        array<{
+          "chain_id": int,
+          "events_processed_diff": string,
+          "new_progress_block_number": int,
+        }>,
+      >
+    )
+  }
 }
 
 module RawEvents = {
@@ -363,69 +638,3 @@ module Views = {
      FROM "${pgSchema}"."${Chains.table.tableName}";`
   }
 }
-
-module DynamicContractRegistry = {
-  let name = "dynamic_contract_registry"
-
-  let makeId = (~chainId, ~contractAddress) => {
-    chainId->Belt.Int.toString ++ "-" ++ contractAddress->Address.toString
-  }
-
-  // @genType Used for Test DB
-  @genType
-  type t = {
-    id: string,
-    @as("chain_id") chainId: int,
-    @as("registering_event_block_number") registeringEventBlockNumber: int,
-    @as("registering_event_log_index") registeringEventLogIndex: int,
-    @as("registering_event_block_timestamp") registeringEventBlockTimestamp: int,
-    @as("registering_event_contract_name") registeringEventContractName: string,
-    @as("registering_event_name") registeringEventName: string,
-    @as("registering_event_src_address") registeringEventSrcAddress: Address.t,
-    @as("contract_address") contractAddress: Address.t,
-    @as("contract_name") contractName: string,
-  }
-
-  let schema = S.schema(s => {
-    id: s.matches(S.string),
-    chainId: s.matches(S.int),
-    registeringEventBlockNumber: s.matches(S.int),
-    registeringEventLogIndex: s.matches(S.int),
-    registeringEventContractName: s.matches(S.string),
-    registeringEventName: s.matches(S.string),
-    registeringEventSrcAddress: s.matches(Address.schema),
-    registeringEventBlockTimestamp: s.matches(S.int),
-    contractAddress: s.matches(Address.schema),
-    contractName: s.matches(S.string),
-  })
-
-  let rowsSchema = S.array(schema)
-
-  let table = mkTable(
-    name,
-    ~fields=[
-      mkField("id", Text, ~isPrimaryKey, ~fieldSchema=S.string),
-      mkField("chain_id", Integer, ~fieldSchema=S.int),
-      mkField("registering_event_block_number", Integer, ~fieldSchema=S.int),
-      mkField("registering_event_log_index", Integer, ~fieldSchema=S.int),
-      mkField("registering_event_block_timestamp", Integer, ~fieldSchema=S.int),
-      mkField("registering_event_contract_name", Text, ~fieldSchema=S.string),
-      mkField("registering_event_name", Text, ~fieldSchema=S.string),
-      mkField("registering_event_src_address", Text, ~fieldSchema=Address.schema),
-      mkField("contract_address", Text, ~fieldSchema=Address.schema),
-      mkField("contract_name", Text, ~fieldSchema=S.string),
-    ],
-  )
-
-  let entityHistory = table->EntityHistory.fromTable(~schema)
-
-  external castToInternal: t => Internal.entity = "%identity"
-
-  let config = {
-    name,
-    schema,
-    rowsSchema,
-    table,
-    entityHistory,
-  }->Internal.fromGenericEntityConfig
-}