envio 3.11.0 → 3.12.0

package/src/db/InternalTable.res

@@ -170,6 +170,7 @@ FROM "${pgSchema}"."${table.tableName}";`
 module Chains = {
   type progressFields = [
     | #progress_block
+    | #progress_block_time
     | #events_processed
     | #source_block
   ]
@@ -186,6 +187,7 @@ module Chains = {
     | #buffer_block
     | #ready_at
     | #_is_hyper_sync
+    | #checkpoint_id
   ]
 
   let fields: array<field> = [
@@ -198,19 +200,16 @@ module Chains = {
     #first_event_block,
     #buffer_block,
     #progress_block,
+    #progress_block_time,
     #ready_at,
     #events_processed,
     #_is_hyper_sync,
+    #checkpoint_id,
   ]
 
   type metaFields = {
     @as("first_event_block")
-    firstEventBlockNumber: Null.t<
-      // Push id first (for WHERE clause)
-
-      // Then push all updateable field values (for SET clause)
-      int,
-    >,
+    firstEventBlockNumber: Null.t<int>,
     @as("buffer_block") latestFetchedBlockNumber: int,
     @as("ready_at")
     timestampCaughtUpToHeadOrEndblock: Null.t<Date.t>,
@@ -225,7 +224,9 @@ module Chains = {
     @as("max_reorg_depth") maxReorgDepth: int,
     @as("source_block") blockHeight: int,
     @as("progress_block") progressBlockNumber: int,
+    @as("progress_block_time") progressBlockTime: Null.t<Date.t>,
     @as("events_processed") numEventsProcessed: float,
+    @as("checkpoint_id") checkpointId: Internal.checkpointId,
     ...metaFields,
   }
 
@@ -265,6 +266,22 @@ module Chains = {
       mkField((#_is_hyper_sync: field :> string), Boolean, ~fieldSchema=S.bool),
       // Fully processed block number
       mkField((#progress_block: field :> string), Int32, ~fieldSchema=S.int),
+      // When the block in progress_block was produced, so that `now() -
+      // progress_block_time` is how far behind chain time the indexer is. Null
+      // while the source hasn't reported that block's header.
+      mkField(
+        (#progress_block_time: field :> string),
+        Date,
+        ~fieldSchema=S.null(Utils.Schema.dbDate),
+        ~isNullable,
+      ),
+      // The last checkpoint id the chain has committed. Kept here rather than
+      // read off the checkpoints table: checkpoint rows are only written while
+      // a rollback could reach them, but an append-only sink holds a row for
+      // every id ever handed out and resolves current state through them, so a
+      // resume has to continue the sequence even where no checkpoint row backs
+      // it.
+      mkField((#checkpoint_id: field :> string), UInt64, ~fieldSchema=S.bigint),
     ],
   )
 
@@ -280,8 +297,10 @@ module Chains = {
       latestFetchedBlockNumber: -1,
       timestampCaughtUpToHeadOrEndblock: Null.null,
       progressBlockNumber: -1,
+      progressBlockTime: Null.null,
       isHyperSync: false,
       numEventsProcessed: 0.,
+      checkpointId: Internal.initialCheckpointId,
     }
   }
 
@@ -358,8 +377,11 @@ WHERE "${(#id: field :> string)}" = $2
     timestampCaughtUpToHeadOrEndblock: Null.t<Date.t>,
     numEventsProcessed: float,
     progressBlockNumber: int,
+    progressBlockTime: Null.t<Date.t>,
     addressRows: AddressRows.seedRows,
     sourceBlockNumber: int,
+    // BIGINT, which the driver hands back as a string.
+    checkpointId: string,
   }
 
   let makeGetInitialStateQuery = (~pgSchema) => {
@@ -371,7 +393,9 @@ WHERE "${(#id: field :> string)}" = $2
 "${(#ready_at: field :> string)}" as "timestampCaughtUpToHeadOrEndblock",
 "${(#events_processed: field :> string)}"::float8 as "numEventsProcessed",
 "${(#progress_block: field :> string)}" as "progressBlockNumber",
-"${(#source_block: field :> string)}" as "sourceBlockNumber"
+"${(#progress_block_time: field :> string)}" as "progressBlockTime",
+"${(#source_block: field :> string)}" as "sourceBlockNumber",
+"${(#checkpoint_id: field :> string)}"::TEXT as "checkpointId"
 FROM "${pgSchema}"."${table.tableName}";`
   }
 
@@ -403,7 +427,20 @@ FROM "${pgSchema}"."${table.tableName}";`
     })
   }
 
-  let progressFields: array<progressFields> = [#progress_block, #events_processed, #source_block]
+  // Block timestamps are unix seconds everywhere inside the indexer - that's
+  // what a block header carries - and a timestamp in storage, where `now() -
+  // progress_block_time` is the query the column exists for.
+  let blockTimeToDb = (blockTime: option<int>) =>
+    blockTime->Option.map(seconds => Date.fromTime(seconds->Int.toFloat *. 1000.))->Null.fromOption
+  let blockTimeFromDb = (date: Null.t<Date.t>) =>
+    date->Null.toOption->Option.map(date => (date->Date.getTime /. 1000.)->Float.toInt)
+
+  let progressFields: array<progressFields> = [
+    #progress_block,
+    #progress_block_time,
+    #events_processed,
+    #source_block,
+  ]
 
   let makeProgressFieldsUpdateQuery = (~pgSchema) => {
     let setClauses = Array.mapWithIndex(progressFields, (field, index) => {
@@ -444,6 +481,7 @@ WHERE "id" = $1;`
   type progressedChain = {
     chainId: ChainId.t,
     progressBlockNumber: int,
+    progressBlockTime: option<int>,
     sourceBlockNumber: int,
     totalEventsProcessed: float,
   }
@@ -463,6 +501,8 @@ WHERE "id" = $1;`
         ->Array.push(
           switch field {
           | #progress_block => data.progressBlockNumber->(Utils.magic: int => unknown)
+          | #progress_block_time =>
+            data.progressBlockTime->blockTimeToDb->(Utils.magic: Null.t<Date.t> => unknown)
           | #events_processed => data.totalEventsProcessed->(Utils.magic: float => unknown)
           | #source_block => data.sourceBlockNumber->(Utils.magic: int => unknown)
           },
@@ -475,6 +515,31 @@ WHERE "id" = $1;`
 
     Promise.all(promises)->Utils.Promise.ignoreValue
   }
+
+  let makeSetCheckpointFrontierQuery = (~pgSchema, ~chainIdMode: ChainId.mode=Int32) => {
+    let chainIdArrayType = chainIdArrayType(~pgSchema, ~chainIdMode)
+    `UPDATE "${pgSchema}"."${table.tableName}"
+SET "${(#checkpoint_id: field :> string)}" = envio_frontier.checkpoint_id
+FROM unnest($1::${chainIdArrayType},$2::${(BigInt: Postgres.columnType :> string)}[]) AS envio_frontier(chain_id, checkpoint_id)
+WHERE "${table.tableName}"."${(#id: field :> string)}" = envio_frontier.chain_id;`
+  }
+
+  // The chains the write moved, in one statement and in the batch's own
+  // transaction — so a chain's stored id can never outlive the rows it covers,
+  // nor lag behind them.
+  let setCheckpointFrontier = (
+    sql,
+    ~pgSchema,
+    ~frontier: Frontier.t,
+    ~chainIdMode: ChainId.mode=Int32,
+  ) => {
+    sql
+    ->Postgres.preparedUnsafe(
+      makeSetCheckpointFrontierQuery(~pgSchema, ~chainIdMode),
+      frontier->Frontier.unnestParams->(Utils.magic: Frontier.unnestParams => unknown),
+    )
+    ->Utils.Promise.ignoreValue
+  }
 }
 
 module EnvioInfo = {
@@ -556,31 +621,58 @@ module Checkpoints = {
 
   let initialCheckpointId = 0n
 
+  // The checkpoint a rollback's diff rows are stamped with. It never reaches
+  // Postgres — there the diff is written straight to the entity table — but an
+  // append-only sink resolves current state through the checkpoints, so without
+  // one of these the diff sits above the frontier while the rows it supersedes
+  // sit below it, and the orphaned values are what a reader sees.
+  type diffCheckpoint = {
+    chainId: ChainId.t,
+    checkpointId: Internal.checkpointId,
+    // Where the rollback left the chain. At or below its stored progress, so a
+    // resume counts the row as covered rather than as something to trim back to.
+    blockNumber: int,
+  }
+
   // One definition per column, carrying what each storage needs: the field
   // itself, the type ClickHouse gives it where that differs from Postgres, and
-  // where a batch keeps the column's values.
+  // where a batch — or a rollback diff — keeps the column's values.
   type column = {
     field: fieldOrDerived,
     clickHouseFieldType: fieldType,
     valuesOf: Batch.t => array<unknown>,
+    diffValuesOf: array<diffCheckpoint> => array<unknown>,
   }
 
+  // The chain leads the key: ids are only unique within a chain, and every
+  // query that narrows to one chain reads a contiguous run of it.
   let columns: array<column> = [
     {
-      field: mkField((#id: field :> string), UInt64, ~fieldSchema=S.bigint, ~isPrimaryKey),
-      clickHouseFieldType: UInt64,
-      valuesOf: batch => batch.checkpointIds->(Utils.magic: array<bigint> => array<unknown>),
-    },
-    {
-      field: mkField((#chain_id: field :> string), ChainId, ~fieldSchema=ChainId.schema),
+      field: mkField(
+        (#chain_id: field :> string),
+        ChainId,
+        ~fieldSchema=ChainId.schema,
+        ~isPrimaryKey,
+      ),
       clickHouseFieldType: ChainId,
       valuesOf: batch =>
         batch.checkpointChainIds->(Utils.magic: array<ChainId.t> => array<unknown>),
+      diffValuesOf: diffs =>
+        diffs->Array.map(diff => diff.chainId->(Utils.magic: ChainId.t => unknown)),
+    },
+    {
+      field: mkField((#id: field :> string), UInt64, ~fieldSchema=S.bigint, ~isPrimaryKey),
+      clickHouseFieldType: UInt64,
+      valuesOf: batch => batch.checkpointIds->(Utils.magic: array<bigint> => array<unknown>),
+      diffValuesOf: diffs =>
+        diffs->Array.map(diff => diff.checkpointId->(Utils.magic: bigint => unknown)),
     },
     {
       field: mkField((#block_number: field :> string), Int32, ~fieldSchema=S.int),
       clickHouseFieldType: Int32,
       valuesOf: batch => batch.checkpointBlockNumbers->(Utils.magic: array<int> => array<unknown>),
+      diffValuesOf: diffs =>
+        diffs->Array.map(diff => diff.blockNumber->(Utils.magic: int => unknown)),
     },
     {
       field: mkField(
@@ -592,6 +684,8 @@ module Checkpoints = {
       clickHouseFieldType: String,
       valuesOf: batch =>
         batch.checkpointBlockHashes->(Utils.magic: array<Null.t<string>> => array<unknown>),
+      diffValuesOf: diffs =>
+        diffs->Array.map(_ => Null.Null->(Utils.magic: Null.t<string> => unknown)),
     },
     {
       field: mkField((#events_processed: field :> string), Int32, ~fieldSchema=S.int),
@@ -600,10 +694,34 @@ module Checkpoints = {
       clickHouseFieldType: UInt64,
       valuesOf: batch =>
         batch.checkpointEventsProcessed->(Utils.magic: array<int> => array<unknown>),
+      diffValuesOf: diffs => diffs->Array.map(_ => 0->(Utils.magic: int => unknown)),
     },
   ]
 
-  let table = mkTable("envio_checkpoints", ~fields=columns->Array.map(({field}) => field))
+  let tableName = "envio_checkpoints"
+
+  let table = mkTable(tableName, ~fields=columns->Array.map(({field}) => field))
+
+  // Where each chain counts its own ids the chain has to be part of the key,
+  // and every bound a rollback or a prune applies names it. Under one shared
+  // sequence the id is unique by itself and those bounds are id ranges with no
+  // chain in them — which a key led by the chain can't serve.
+  let globalTable = mkTable(
+    tableName,
+    ~fields=columns->Array.map(({field: column}) =>
+      switch column {
+      | Table.Field(f) if f.fieldName === (#chain_id: field :> string) =>
+        Table.Field({...f, isPrimaryKey: false})
+      | column => column
+      }
+    ),
+  )
+
+  let tableFor = (sequence: CheckpointSequence.t) =>
+    switch sequence {
+    | SharedAcrossChains => globalTable
+    | PerChain => table
+    }
 
   let makeGetReorgCheckpointsQuery = (~pgSchema): string => {
     // The safe_block checkpoint itself is included, so it can be used for safe
@@ -640,10 +758,6 @@ WHERE cp."${(#block_hash: field :> string)}" IS NOT NULL
 ORDER BY cp."${(#id: field :> string)}";`
   }
 
-  let makeCommitedCheckpointIdQuery = (~pgSchema) => {
-    `SELECT COALESCE(MAX(${(#id: field :> string)}), ${initialCheckpointId->BigInt.toString}) AS id FROM "${pgSchema}"."${table.tableName}";`
-  }
-
   let makeInsertCheckpointQuery = (~pgSchema, ~chainIdMode: ChainId.mode=Int32) => {
     let chainIdArrayType = chainIdArrayType(~pgSchema, ~chainIdMode)
     `INSERT INTO "${pgSchema}"."${table.tableName}" ("${(#id: field :> string)}", "${(#chain_id: field :> string)}", "${(#block_number: field :> string)}", "${(#block_hash: field :> string)}", "${(#events_processed: field :> string)}")
@@ -687,18 +801,21 @@ SELECT * FROM unnest($1::${(BigInt: Postgres.columnType :> string)}[],$2::${chai
 
   let rollback = (sql, ~pgSchema, ~floors: RollbackFloors.t) => {
     let tableRef = `"${pgSchema}"."${table.tableName}"`
-    let bounds = floors.floors->CheckpointBounds.sql(~chainIdColumn, ~tableRef)
+    let bounds = floors.checkpointBounds->CheckpointSequence.sql(~chainIdColumn, ~tableRef)
     sql
     ->Postgres.preparedUnsafe(
       `DELETE FROM ${tableRef}${bounds.using} WHERE "${(#id: field :> string)}" > ${bounds.checkpointId}${bounds.usingMatch};`,
-      floors.floors->CheckpointBounds.params,
+      floors.checkpointBounds->CheckpointSequence.params,
     )
     ->Utils.Promise.ignoreValue
   }
 
-  let makePruneStaleCheckpointsQuery = (~pgSchema, ~safeCheckpoints: CheckpointBounds.t) => {
+  let makePruneStaleCheckpointsQuery = (
+    ~pgSchema,
+    ~safeCheckpoints: CheckpointSequence.checkpointBoundsByChain,
+  ) => {
     let tableRef = `"${pgSchema}"."${table.tableName}"`
-    let bounds = safeCheckpoints->CheckpointBounds.sql(~chainIdColumn, ~tableRef)
+    let bounds = safeCheckpoints->CheckpointSequence.sql(~chainIdColumn, ~tableRef)
     `DELETE FROM ${tableRef}${bounds.using} WHERE "${(#id: field :> string)}" < ${bounds.checkpointId}${bounds.usingMatch};`
   }
 
@@ -706,7 +823,7 @@ SELECT * FROM unnest($1::${(BigInt: Postgres.columnType :> string)}[],$2::${chai
     sql
     ->Postgres.preparedUnsafe(
       makePruneStaleCheckpointsQuery(~pgSchema, ~safeCheckpoints),
-      safeCheckpoints->CheckpointBounds.params,
+      safeCheckpoints->CheckpointSequence.params,
     )
     ->Utils.Promise.ignoreValue
 
@@ -738,7 +855,7 @@ LIMIT 1;`
   }
 
   let makeGetRollbackProgressDiffQuery = (~pgSchema, ~floors: RollbackFloors.t) => {
-    let bounds = floors.floors->CheckpointBounds.sql(~chainIdColumn, ~tableRef="t")
+    let bounds = floors.checkpointBounds->CheckpointSequence.sql(~chainIdColumn, ~tableRef="t")
     `SELECT 
   t."${(#chain_id: field :> string)}"::float8 as "${(#chain_id: field :> string)}",
   SUM(t."${(#events_processed: field :> string)}") as events_processed_diff,
@@ -752,7 +869,7 @@ GROUP BY t."${(#chain_id: field :> string)}";`
     sql
     ->Postgres.preparedUnsafe(
       makeGetRollbackProgressDiffQuery(~pgSchema, ~floors),
-      floors.floors->CheckpointBounds.params,
+      floors.checkpointBounds->CheckpointSequence.params,
     )
     ->(
       Utils.magic: promise<unknown> => promise<
@@ -816,6 +933,7 @@ SELECT
   "${(#start_block: Chains.field :> string)}" AS "startBlock", 
   "${(#end_block: Chains.field :> string)}" AS "endBlock",
   "${(#progress_block: Chains.field :> string)}" AS "progressBlock",
+  "${(#progress_block_time: Chains.field :> string)}" AS "progressBlockTime",
   "${(#buffer_block: Chains.field :> string)}" AS "bufferBlock",
   "${(#first_event_block: Chains.field :> string)}" AS "firstEventBlock",
   "${(#events_processed: Chains.field :> string)}"::float4 AS "eventsProcessed",

package/src/db/InternalTable.res.mjs

@@ -5,13 +5,16 @@ import * as Utils from "../Utils.res.mjs";
 import * as Config from "../Config.res.mjs";
 import * as Address from "../Address.res.mjs";
 import * as ChainId from "../ChainId.res.mjs";
+import * as Frontier from "./Frontier.res.mjs";
+import * as Internal from "../Internal.res.mjs";
 import * as Postgres from "../bindings/Postgres.res.mjs";
 import * as AddressRows from "../AddressRows.res.mjs";
 import * as Stdlib_Null from "@rescript/runtime/lib/es6/Stdlib_Null.js";
 import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js";
 import * as Stdlib_JsError from "@rescript/runtime/lib/es6/Stdlib_JsError.js";
-import * as CheckpointBounds from "./CheckpointBounds.res.mjs";
+import * as Primitive_option from "@rescript/runtime/lib/es6/Primitive_option.js";
 import * as S$RescriptSchema from "rescript-schema/src/S.res.mjs";
+import * as CheckpointSequence from "./CheckpointSequence.res.mjs";
 import * as Primitive_exceptions from "@rescript/runtime/lib/es6/Primitive_exceptions.js";
 
 let undefinedTableSqlState = "42P01";
@@ -158,9 +161,11 @@ let fields = [
   "first_event_block",
   "buffer_block",
   "progress_block",
+  "progress_block_time",
   "ready_at",
   "events_processed",
-  "_is_hyper_sync"
+  "_is_hyper_sync",
+  "checkpoint_id"
 ];
 
 let table$2 = Table.mkTable("envio_chains", undefined, [
@@ -175,7 +180,9 @@ let table$2 = Table.mkTable("envio_chains", undefined, [
   Table.mkField("ready_at", "Date", S$RescriptSchema.$$null(Utils.Schema.dbDate), undefined, undefined, true, undefined, undefined, undefined, undefined, undefined, undefined, undefined),
   Table.mkField("events_processed", "UInt52", S$RescriptSchema.float, undefined, undefined, undefined, undefined, undefined, undefined, undefined, undefined, undefined, undefined),
   Table.mkField("_is_hyper_sync", "Boolean", S$RescriptSchema.bool, undefined, undefined, undefined, undefined, undefined, undefined, undefined, undefined, undefined, undefined),
-  Table.mkField("progress_block", "Int32", S$RescriptSchema.int, undefined, undefined, undefined, undefined, undefined, undefined, undefined, undefined, undefined, undefined)
+  Table.mkField("progress_block", "Int32", S$RescriptSchema.int, undefined, undefined, undefined, undefined, undefined, undefined, undefined, undefined, undefined, undefined),
+  Table.mkField("progress_block_time", "Date", S$RescriptSchema.$$null(Utils.Schema.dbDate), undefined, undefined, true, undefined, undefined, undefined, undefined, undefined, undefined, undefined),
+  Table.mkField("checkpoint_id", "UInt64", S$RescriptSchema.bigint, undefined, undefined, undefined, undefined, undefined, undefined, undefined, undefined, undefined, undefined)
 ], undefined);
 
 function initialFromConfig(chainConfig) {
@@ -187,7 +194,9 @@ function initialFromConfig(chainConfig) {
     max_reorg_depth: chainConfig.maxReorgDepth,
     source_block: 0,
     progress_block: -1,
+    progress_block_time: null,
     events_processed: 0,
+    checkpoint_id: Internal.initialCheckpointId,
     first_event_block: null,
     buffer_block: -1,
     ready_at: null,
@@ -260,7 +269,9 @@ function makeGetInitialStateQuery(pgSchema) {
 "` + "ready_at" + `" as "timestampCaughtUpToHeadOrEndblock",
 "` + "events_processed" + `"::float8 as "numEventsProcessed",
 "` + "progress_block" + `" as "progressBlockNumber",
-"` + "source_block" + `" as "sourceBlockNumber"
+"` + "progress_block_time" + `" as "progressBlockTime",
+"` + "source_block" + `" as "sourceBlockNumber",
+"` + "checkpoint_id" + `"::TEXT as "checkpointId"
 FROM "` + pgSchema + `"."` + table$2.tableName + `";`;
 }
 
@@ -281,14 +292,25 @@ async function getInitialState(sql, pgSchema) {
       timestampCaughtUpToHeadOrEndblock: rawInitialState.timestampCaughtUpToHeadOrEndblock,
       numEventsProcessed: rawInitialState.numEventsProcessed,
       progressBlockNumber: rawInitialState.progressBlockNumber,
+      progressBlockTime: rawInitialState.progressBlockTime,
       addressRows: Stdlib_Option.getOr(addressRowsByChainId[ChainId.toString(id)], AddressRows.emptySeedRows()),
-      sourceBlockNumber: rawInitialState.sourceBlockNumber
+      sourceBlockNumber: rawInitialState.sourceBlockNumber,
+      checkpointId: rawInitialState.checkpointId
     };
   });
 }
 
+function blockTimeToDb(blockTime) {
+  return Stdlib_Null.fromOption(Stdlib_Option.map(blockTime, seconds => new Date(seconds * 1000)));
+}
+
+function blockTimeFromDb(date) {
+  return Stdlib_Option.map(date === null ? undefined : Primitive_option.some(date), date => date.getTime() / 1000 | 0);
+}
+
 let progressFields = [
   "progress_block",
+  "progress_block_time",
   "events_processed",
   "source_block"
 ];
@@ -325,8 +347,10 @@ function setProgressedChains(sql, pgSchema, progressedChains) {
     let params = [];
     params.push(data.chainId);
     progressFields.forEach(field => {
-      params.push(field === "source_block" ? data.sourceBlockNumber : (
-          field === "progress_block" ? data.progressBlockNumber : data.totalEventsProcessed
+      params.push(field === "progress_block_time" ? blockTimeToDb(data.progressBlockTime) : (
+          field === "source_block" ? data.sourceBlockNumber : (
+              field === "progress_block" ? data.progressBlockNumber : data.totalEventsProcessed
+            )
         ));
     });
     promises.push(sql.unsafe(query, params, {prepare: true}));
@@ -334,6 +358,20 @@ function setProgressedChains(sql, pgSchema, progressedChains) {
   return Promise.all(promises);
 }
 
+function makeSetCheckpointFrontierQuery(pgSchema, chainIdModeOpt) {
+  let chainIdMode = chainIdModeOpt !== undefined ? chainIdModeOpt : "int32";
+  let chainIdArrayType$1 = chainIdArrayType(pgSchema, chainIdMode);
+  return `UPDATE "` + pgSchema + `"."` + table$2.tableName + `"
+SET "` + "checkpoint_id" + `" = envio_frontier.checkpoint_id
+FROM unnest($1::` + chainIdArrayType$1 + `,$2::` + "BIGINT" + `[]) AS envio_frontier(chain_id, checkpoint_id)
+WHERE "` + table$2.tableName + `"."` + "id" + `" = envio_frontier.chain_id;`;
+}
+
+function setCheckpointFrontier(sql, pgSchema, frontier, chainIdModeOpt) {
+  let chainIdMode = chainIdModeOpt !== undefined ? chainIdModeOpt : "int32";
+  return sql.unsafe(makeSetCheckpointFrontierQuery(pgSchema, chainIdMode), Frontier.unnestParams(frontier), {prepare: true});
+}
+
 let Chains = {
   fields: fields,
   table: table$2,
@@ -344,10 +382,14 @@ let Chains = {
   makeSetReadyAtQuery: makeSetReadyAtQuery,
   makeGetInitialStateQuery: makeGetInitialStateQuery,
   getInitialState: getInitialState,
+  blockTimeToDb: blockTimeToDb,
+  blockTimeFromDb: blockTimeFromDb,
   progressFields: progressFields,
   makeProgressFieldsUpdateQuery: makeProgressFieldsUpdateQuery,
   setMeta: setMeta,
-  setProgressedChains: setProgressedChains
+  setProgressedChains: setProgressedChains,
+  makeSetCheckpointFrontierQuery: makeSetCheckpointFrontierQuery,
+  setCheckpointFrontier: setCheckpointFrontier
 };
 
 let table$3 = Table.mkTable("envio_info", undefined, [
@@ -393,33 +435,78 @@ let dbSchema = S$RescriptSchema.object(s => ({
 
 let columns = [
   {
-    field: Table.mkField("id", "UInt64", S$RescriptSchema.bigint, undefined, undefined, undefined, true, undefined, undefined, undefined, undefined, undefined, undefined),
-    clickHouseFieldType: "UInt64",
-    valuesOf: batch => batch.checkpointIds
+    field: Table.mkField("chain_id", "ChainId", ChainId.schema, undefined, undefined, undefined, true, undefined, undefined, undefined, undefined, undefined, undefined),
+    clickHouseFieldType: "ChainId",
+    valuesOf: batch => batch.checkpointChainIds,
+    diffValuesOf: diffs => diffs.map(diff => diff.chainId)
   },
   {
-    field: Table.mkField("chain_id", "ChainId", ChainId.schema, undefined, undefined, undefined, undefined, undefined, undefined, undefined, undefined, undefined, undefined),
-    clickHouseFieldType: "ChainId",
-    valuesOf: batch => batch.checkpointChainIds
+    field: Table.mkField("id", "UInt64", S$RescriptSchema.bigint, undefined, undefined, undefined, true, undefined, undefined, undefined, undefined, undefined, undefined),
+    clickHouseFieldType: "UInt64",
+    valuesOf: batch => batch.checkpointIds,
+    diffValuesOf: diffs => diffs.map(diff => diff.checkpointId)
   },
   {
     field: Table.mkField("block_number", "Int32", S$RescriptSchema.int, undefined, undefined, undefined, undefined, undefined, undefined, undefined, undefined, undefined, undefined),
     clickHouseFieldType: "Int32",
-    valuesOf: batch => batch.checkpointBlockNumbers
+    valuesOf: batch => batch.checkpointBlockNumbers,
+    diffValuesOf: diffs => diffs.map(diff => diff.blockNumber)
   },
   {
     field: Table.mkField("block_hash", "String", S$RescriptSchema.$$null(S$RescriptSchema.string), undefined, undefined, true, undefined, undefined, undefined, undefined, undefined, undefined, undefined),
     clickHouseFieldType: "String",
-    valuesOf: batch => batch.checkpointBlockHashes
+    valuesOf: batch => batch.checkpointBlockHashes,
+    diffValuesOf: diffs => diffs.map(param => null)
   },
   {
     field: Table.mkField("events_processed", "Int32", S$RescriptSchema.int, undefined, undefined, undefined, undefined, undefined, undefined, undefined, undefined, undefined, undefined),
     clickHouseFieldType: "UInt64",
-    valuesOf: batch => batch.checkpointEventsProcessed
+    valuesOf: batch => batch.checkpointEventsProcessed,
+    diffValuesOf: diffs => diffs.map(param => 0)
   }
 ];
 
-let table$4 = Table.mkTable("envio_checkpoints", undefined, columns.map(param => param.field), undefined);
+let tableName = "envio_checkpoints";
+
+let table$4 = Table.mkTable(tableName, undefined, columns.map(param => param.field), undefined);
+
+let globalTable = Table.mkTable(tableName, undefined, columns.map(param => {
+  let column = param.field;
+  if (column.TAG !== "Field") {
+    return column;
+  }
+  let f = column._0;
+  if (f.fieldName === "chain_id") {
+    return {
+      TAG: "Field",
+      _0: {
+        fieldName: f.fieldName,
+        fieldType: f.fieldType,
+        fieldSchema: f.fieldSchema,
+        isArray: f.isArray,
+        isNullable: f.isNullable,
+        isPrimaryKey: false,
+        isIndex: f.isIndex,
+        isChainId: f.isChainId,
+        linkedEntity: f.linkedEntity,
+        defaultValue: f.defaultValue,
+        description: f.description,
+        postgresDbName: f.postgresDbName,
+        clickhouseDbName: f.clickhouseDbName
+      }
+    };
+  } else {
+    return column;
+  }
+}), undefined);
+
+function tableFor(sequence) {
+  if (sequence === "SharedAcrossChains") {
+    return globalTable;
+  } else {
+    return table$4;
+  }
+}
 
 function makeGetReorgCheckpointsQuery(pgSchema) {
   return `WITH reorg_chains AS (
@@ -443,10 +530,6 @@ WHERE cp."` + "block_hash" + `" IS NOT NULL
 ORDER BY cp."` + "id" + `";`;
 }
 
-function makeCommitedCheckpointIdQuery(pgSchema) {
-  return `SELECT COALESCE(MAX(` + "id" + `), ` + (0n).toString() + `) AS id FROM "` + pgSchema + `"."` + table$4.tableName + `";`;
-}
-
 function makeInsertCheckpointQuery(pgSchema, chainIdModeOpt) {
   let chainIdMode = chainIdModeOpt !== undefined ? chainIdModeOpt : "int32";
   let chainIdArrayType$1 = chainIdArrayType(pgSchema, chainIdMode);
@@ -471,18 +554,18 @@ let chainIdColumn = "chain_id";
 
 function rollback(sql, pgSchema, floors) {
   let tableRef = `"` + pgSchema + `"."` + table$4.tableName + `"`;
-  let bounds = CheckpointBounds.sql(floors.floors, chainIdColumn, tableRef);
-  return sql.unsafe(`DELETE FROM ` + tableRef + bounds.using + ` WHERE "` + "id" + `" > ` + bounds.checkpointId + bounds.usingMatch + `;`, CheckpointBounds.params(floors.floors), {prepare: true});
+  let bounds = CheckpointSequence.sql(floors.checkpointBounds, chainIdColumn, tableRef);
+  return sql.unsafe(`DELETE FROM ` + tableRef + bounds.using + ` WHERE "` + "id" + `" > ` + bounds.checkpointId + bounds.usingMatch + `;`, CheckpointSequence.params(floors.checkpointBounds), {prepare: true});
 }
 
 function makePruneStaleCheckpointsQuery(pgSchema, safeCheckpoints) {
   let tableRef = `"` + pgSchema + `"."` + table$4.tableName + `"`;
-  let bounds = CheckpointBounds.sql(safeCheckpoints, chainIdColumn, tableRef);
+  let bounds = CheckpointSequence.sql(safeCheckpoints, chainIdColumn, tableRef);
   return `DELETE FROM ` + tableRef + bounds.using + ` WHERE "` + "id" + `" < ` + bounds.checkpointId + bounds.usingMatch + `;`;
 }
 
 function pruneStaleCheckpoints(sql, pgSchema, safeCheckpoints) {
-  return sql.unsafe(makePruneStaleCheckpointsQuery(pgSchema, safeCheckpoints), CheckpointBounds.params(safeCheckpoints), {prepare: true});
+  return sql.unsafe(makePruneStaleCheckpointsQuery(pgSchema, safeCheckpoints), CheckpointSequence.params(safeCheckpoints), {prepare: true});
 }
 
 function makeGetRollbackTargetCheckpointQuery(pgSchema) {
@@ -503,7 +586,7 @@ function getRollbackTargetCheckpoint(sql, pgSchema, reorgChainId, lastKnownValid
 }
 
 function makeGetRollbackProgressDiffQuery(pgSchema, floors) {
-  let bounds = CheckpointBounds.sql(floors.floors, chainIdColumn, "t");
+  let bounds = CheckpointSequence.sql(floors.checkpointBounds, chainIdColumn, "t");
   return `SELECT 
   t."` + "chain_id" + `"::float8 as "` + "chain_id" + `",
   SUM(t."` + "events_processed" + `") as events_processed_diff,
@@ -514,16 +597,18 @@ GROUP BY t."` + "chain_id" + `";`;
 }
 
 function getRollbackProgressDiff(sql, pgSchema, floors) {
-  return sql.unsafe(makeGetRollbackProgressDiffQuery(pgSchema, floors), CheckpointBounds.params(floors.floors), {prepare: true});
+  return sql.unsafe(makeGetRollbackProgressDiffQuery(pgSchema, floors), CheckpointSequence.params(floors.checkpointBounds), {prepare: true});
 }
 
 let Checkpoints = {
   dbSchema: dbSchema,
   initialCheckpointId: 0n,
   columns: columns,
+  tableName: tableName,
   table: table$4,
+  globalTable: globalTable,
+  tableFor: tableFor,
   makeGetReorgCheckpointsQuery: makeGetReorgCheckpointsQuery,
-  makeCommitedCheckpointIdQuery: makeCommitedCheckpointIdQuery,
   makeInsertCheckpointQuery: makeInsertCheckpointQuery,
   insert: insert$2,
   chainIdColumn: chainIdColumn,
@@ -584,6 +669,7 @@ SELECT
   "` + "start_block" + `" AS "startBlock", 
   "` + "end_block" + `" AS "endBlock",
   "` + "progress_block" + `" AS "progressBlock",
+  "` + "progress_block_time" + `" AS "progressBlockTime",
   "` + "buffer_block" + `" AS "bufferBlock",
   "` + "first_event_block" + `" AS "firstEventBlock",
   "` + "events_processed" + `"::float4 AS "eventsProcessed",