envio 3.11.0 → 3.12.1

package/src/Writing.res

@@ -42,12 +42,10 @@ let snapshotEffects = (state: IndexerState.t, ~cache): array<Persistence.updated
     switch idsToStore {
     | [] => ()
     | ids =>
-      let items = ids->Array.filterMap((id): option<Internal.effectCacheItem> =>
-        switch dict->Dict.getUnsafe(id) {
-        | Set({entity: output}) => Some({id, output})
-        | Delete(_) => None
-        }
-      )
+      let items = ids->Array.map((id): Internal.effectCacheItem => {
+        id,
+        output: (dict->Dict.getUnsafe(id)).output,
+      })
       let effectName = effect.name
       let tableName = table.tableName
       let effectCacheRecord = switch cache->Utils.Dict.dangerouslyGetNonOption(tableName) {
@@ -101,22 +99,34 @@ let runOneWrite = async (state: IndexerState.t) => {
     | None => ()
     }
   | _ =>
-    let committedCheckpointId = state->IndexerState.committedCheckpointId
     let batch = state->IndexerState.drainBatchRun
-    // The run's last checkpoint; entity changes above it stay queued for the next write.
-    let upToCheckpointId = switch batch.checkpointIds->Utils.Array.last {
-    | Some(checkpointId) => checkpointId
-    | None => committedCheckpointId
-    }
 
     let rollback = state->IndexerState.takeRollback
 
+    // Entity changes above it stay queued for the next write.
+    let writtenFrontier = Persistence.writtenFrontier(~batch, ~rollback)
+
     let updatedEntities = []
     state->IndexerState.eachEntityTable((~entityConfig, ~scope, ~table) => {
       let changes =
-        table->InMemoryTable.Entity.snapshotChanges(~committedCheckpointId, ~upToCheckpointId)
+        table->InMemoryTable.Entity.snapshotChanges(
+          ~committedCheckpointId=state->IndexerState.committedCheckpointIdFor(~scope),
+          ~upToCheckpointId=config.checkpointSequence->CheckpointSequence.forScope(
+            writtenFrontier,
+            ~scope,
+          ),
+        )
       if changes->Utils.Array.notEmpty {
-        updatedEntities->Array.push(({entityConfig, scope, changes}: Persistence.updatedEntity))
+        updatedEntities->Array.push(
+          (
+            {
+              entityConfig,
+              scope,
+              changes,
+              shouldSaveHistory: batch.history->HistoryPolicy.forScope(~scope),
+            }: Persistence.updatedEntity
+          ),
+        )
       }
     })
     let updatedEffectsCache = snapshotEffects(state, ~cache)
@@ -142,7 +152,6 @@ let runOneWrite = async (state: IndexerState.t) => {
       persistence.storage.writeBatch(
         ~batch,
         ~rollback,
-        ~isInReorgThreshold=batch.isInReorgThreshold,
         ~config,
         ~allEntities=persistence.allEntities,
         ~updatedEntities,
@@ -155,7 +164,7 @@ let runOneWrite = async (state: IndexerState.t) => {
       PruneStaleHistory.runConcurrent(state, ~targets=pruneTargets),
     ))
 
-    state->IndexerState.markCommitted(~upToCheckpointId)
+    state->IndexerState.markCommitted(~writtenFrontier)
 
     switch rollback {
     | Some({progressedChains}) if RollbackCommit.callbacks->Utils.Array.notEmpty =>
@@ -218,14 +227,19 @@ let commitBatch = (state: IndexerState.t, ~batch: Batch.t) => {
 // Drops committed entity and effect entries across all tables. With
 // keepLoadedFromDb, entries seeded from a db read are spared.
 let dropCommitted = (state: IndexerState.t, ~keepLoadedFromDb) => {
-  let committedCheckpointId = state->IndexerState.committedCheckpointId
-  state->IndexerState.eachEntityTable((~entityConfig as _, ~scope as _, ~table) =>
-    table->InMemoryTable.Entity.dropCommittedChanges(~committedCheckpointId, ~keepLoadedFromDb)
+  state->IndexerState.eachEntityTable((~entityConfig as _, ~scope, ~table) =>
+    table->InMemoryTable.Entity.dropCommittedChanges(
+      ~committedCheckpointId=state->IndexerState.committedCheckpointIdFor(~scope),
+      ~keepLoadedFromDb,
+    )
   )
   state
   ->IndexerState.effectState
   ->EffectState.forEach(inMemTable =>
-    inMemTable->InMemoryStore.dropCommittedEffects(~committedCheckpointId, ~keepLoadedFromDb)
+    inMemTable->InMemoryStore.dropCommittedEffects(
+      ~committedFrontier=state->IndexerState.committedFrontier,
+      ~keepLoadedFromDb,
+    )
   )
 }
 

package/src/Writing.res.mjs

@@ -5,14 +5,16 @@ import * as Utils from "./Utils.res.mjs";
 import * as Throttler from "./Throttler.res.mjs";
 import * as EffectState from "./EffectState.res.mjs";
 import * as Performance from "./bindings/Performance.res.mjs";
+import * as Persistence from "./Persistence.res.mjs";
 import * as IndexerState from "./IndexerState.res.mjs";
-import * as Stdlib_Array from "@rescript/runtime/lib/es6/Stdlib_Array.js";
+import * as HistoryPolicy from "./HistoryPolicy.res.mjs";
 import * as InMemoryStore from "./InMemoryStore.res.mjs";
 import * as InMemoryTable from "./InMemoryTable.res.mjs";
 import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js";
 import * as RollbackCommit from "./RollbackCommit.res.mjs";
 import * as Stdlib_JsError from "@rescript/runtime/lib/es6/Stdlib_JsError.js";
 import * as PruneStaleHistory from "./PruneStaleHistory.res.mjs";
+import * as CheckpointSequence from "./db/CheckpointSequence.res.mjs";
 import * as Primitive_exceptions from "@rescript/runtime/lib/es6/Primitive_exceptions.js";
 
 function getChangesCount(state) {
@@ -44,15 +46,10 @@ function snapshotEffects(state, cache) {
     let table = inMemTable.table;
     let effect = inMemTable.effect;
     if (idsToStore.length !== 0) {
-      let items = Stdlib_Array.filterMap(idsToStore, id => {
-        let match = dict[id];
-        if (match.type === "SET") {
-          return {
-            id: id,
-            output: match.entity
-          };
-        }
-      });
+      let items = idsToStore.map(id => ({
+        id: id,
+        output: dict[id].output
+      }));
       let effectName = effect.name;
       let tableName = table.tableName;
       let c = cache[tableName];
@@ -100,19 +97,18 @@ async function runOneWrite(state) {
       return;
     }
   }
-  let committedCheckpointId = IndexerState.committedCheckpointId(state);
   let batch = IndexerState.drainBatchRun(state);
-  let checkpointId = Utils.$$Array.last(batch.checkpointIds);
-  let upToCheckpointId = checkpointId !== undefined ? checkpointId : committedCheckpointId;
   let rollback = IndexerState.takeRollback(state);
+  let writtenFrontier = Persistence.writtenFrontier(batch, rollback);
   let updatedEntities = [];
   IndexerState.eachEntityTable(state, (entityConfig, scope, table) => {
-    let changes = InMemoryTable.Entity.snapshotChanges(table, committedCheckpointId, upToCheckpointId);
+    let changes = InMemoryTable.Entity.snapshotChanges(table, IndexerState.committedCheckpointIdFor(state, scope), CheckpointSequence.forScope(config.checkpointSequence, writtenFrontier, scope));
     if (Utils.$$Array.notEmpty(changes)) {
       updatedEntities.push({
         entityConfig: entityConfig,
         scope: scope,
-        changes: changes
+        changes: changes,
+        shouldSaveHistory: HistoryPolicy.forScope(batch.history, scope)
       });
       return;
     }
@@ -125,10 +121,10 @@ async function runOneWrite(state) {
   });
   let pruneTargets = PruneStaleHistory.select(state, writtenEntityNames, Stdlib_Option.isSome(rollback));
   await Promise.all([
-    persistence.storage.writeBatch(batch, rollback, batch.isInReorgThreshold, config, persistence.allEntities, updatedEffectsCache, updatedEntities, registeredAddresses, chainMetaData, (storage, timeSeconds) => IndexerState.recordStorageWrite(state, storage, timeSeconds)),
+    persistence.storage.writeBatch(batch, rollback, config, persistence.allEntities, updatedEffectsCache, updatedEntities, registeredAddresses, chainMetaData, (storage, timeSeconds) => IndexerState.recordStorageWrite(state, storage, timeSeconds)),
     PruneStaleHistory.runConcurrent(state, pruneTargets)
   ]);
-  IndexerState.markCommitted(state, upToCheckpointId);
+  IndexerState.markCommitted(state, writtenFrontier);
   if (rollback !== undefined && Utils.$$Array.notEmpty(RollbackCommit.callbacks)) {
     await RollbackCommit.fire(rollback.progressedChains);
   }
@@ -179,9 +175,8 @@ function commitBatch(state, batch) {
 }
 
 function dropCommitted(state, keepLoadedFromDb) {
-  let committedCheckpointId = IndexerState.committedCheckpointId(state);
-  IndexerState.eachEntityTable(state, (param, param$1, table) => InMemoryTable.Entity.dropCommittedChanges(table, committedCheckpointId, keepLoadedFromDb));
-  EffectState.forEach(IndexerState.effectState(state), inMemTable => InMemoryStore.dropCommittedEffects(inMemTable, committedCheckpointId, keepLoadedFromDb));
+  IndexerState.eachEntityTable(state, (param, scope, table) => InMemoryTable.Entity.dropCommittedChanges(table, IndexerState.committedCheckpointIdFor(state, scope), keepLoadedFromDb));
+  EffectState.forEach(IndexerState.effectState(state), inMemTable => InMemoryStore.dropCommittedEffects(inMemTable, IndexerState.committedFrontier(state), keepLoadedFromDb));
 }
 
 async function awaitCapacityLoop(state) {

package/src/bindings/ClickHouse.res

@@ -90,6 +90,7 @@ let makeColumnSpec = (
 type checkpointColumn = {
   spec: ClickHouseSink.columnSpec,
   valuesOf: Batch.t => array<unknown>,
+  diffValuesOf: array<InternalTable.Checkpoints.diffCheckpoint> => array<unknown>,
 }
 
 // Registration and staging both read this one list, so the column a batch's
@@ -98,12 +99,14 @@ let checkpointColumns = InternalTable.Checkpoints.columns->Array.filterMap(({
   field,
   clickHouseFieldType,
   valuesOf,
+  diffValuesOf,
 }) =>
   switch field {
   | Table.Field({fieldName, isNullable}) =>
     Some({
       spec: makeColumnSpec(~name=fieldName, ~fieldType=clickHouseFieldType, ~isNullable),
       valuesOf,
+      diffValuesOf,
     })
   | DerivedFrom(_) => None
   }
@@ -259,25 +262,18 @@ let makeRegistry = () => {
 )
 
 %%private(
-  let fillBuilders = (
-    ~table: ClickHouseSink.table,
-    ~converters,
-    ~changes: array<Change.t<Internal.entity>>,
-  ) => {
-    let rows = changes->Array.length
-    let builders = table.columns->Array.map(ClickHouseSink.makeBuilder(_, ~rows))
-    let columns = builders->Array.length
-    for row in 0 to rows - 1 {
+  let fillStage = (stage, ~converters, ~changes: array<Change.t<Internal.entity>>) => {
+    let columns = stage->Staging.columnCount
+    for row in 0 to changes->Array.length - 1 {
       let change = changes->Array.getUnsafe(row)
       let (cells, write) = switch change {
-      | Change.Set(_) => (converters.setCells, ClickHouseSink.writeValue)
-      | Delete(_) => (converters.deleteCells, ClickHouseSink.writeDeletedValue)
+      | Change.Set(_) => (converters.setCells, Staging.writeValue)
+      | Delete(_) => (converters.deleteCells, Staging.writeDeletedValue)
       }
       for column in 0 to columns - 1 {
-        builders->Array.getUnsafe(column)->write(~row, (cells->Array.getUnsafe(column))(change))
+        stage->write(~column, ~row, (cells->Array.getUnsafe(column))(change))
       }
     }
-    builders
   }
 )
 
@@ -306,31 +302,33 @@ let checkpointsTable = (sink, ~registry) =>
     table
   }
 
-let stageBuilders = (sink, ~table: ClickHouseSink.table, ~builders, ~rows) =>
-  sink->ClickHouseSink.stage(
-    ~table=table.handle,
-    ~rows,
-    ~columns=builders->Array.map(ClickHouseSink.builderPayload),
-  )
-
-let stageCheckpointsOrThrow = (sink, ~registry, ~batch: Batch.t) => {
-  let rows = batch.checkpointIds->Array.length
+let stageCheckpointsOrThrow = (
+  sink,
+  ~registry,
+  ~batch: Batch.t,
+  ~diffCheckpoints: array<InternalTable.Checkpoints.diffCheckpoint>,
+) => {
+  let rows = batch.checkpointIds->Array.length + diffCheckpoints->Array.length
   if rows === 0 {
     Null.null
   } else {
     let table = sink->checkpointsTable(~registry)
+    let stage =
+      sink->ClickHouseSink.arena->Staging.begin(~table=table.handle, ~rows, ~columns=table.columns)
     try {
       // The table was registered from `checkpointColumns`, in this order.
-      let builders = table.columns->Array.map(ClickHouseSink.makeBuilder(_, ~rows))
-      builders->Array.forEachWithIndex((builder, index) => {
-        let columnValues = (checkpointColumns->Array.getUnsafe(index)).valuesOf(batch)
+      table.columns->Array.forEachWithIndex((_, column) => {
+        let source = checkpointColumns->Array.getUnsafe(column)
+        let columnValues =
+          source.valuesOf(batch)->Array.concat(source.diffValuesOf(diffCheckpoints))
         for row in 0 to rows - 1 {
-          builder->ClickHouseSink.writeValue(~row, columnValues->Array.getUnsafe(row))
+          stage->Staging.writeValue(~column, ~row, columnValues->Array.getUnsafe(row))
         }
       })
-      Null.make(sink->stageBuilders(~table, ~builders, ~rows))
+      Null.make(stage->Staging.commit)
     } catch {
     | exn =>
+      stage->Staging.abort
       throw(
         Persistence.StorageError({
           message: `Failed to convert checkpoints for ClickHouse table "${table.name}"`,
@@ -355,6 +353,8 @@ let stageUpdatesOrThrow = (
     let table = sink->entityTable(~registry, ~entityConfig)
     let tableName = table.name
     let cacheKey = `${entityConfig.name}|${scope->Internal.chainScopeToString}`
+    let stage =
+      sink->ClickHouseSink.arena->Staging.begin(~table=table.handle, ~rows, ~columns=table.columns)
     try {
       let converters = switch registry.converters->Utils.Dict.dangerouslyGetNonOption(cacheKey) {
       | Some(cached) => cached
@@ -363,10 +363,11 @@ let stageUpdatesOrThrow = (
         registry.converters->Dict.set(cacheKey, cached)
         cached
       }
-      let builders = fillBuilders(~table, ~converters, ~changes)
-      Some(sink->stageBuilders(~table, ~builders, ~rows))
+      stage->fillStage(~converters, ~changes)
+      Some(stage->Staging.commit)
     } catch {
     | exn =>
+      stage->Staging.abort
       throw(
         Persistence.StorageError({
           message: `Failed to convert items for ClickHouse table "${tableName}"`,
@@ -415,20 +416,26 @@ let initialize = async (sink, ~entities: array<Internal.entityConfig>) => {
 
 let resume = async (
   sink,
-  ~checkpointId: Internal.checkpointId,
+  ~sequence: CheckpointSequence.t,
+  ~frontier: Frontier.t,
   ~chains: array<Persistence.initialChainState>,
   ~entities: array<Internal.entityConfig>,
 ) => {
   let chainProgress = chains->Array.map(chain => {
     ClickHouseSink.chainId: chain.id->ChainId.toString,
     progressBlockNumber: chain.progressBlockNumber,
+    committedCheckpointId: frontier->Frontier.get(chain.id)->BigInt.toString,
   })
   try await sink->ClickHouseSink.resume({
-    checkpointId: checkpointId->BigInt.toString,
+    perChain: switch sequence {
+    | PerChain => true
+    | SharedAcrossChains => false
+    },
     chainProgress,
-    historyTables: entities->Array.map(entityConfig => entitySpec(~entityConfig).historyTable),
-    replicated: Env.ClickHouse.replicated(),
-    databaseEngine: ?Env.ClickHouse.databaseEngine(),
+    historyTables: entities->Array.map(entityConfig => {
+      let spec = entitySpec(~entityConfig)
+      {ClickHouseSink.name: spec.historyTable, chainIdColumn: ?spec.chainIdColumn}
+    }),
   }) catch {
   | exn => {
       Logging.errorWithExn(exn, "Failed to resume ClickHouse storage")

package/src/bindings/ClickHouse.res.mjs

@@ -5,6 +5,8 @@ import * as Table from "../db/Table.res.mjs";
 import * as Utils from "../Utils.res.mjs";
 import * as ChainId from "../ChainId.res.mjs";
 import * as Logging from "../Logging.res.mjs";
+import * as Staging from "../Staging.res.mjs";
+import * as Frontier from "../db/Frontier.res.mjs";
 import * as Internal from "../Internal.res.mjs";
 import * as Persistence from "../Persistence.res.mjs";
 import * as Stdlib_Dict from "@rescript/runtime/lib/es6/Stdlib_Dict.js";
@@ -176,7 +178,8 @@ let checkpointColumns = Stdlib_Array.filterMap(InternalTable.Checkpoints.columns
   let match = field._0;
   return {
     spec: makeColumnSpec(match.fieldName, undefined, param.clickHouseFieldType, match.isNullable, undefined),
-    valuesOf: param.valuesOf
+    valuesOf: param.valuesOf,
+    diffValuesOf: param.diffValuesOf
   };
 });
 
@@ -303,27 +306,24 @@ function makeConverters(entityConfig, scope, table) {
   };
 }
 
-function fillBuilders(table, converters, changes) {
-  let rows = changes.length;
-  let builders = table.columns.map(__x => ClickHouseSink.makeBuilder(__x, rows));
-  let columns = builders.length;
-  for (let row = 0; row < rows; ++row) {
+function fillStage(stage, converters, changes) {
+  let columns = Staging.columnCount(stage);
+  for (let row = 0, row_finish = changes.length; row < row_finish; ++row) {
     let change = changes[row];
     let match;
     match = change.type === "SET" ? [
         converters.setCells,
-        ClickHouseSink.writeValue
+        Staging.writeValue
       ] : [
         converters.deleteCells,
-        ClickHouseSink.writeDeletedValue
+        Staging.writeDeletedValue
       ];
     let write = match[1];
     let cells = match[0];
     for (let column = 0; column < columns; ++column) {
-      write(builders[column], row, cells[column](change));
+      write(stage, column, row, cells[column](change));
     }
   }
-  return builders;
 }
 
 function entityTable(sink, registry, entityConfig) {
@@ -347,27 +347,25 @@ function checkpointsTable(sink, registry) {
   return table$1;
 }
 
-function stageBuilders(sink, table, builders, rows) {
-  return sink.stage(table.handle, rows, builders.map(ClickHouseSink.builderPayload));
-}
-
-function stageCheckpointsOrThrow(sink, registry, batch) {
-  let rows = batch.checkpointIds.length;
+function stageCheckpointsOrThrow(sink, registry, batch, diffCheckpoints) {
+  let rows = batch.checkpointIds.length + diffCheckpoints.length | 0;
   if (rows === 0) {
     return null;
   }
   let table = checkpointsTable(sink, registry);
+  let stage = Staging.begin(ClickHouseSink.arena(sink), table.handle, rows, table.columns);
   try {
-    let builders = table.columns.map(__x => ClickHouseSink.makeBuilder(__x, rows));
-    builders.forEach((builder, index) => {
-      let columnValues = checkpointColumns[index].valuesOf(batch);
+    table.columns.forEach((param, column) => {
+      let source = checkpointColumns[column];
+      let columnValues = source.valuesOf(batch).concat(source.diffValuesOf(diffCheckpoints));
       for (let row = 0; row < rows; ++row) {
-        ClickHouseSink.writeValue(builder, row, columnValues[row]);
+        Staging.writeValue(stage, column, row, columnValues[row]);
       }
     });
-    return stageBuilders(sink, table, builders, rows);
+    return Staging.commit(stage);
   } catch (raw_exn) {
     let exn = Primitive_exceptions.internalToException(raw_exn);
+    Staging.abort(stage);
     throw {
       RE_EXN_ID: Persistence.StorageError,
       message: `Failed to convert checkpoints for ClickHouse table "` + table.name + `"`,
@@ -385,6 +383,7 @@ function stageUpdatesOrThrow(sink, registry, changes, entityConfig, scope) {
   let table = entityTable(sink, registry, entityConfig);
   let tableName = table.name;
   let cacheKey = entityConfig.name + `|` + Internal.chainScopeToString(scope);
+  let stage = Staging.begin(ClickHouseSink.arena(sink), table.handle, rows, table.columns);
   try {
     let cached = registry.converters[cacheKey];
     let converters;
@@ -395,10 +394,11 @@ function stageUpdatesOrThrow(sink, registry, changes, entityConfig, scope) {
       registry.converters[cacheKey] = cached$1;
       converters = cached$1;
     }
-    let builders = fillBuilders(table, converters, changes);
-    return stageBuilders(sink, table, builders, rows);
+    fillStage(stage, converters, changes);
+    return Staging.commit(stage);
   } catch (raw_exn) {
     let exn = Primitive_exceptions.internalToException(raw_exn);
+    Staging.abort(stage);
     throw {
       RE_EXN_ID: Persistence.StorageError,
       message: `Failed to convert items for ClickHouse table "` + tableName + `"`,
@@ -443,18 +443,25 @@ async function initialize(sink, entities) {
   }
 }
 
-async function resume(sink, checkpointId, chains, entities) {
+async function resume(sink, sequence, frontier, chains, entities) {
   let chainProgress = chains.map(chain => ({
     chainId: ChainId.toString(chain.id),
-    progressBlockNumber: chain.progressBlockNumber
+    progressBlockNumber: chain.progressBlockNumber,
+    committedCheckpointId: Frontier.get(frontier, chain.id).toString()
   }));
   try {
+    let tmp;
+    tmp = sequence !== "SharedAcrossChains";
     return await sink.resume({
-      checkpointId: checkpointId.toString(),
+      perChain: tmp,
       chainProgress: chainProgress,
-      historyTables: entities.map(entityConfig => entitySpec(entityConfig).historyTable),
-      replicated: Env.ClickHouse.replicated(),
-      databaseEngine: Env.ClickHouse.databaseEngine()
+      historyTables: entities.map(entityConfig => {
+        let spec = entitySpec(entityConfig);
+        return {
+          name: spec.historyTable,
+          chainIdColumn: spec.chainIdColumn
+        };
+      })
     });
   } catch (raw_exn) {
     let exn = Primitive_exceptions.internalToException(raw_exn);
@@ -480,7 +487,6 @@ export {
   makeRegistry,
   entityTable,
   checkpointsTable,
-  stageBuilders,
   stageCheckpointsOrThrow,
   stageUpdatesOrThrow,
   writeStagedOrThrow,