envio 3.9.0 → 3.10.0

package/src/bindings/ClickHouse.res

@@ -1,169 +1,32 @@
-// ClickHouse client bindings for @clickhouse/client
-
-type client
-
-type clientConfig = {
-  url?: string,
-  database?: string,
-  username?: string,
-  password?: string,
-}
-
-type execParams = {query: string}
-
-@module("@clickhouse/client")
-external createClient: clientConfig => client = "createClient"
-
-// `command`, not `exec`: exec hands its response stream to the caller and holds
-// the socket until it is consumed, which nothing here does. The pool is 10
-// sockets wide, so a schema whose DDL needs more than that — initialize issues
-// 2N+3 statements for N entities — stalled every statement past the tenth until
-// the 30s request timeout freed one. command destroys the stream for us.
-@send
-external command: (client, execParams) => promise<unit> = "command"
-
-@send
-external close: client => promise<unit> = "close"
-
-type insertParams<'a> = {
-  table: string,
-  values: array<'a>,
-  format: string,
-}
-
-@send
-external insert: (client, insertParams<'a>) => promise<unit> = "insert"
-
-type queryParams = {query: string}
-type queryResult<'a>
-
-@send
-external query: (client, queryParams) => promise<queryResult<'a>> = "query"
-
-// The default `JSON` query format resolves to a `ResponseJSON` wrapper whose
-// rows live under `data`, not at the top level.
-@send
-external json: queryResult<'a> => promise<{"data": array<'a>}> = "json"
-
-let getClickHouseFieldType = (
-  ~fieldType: Table.fieldType,
-  ~isNullable: bool,
-  ~isArray: bool,
-  ~chainIdMode: ChainId.mode=Int32,
-): string => {
-  let baseType = switch fieldType {
-  | Int32 => "Int32"
-  | ChainId =>
-    switch chainIdMode {
-    | Int32 => "Int32"
-    | Int64 => "UInt64"
-    }
-  | Uint32 => "UInt32"
-  | UInt52 => "UInt64"
-  // Internal-only column types, never on an entity the sink mirrors.
-  | SmallInt
-  | Bytea =>
-    JsError.throwWithMessage(
-      "ClickHouse doesn't support the internal SmallInt and Bytea column types",
-    )
-  | UInt64 => "UInt64"
-  | Serial => "Int32"
-  | BigSerial => "Int64"
-  | BigInt({?precision}) =>
-    switch precision {
-    | None => "String" // Fallback for unbounded BigInt
-    | Some(precision) =>
-      if precision > 38 {
-        "String"
-      } else {
-        `Decimal(${precision->Int.toString},0)`
-      }
-    }
-  | BigDecimal({?config}) =>
-    switch config {
-    | None => "String" // Fallback for unbounded BigDecimal
-    | Some((precision, scale)) =>
-      if precision > 38 || scale > precision {
-        "String"
-      } else {
-        `Decimal(${precision->Int.toString},${scale->Int.toString})`
-      }
-    }
-  | Boolean => "Bool"
-  | Number => "Float64"
-  | String => "String"
-  | Json => "String"
-  | Date => "DateTime64(3, 'UTC')"
-  | Enum({config}) => {
-      let variantsLength = config.variants->Array.length
-      // Theoretically we can store 256 variants in Enum8,
-      // but it'd require to explicitly start with a negative index (probably)
-      let enumType = variantsLength <= 127 ? "Enum8" : "Enum16"
-      let enumValues =
-        config.variants
-        ->Array.map(variant => {
-          let variantStr = variant->(Utils.magic: 'a => string)
-          `'${variantStr}'`
-        })
-        ->Array.joinUnsafe(", ")
-      `${enumType}(${enumValues})`
+let clickHouseFieldSchema = (f: Table.field) => {
+  let shaped = baseSchema =>
+    if f.isNullable {
+      Utils.Schema.nullTolerant(baseSchema)->S.toUnknown
+    } else if f.isArray {
+      S.array(baseSchema)->S.toUnknown
+    } else {
+      baseSchema
     }
+  switch f.fieldType {
+  | Date => shaped(Utils.Schema.clickHouseDate->S.toUnknown)
+  | Bytea => shaped(S.json(~validate=false)->S.toUnknown)
+  | ChainId => ChainId.schema->S.toUnknown
+  | Json if !f.isArray => Utils.Schema.clickHouseJson->S.toUnknown
+  | _ => f.fieldSchema
   }
-
-  let baseType = if isArray {
-    `Array(${baseType})`
-  } else {
-    baseType
-  }
-
-  isNullable ? `Nullable(${baseType})` : baseType
 }
 
-// Creates an entity schema from table definition, using clickHouseDate for Date fields.
-// Serialized keys are the db column names, while the entity values are keyed
-// by API field names (they only differ when column renaming is configured).
 let makeClickHouseEntitySchema = (table: Table.table): S.t<Internal.entity> => {
   S.object(s => {
     let dict = Dict.make()
     table.fields->Array.forEach(field => {
       switch field {
-      | Field(f) => {
-          let fieldName = f->Table.getClickHouseDbFieldName
-          let fieldSchema = switch f.fieldType {
-          | Date => {
-              let dateSchema = Utils.Schema.clickHouseDate->S.toUnknown
-              if f.isNullable {
-                S.null(dateSchema)->S.toUnknown
-              } else if f.isArray {
-                S.array(dateSchema)->S.toUnknown
-              } else {
-                dateSchema
-              }
-            }
-          | ChainId => ChainId.schema->S.toUnknown
-          // ClickHouse returns UInt64 values as strings, need to parse to float
-          | UInt52 => {
-              let uint52Schema =
-                S.float
-                ->S.preprocess(
-                  _ => {
-                    parser: unknown => unknown->(Utils.magic: unknown => string)->Float.parseFloat,
-                  },
-                )
-                ->S.toUnknown
-              if f.isNullable {
-                S.null(uint52Schema)->S.toUnknown
-              } else if f.isArray {
-                S.array(uint52Schema)->S.toUnknown
-              } else {
-                uint52Schema
-              }
-            }
-          | _ => f.fieldSchema
-          }
-          dict->Dict.set(f->Table.getApiFieldName, s.field(fieldName, fieldSchema))
-        }
-      | DerivedFrom(_) => () // Skip derived fields
+      | Field(f) =>
+        dict->Dict.set(
+          f->Table.getApiFieldName,
+          s.field(f->Table.getClickHouseDbFieldName, clickHouseFieldSchema(f)),
+        )
+      | DerivedFrom(_) => ()
       }
     })
     dict->(Utils.magic: dict<unknown> => Internal.entity)
@@ -172,604 +35,409 @@ let makeClickHouseEntitySchema = (table: Table.table): S.t<Internal.entity> => {
 
 let logger = Logging.createChild(~params={"context": "ClickHouse"})
 
-// On transient failure, split values in half and retry each half.
-// If only 1 row remains, retry with delay.
-// Delay scales from 100ms to 1000ms as retries decrease.
-let rec insertWithRetry = async (
-  client,
-  ~table: string,
-  ~values: array<'a>,
-  ~format: string,
-  ~retries=8,
-) => {
-  try {
-    await client->insert({table, values, format})
-  } catch {
-  | exn if retries > 0 =>
-    let delayMs = Math.Int.min(1000, 100 + 900 * (8 - retries) / 7)
-    if Array.length(values) > 1 {
-      logger->Logging.childWarn({
-        "msg": "ClickHouse insert failed, splitting batch in half and retrying",
-        "table": table,
-        "batchSize": Array.length(values),
-        "retriesLeft": retries,
-        "err": exn->Utils.prettifyExn,
-      })
-      await Utils.delay(delayMs)
-      let mid = Array.length(values) / 2
-      let first = values->Array.slice(~start=0, ~end=mid)
-      let second = values->Array.slice(~start=mid)
-      await insertWithRetry(client, ~table, ~values=first, ~format, ~retries=retries - 1)
-      await insertWithRetry(client, ~table, ~values=second, ~format, ~retries=retries - 1)
-    } else {
-      logger->Logging.childWarn({
-        "msg": "ClickHouse insert failed, retrying after delay",
-        "table": table,
-        "retriesLeft": retries,
-        "err": exn->Utils.prettifyExn,
-      })
-      await Utils.delay(delayMs)
-      await insertWithRetry(client, ~table, ~values, ~format, ~retries=retries - 1)
-    }
-  }
+let makeSink = (~host, ~username, ~password, ~database, ~chainIdMode) =>
+  ClickHouseSink.make(~url=host, ~username, ~password, ~database, ~chainIdMode, ~onWarning=msg =>
+    logger->Logging.childWarn({"msg": msg})
+  )
+
+let makeColumnSpec = (
+  ~name,
+  ~fieldName=name,
+  ~fieldType: Table.fieldType,
+  ~isNullable=false,
+  ~isArray=false,
+): ClickHouseSink.columnSpec => {
+  name,
+  fieldName,
+  fieldType: switch fieldType {
+  | String => "String"
+  | Boolean => "Boolean"
+  | Uint32 => "Uint32"
+  | UInt52 => "UInt52"
+  | SmallInt =>
+    JsError.throwWithMessage("ClickHouse doesn't support the internal SmallInt column type")
+  | Bytea => "Bytea"
+  | UInt64 => "UInt64"
+  | Int32 => "Int32"
+  | ChainId => "ChainId"
+  | Number => "Number"
+  | Serial => "Serial"
+  | BigSerial => "BigSerial"
+  | Json => "Json"
+  | Date => "Date"
+  | BigInt(_) => "BigInt"
+  | BigDecimal(_) => "BigDecimal"
+  | Enum(_) => "Enum"
+  },
+  isNullable,
+  isArray,
+  precision: ?switch fieldType {
+  | BigInt({?precision}) => precision
+  | BigDecimal({?config}) => config->Option.map(((precision, _)) => precision)
+  | _ => None
+  },
+  scale: ?switch fieldType {
+  | BigDecimal({?config}) => config->Option.map(((_, scale)) => scale)
+  | _ => None
+  },
+  enumVariants: ?switch fieldType {
+  | Enum({config}) =>
+    Some(config.variants->Array.map(variant => variant->(Utils.magic: Table.enum => string)))
+  | _ => None
+  },
 }
 
-let setCheckpointsOrThrow = async (client, ~batch: Batch.t, ~database: string) => {
-  let checkpointsCount = batch.checkpointIds->Array.length
-  if checkpointsCount === 0 {
-    ()
-  } else {
-    // Convert columnar data to row format for JSONCompactEachRow
-    let checkpointRows = []
-    for idx in 0 to checkpointsCount - 1 {
-      checkpointRows
-      ->Array.push((
-        batch.checkpointIds->Array.getUnsafe(idx)->BigInt.toString,
-        batch.checkpointChainIds->Array.getUnsafe(idx),
-        batch.checkpointBlockNumbers->Array.getUnsafe(idx),
-        batch.checkpointBlockHashes->Array.getUnsafe(idx),
-        batch.checkpointEventsProcessed->Array.getUnsafe(idx),
-      ))
-      ->ignore
-    }
+type checkpointColumn = {
+  spec: ClickHouseSink.columnSpec,
+  valuesOf: Batch.t => array<unknown>,
+}
 
-    try {
-      await insertWithRetry(
-        client,
-        ~table=`${database}.\`${InternalTable.Checkpoints.table.tableName}\``,
-        ~values=checkpointRows,
-        ~format="JSONCompactEachRow",
-      )
-    } catch {
-    | exn =>
-      throw(
-        Persistence.StorageError({
-          message: `Failed to insert checkpoints into ClickHouse table "${InternalTable.Checkpoints.table.tableName}"`,
-          reason: exn->Utils.prettifyExn,
-        }),
-      )
-    }
+// Registration and staging both read this one list, so the column a batch's
+// values are written to is the column that was registered in its place.
+let checkpointColumns = InternalTable.Checkpoints.columns->Array.filterMap(({
+  field,
+  clickHouseFieldType,
+  valuesOf,
+}) =>
+  switch field {
+  | Table.Field({fieldName, isNullable}) =>
+    Some({
+      spec: makeColumnSpec(~name=fieldName, ~fieldType=clickHouseFieldType, ~isNullable),
+      valuesOf,
+    })
+  | DerivedFrom(_) => None
   }
-}
+)
 
-type setUpdatesCache = {
-  tableName: string,
-  convertOrThrow: array<Change.t<Internal.entity>> => array<JSON.t>,
-}
+let checkpointColumnSpecs = checkpointColumns->Array.map(({spec}) => spec)
 
-let setUpdatesOrThrow = async (
-  client,
-  ~cache: dict<setUpdatesCache>,
-  ~changes: array<Change.t<Internal.entity>>,
-  ~entityConfig: Internal.entityConfig,
-  ~scope: Internal.chainScope,
-  ~database: string,
-) => {
-  if changes->Array.length === 0 {
-    ()
-  } else {
-    let chainIdField = entityConfig.table->Table.getChainIdField
-    let scopeChainId = scope->Internal.chainScopeChainId
-    // A delete row carries no entity to stamp, so the chain id is baked into
-    // the schema instead — which is why the cache is keyed per scope.
-    let cacheKey = `${entityConfig.name}|${scope->Internal.chainScopeToString}`
-    let {convertOrThrow, tableName} = switch cache->Utils.Dict.dangerouslyGetNonOption(cacheKey) {
-    | Some(cached) => cached
-    | None =>
-      let cached: setUpdatesCache = {
-        tableName: `${database}.\`${EntityHistory.historyTableName(
-            ~entityName=entityConfig.name,
-            ~entityIndex=entityConfig.index,
-          )}\``,
-        convertOrThrow: S.compile(
-          S.array(
-            S.union([
-              EntityHistory.makeSetUpdateSchema(
-                ~idSchema=entityConfig.table->Table.getIdSchema,
-                makeClickHouseEntitySchema(entityConfig.table),
-              ),
-              S.object(s => {
-                s.tag(EntityHistory.changeFieldName, EntityHistory.RowAction.DELETE)
-                switch (chainIdField, scopeChainId) {
-                | (Some(field), Some(chainId)) =>
-                  s.tag(field->Table.getClickHouseDbFieldName, chainId)
-                | _ => ()
-                }
-                Change.Delete({
-                  entityId: s.field(Table.idFieldName, entityConfig.table->Table.getIdSchema),
-                  checkpointId: s.field(
-                    EntityHistory.checkpointIdFieldName,
-                    EntityHistory.unsafeCheckpointIdSchema,
-                  ),
-                })
-              }),
-            ]),
+let entitySpec = (~entityConfig: Internal.entityConfig): ClickHouseSink.entitySpec => {
+  let options = entityConfig.storage.clickhouseOptions
+  {
+    name: entityConfig.name,
+    historyTable: EntityHistory.historyTableName(
+      ~entityName=entityConfig.name,
+      ~entityIndex=entityConfig.index,
+    ),
+    columns: entityConfig.table.fields->Array.filterMap(field =>
+      switch field {
+      | Table.Field(f) =>
+        Some(
+          makeColumnSpec(
+            ~name=f->Table.getClickHouseDbFieldName,
+            ~fieldName=f.fieldName,
+            ~fieldType=f.fieldType,
+            ~isNullable=f.isNullable,
+            ~isArray=f.isArray,
           ),
-          ~input=Value,
-          ~output=Json,
-          ~typeValidation=false,
-          ~mode=Sync,
-        )->(
-          Utils.magic: (array<Change.t<Internal.entity>> => JSON.t) => array<
-            Change.t<Internal.entity>,
-          > => array<JSON.t>
-        ),
+        )
+      | DerivedFrom(_) => None
       }
-
-      cache->Dict.set(cacheKey, cached)
-      cached
-    }
-
-    let changes = switch (chainIdField, scopeChainId) {
-    | (Some(field), Some(chainId)) =>
-      changes->Array.map(change =>
-        switch change {
-        | Change.Set(set) =>
-          Change.Set({
-            ...set,
-            entity: set.entity->Internal.stampChainId(~fieldName=field.fieldName, ~chainId),
-          })
-        | Delete(_) => change
-        }
+    ),
+    chainIdColumn: ?(
+      entityConfig.table->Table.getChainIdField->Option.map(Table.getClickHouseDbFieldName)
+    ),
+    partitionBy: ?(options->Option.flatMap(options => options.partitionBy)),
+    orderBy: ?(options->Option.flatMap(options => options.orderBy)),
+    ttl: ?(options->Option.flatMap(options => options.ttl)),
+    skippingIndexes: ?(
+      options->Option.flatMap(options =>
+        options.skippingIndexes->Option.map(indexes =>
+          indexes->Array.map(
+            (index): ClickHouseSink.skippingIndexSpec => {
+              name: index.name,
+              expr: index.expr,
+              indexType: index.type_,
+              granularity: ?index.granularity,
+            },
+          )
+        )
       )
-    | _ => changes
-    }
+    ),
+  }
+}
 
-    try {
-      // The entity history table is the source of truth for ClickHouse, so every
-      // intermediate change must be persisted, not only the current value.
-      let values = changes->convertOrThrow
+type cell = Change.t<Internal.entity> => unknown
 
-      await insertWithRetry(client, ~table=tableName, ~values, ~format="JSONEachRow")
-    } catch {
-    | exn =>
-      throw(
-        Persistence.StorageError({
-          message: `Failed to insert items into ClickHouse table "${tableName}"`,
-          reason: exn->Utils.prettifyExn,
-        }),
-      )
-    }
-  }
+type converters = {
+  setCells: array<cell>,
+  deleteCells: array<cell>,
 }
 
-// A plain database created with ON CLUSTER doesn't turn subsequent DDL into
-// cluster-wide statements; ClickHouse keeps no "this database is clustered"
-// flag. Without a Replicated database engine, every CREATE must carry its own
-// ON CLUSTER to reach all replicas, otherwise it runs only on the connected
-// node. With a Replicated database engine the DDL propagates via the database's
-// own log, and combining it with ON CLUSTER is rejected/double-applied — so
-// table-level DDL must carry the clause only in the plain-database case.
-// The '{cluster}' macro resolves to each node's configured cluster name.
-let onClusterClause = (~onCluster: bool) => onCluster ? ` ON CLUSTER '{cluster}'` : ""
+type registry = {
+  entities: dict<ClickHouseSink.table>,
+  mutable checkpoints: option<ClickHouseSink.table>,
+  converters: dict<converters>,
+}
 
-// ReplicatedMergeTree drops an insert whose block hash is already in Keeper, and
-// mutations don't clear those hashes. Crash recovery trims the history tail past
-// the committed Postgres checkpoint with ALTER ... DELETE and then replays it, so
-// an identical replayed block would be discarded while still reporting success —
-// a permanent gap that nothing surfaces. Trim-then-replay is what makes recovery
-// correct here, and the duplicates dedup would have caught are already collapsed
-// by the entity view's `LIMIT 1 BY`. Plain MergeTree goes by
-// non_replicated_deduplication_window (0 by default), so it needs no clause.
-let replicatedTableSettingsClause = (~replicated: bool) =>
-  replicated ? "\nSETTINGS replicated_deduplication_window = 0" : ""
+let makeRegistry = () => {
+  entities: Dict.make(),
+  checkpoints: None,
+  converters: Dict.make(),
+}
 
-// Strip both engine arguments `(...)` and a trailing `SETTINGS ...` clause to
-// get the bare engine name, e.g. `Replicated('/p','{shard}','{replica}') SETTINGS x=1`
-// and `Replicated SETTINGS x=1` both yield `Replicated`.
-let databaseEngineName = (engineSpec: string) =>
-  engineSpec
-  ->String.trim
-  ->String.split("(")
-  ->Array.getUnsafe(0)
-  ->String.split(" ")
-  ->Array.getUnsafe(0)
-  ->String.trim
+@get external changeEntity: Change.t<Internal.entity> => dict<unknown> = "entity"
 
-// Generate CREATE TABLE query for entity history table
-let makeCreateHistoryTableQuery = (
-  ~entityConfig: Internal.entityConfig,
-  ~database: string,
-  ~replicated: bool=false,
-  ~onCluster: bool=false,
-  ~chainIdMode: ChainId.mode=Int32,
-) => {
-  let tableEngine = replicated ? "ReplicatedMergeTree" : "MergeTree()"
-  let fieldDefinitions = entityConfig.table.fields->Array.filterMap(field => {
-    switch field {
-    | Field(field) =>
-      Some({
-        let fieldName = field->Table.getClickHouseDbFieldName
-        let clickHouseType = getClickHouseFieldType(
-          ~fieldType=field.fieldType,
-          ~isNullable=field.isNullable,
-          ~isArray=field.isArray,
-          ~chainIdMode,
-        )
-        `\`${fieldName}\` ${clickHouseType}`
-      })
-    | DerivedFrom(_) => None
-    }
-  })
+%%private(let absent: unknown = %raw(`undefined`))
 
-  let (partitionBy, orderBy, ttl, skippingIndexes) = switch entityConfig.storage.clickhouseOptions {
-  | Some(options) => (options.partitionBy, options.orderBy, options.ttl, options.skippingIndexes)
-  | None => (None, None, None, None)
-  }
+%%private(
+  let compileSerializer = schema =>
+    S.compile(schema, ~input=Value, ~output=Json, ~typeValidation=false, ~mode=Sync)->(
+      Utils.magic: (unknown => JSON.t) => unknown => unknown
+    )
+)
 
-  // Schema field name -> ClickHouse column name, so @storage(clickhouse: {...})
-  // options can reference fields the way they're written in the schema and get
-  // renames (`column_name_format: snake_case`) and linked-entity `_id` suffixes
-  // resolved here.
-  let columnByFieldName = Dict.make()
-  entityConfig.table.fields->Array.forEach(field =>
-    switch field {
-    | Field(f) => columnByFieldName->Dict.set(f.fieldName, f->Table.getClickHouseDbFieldName)
-    | DerivedFrom(_) => ()
+%%private(
+  let makeConverters = (
+    ~entityConfig: Internal.entityConfig,
+    ~scope: Internal.chainScope,
+    ~table: ClickHouseSink.table,
+  ): converters => {
+    let {name: tableName, columns} = table
+    let sourceTable = entityConfig.table
+    let chainIdTag = switch (
+      sourceTable->Table.getChainIdField,
+      scope->Internal.chainScopeChainId,
+    ) {
+    | (Some(field), Some(chainId)) =>
+      Some((field->Table.getClickHouseDbFieldName, chainId->(Utils.magic: ChainId.t => unknown)))
+    | _ => None
     }
-  )
+    let fieldsByColumn = Dict.make()
+    sourceTable.fields->Array.forEach(field =>
+      switch field {
+      | Table.Field(f) => fieldsByColumn->Dict.set(f->Table.getClickHouseDbFieldName, f)
+      | DerivedFrom(_) => ()
+      }
+    )
 
-  let orderByColumns = switch orderBy {
-  | Some(fieldNames) =>
-    // envio_checkpoint_id stays appended so the sorting key keeps a
-    // deterministic tie-break and the view's checkpoint dedup gets a clean
-    // ascending run per prefix. id is dropped: ClickHouse entities are
-    // read-only, so nothing looks history rows up by id.
-    let userColumns =
-      fieldNames
-      ->Array.map(fieldName =>
-        switch columnByFieldName->Dict.get(fieldName) {
-        | Some(column) => `\`${column}\``
-        | None =>
-          // Validated at codegen, so a miss means the schema and the
-          // persisted config diverged.
+    let setCells = []
+    let deleteCells = []
+    let registeredNames = columns->Array.map(column => column.name)->Utils.Set.fromArray
+    columns->Array.forEach(({name}) => {
+      let (set, delete) = if name === EntityHistory.checkpointIdFieldName {
+        let cell = change => change->Change.getCheckpointId->(Utils.magic: bigint => unknown)
+        (cell, cell)
+      } else if name === EntityHistory.changeFieldName {
+        (
+          _ => (EntityHistory.RowAction.SET :> string)->(Utils.magic: string => unknown),
+          _ => (EntityHistory.RowAction.DELETE :> string)->(Utils.magic: string => unknown),
+        )
+      } else {
+        switch (chainIdTag, fieldsByColumn->Utils.Dict.dangerouslyGetNonOption(name)) {
+        | (Some((column, chainId)), _) if column === name => (_ => chainId, _ => chainId)
+        | (_, Some(f)) =>
+          let serialize = compileSerializer(clickHouseFieldSchema(f))
+          let apiName = f->Table.getApiFieldName
+          if apiName === Table.idFieldName {
+            let cell = change =>
+              serialize(change->Change.getEntityId->(Utils.magic: EntityId.t => unknown))
+            (cell, cell)
+          } else {
+            (change => serialize(change->changeEntity->Dict.getUnsafe(apiName)), _ => absent)
+          }
+        | (_, None) =>
           JsError.throwWithMessage(
-            `ClickHouse orderBy field "${fieldName}" is not defined on entity "${entityConfig.name}"`,
+            `ClickHouse table "${tableName}" registered a column "${name}" that entity "${entityConfig.name}" has no value for`,
           )
         }
-      )
-      ->Array.joinUnsafe(", ")
-    `${userColumns}, ${EntityHistory.checkpointIdFieldName}`
-  | None => `${Table.idFieldName}, ${EntityHistory.checkpointIdFieldName}`
-  }
+      }
+      setCells->Array.push(set)
+      deleteCells->Array.push(delete)
+    })
 
-  // partitionBy/ttl are raw ClickHouse expressions. Rewrite any bare identifier
-  // that names an entity field to that field's ClickHouse column, leaving
-  // functions, keywords, numbers, string literals and already-backticked
-  // identifiers untouched (a quoted token never matches a bare field name).
-  let resolveExpressionColumns = expression =>
-    expression->String.replaceRegExpBy0Unsafe(/'[^']*'|`[^`]*`|[A-Za-z_][A-Za-z0-9_]*/g, (
-      ~match,
-      ~offset as _,
-      ~input as _,
-    ) =>
-      switch columnByFieldName->Dict.get(match) {
-      | Some(column) => `\`${column}\``
-      | None => match
+    fieldsByColumn->Dict.forEachWithKey((f, column) =>
+      if !(registeredNames->Utils.Set.has(column)) {
+        JsError.throwWithMessage(
+          `ClickHouse table "${tableName}" registered no column for field "${f.fieldName}" of entity "${entityConfig.name}"`,
+        )
       }
     )
 
-  let partitionByClause = switch partitionBy {
-  | Some(expression) => `\nPARTITION BY ${expression->resolveExpressionColumns}`
-  | None => ""
-  }
-  let ttlClause = switch ttl {
-  | Some(expression) => `\nTTL ${expression->resolveExpressionColumns}`
-  | None => ""
+    {setCells, deleteCells}
   }
+)
 
-  // Data skipping indexes live inside the column list, after the last column.
-  // GRANULARITY is omitted when unset, leaving ClickHouse's default of 1.
-  let skippingIndexDefinitions = switch skippingIndexes {
-  | Some(skippingIndexes) =>
-    skippingIndexes
-    ->Array.map(index => {
-      let granularityClause = switch index.granularity {
-      | Some(granularity) => ` GRANULARITY ${granularity->Int.toString}`
-      | None => ""
+%%private(
+  let fillBuilders = (
+    ~table: ClickHouseSink.table,
+    ~converters,
+    ~changes: array<Change.t<Internal.entity>>,
+  ) => {
+    let rows = changes->Array.length
+    let builders = table.columns->Array.map(ClickHouseSink.makeBuilder(_, ~rows))
+    let columns = builders->Array.length
+    for row in 0 to rows - 1 {
+      let change = changes->Array.getUnsafe(row)
+      let (cells, write) = switch change {
+      | Change.Set(_) => (converters.setCells, ClickHouseSink.writeValue)
+      | Delete(_) => (converters.deleteCells, ClickHouseSink.writeDeletedValue)
       }
-      `,\n  INDEX \`${index.name}\` ${index.expr->resolveExpressionColumns} TYPE ${index.type_}${granularityClause}`
-    })
-    ->Array.joinUnsafe("")
-  | None => ""
+      for column in 0 to columns - 1 {
+        builders->Array.getUnsafe(column)->write(~row, (cells->Array.getUnsafe(column))(change))
+      }
+    }
+    builders
   }
-
-  `CREATE TABLE IF NOT EXISTS ${database}.\`${EntityHistory.historyTableName(
-      ~entityName=entityConfig.name,
-      ~entityIndex=entityConfig.index,
-    )}\`${onClusterClause(~onCluster)} (
-  ${fieldDefinitions->Array.joinUnsafe(",\n  ")},
-  \`${EntityHistory.checkpointIdFieldName}\` ${getClickHouseFieldType(
-      ~fieldType=UInt64,
-      ~isNullable=false,
-      ~isArray=false,
-    )},
-  \`${EntityHistory.changeFieldName}\` ${getClickHouseFieldType(
-      ~fieldType=Enum({config: EntityHistory.RowAction.config->Table.fromGenericEnumConfig}),
-      ~isNullable=false,
-      ~isArray=false,
-    )}${skippingIndexDefinitions}
 )
-ENGINE = ${tableEngine}${partitionByClause}
-ORDER BY (${orderByColumns})${ttlClause}${replicatedTableSettingsClause(~replicated)}`
-}
 
-// Generate CREATE TABLE query for checkpoints
-let makeCreateCheckpointsTableQuery = (
-  ~database: string,
-  ~replicated: bool=false,
-  ~onCluster: bool=false,
-  ~chainIdMode: ChainId.mode=Int32,
-) => {
-  let tableEngine = replicated ? "ReplicatedMergeTree" : "MergeTree()"
-  let idField = (#id: InternalTable.Checkpoints.field :> string)
-  let chainIdField = (#chain_id: InternalTable.Checkpoints.field :> string)
-  let blockNumberField = (#block_number: InternalTable.Checkpoints.field :> string)
-  let blockHashField = (#block_hash: InternalTable.Checkpoints.field :> string)
-  let eventsProcessedField = (#events_processed: InternalTable.Checkpoints.field :> string)
+let entityTable = (sink, ~registry, ~entityConfig: Internal.entityConfig) =>
+  switch registry.entities->Utils.Dict.dangerouslyGetNonOption(entityConfig.name) {
+  | Some(table) => table
+  | None =>
+    let spec = entitySpec(~entityConfig)
+    let table =
+      sink
+      ->ClickHouseSink.registerEntityTable(spec)
+      ->ClickHouseSink.makeTable(~name=spec.historyTable)
+    registry.entities->Dict.set(entityConfig.name, table)
+    table
+  }
 
-  `CREATE TABLE IF NOT EXISTS ${database}.\`${InternalTable.Checkpoints.table.tableName}\`${onClusterClause(
-      ~onCluster,
-    )} (
-  \`${idField}\` ${getClickHouseFieldType(~fieldType=UInt64, ~isNullable=false, ~isArray=false)},
-  \`${chainIdField}\` ${getClickHouseFieldType(
-      ~fieldType=ChainId,
-      ~isNullable=false,
-      ~isArray=false,
-      ~chainIdMode,
-    )},
-  \`${blockNumberField}\` ${getClickHouseFieldType(
-      ~fieldType=Int32,
-      ~isNullable=false,
-      ~isArray=false,
-    )},
-  \`${blockHashField}\` ${getClickHouseFieldType(
-      ~fieldType=String,
-      ~isNullable=true,
-      ~isArray=false,
-    )},
-  \`${eventsProcessedField}\` ${getClickHouseFieldType(
-      ~fieldType=UInt64,
-      ~isNullable=false,
-      ~isArray=false,
-    )}
-)
-ENGINE = ${tableEngine}
-ORDER BY (${idField})${replicatedTableSettingsClause(~replicated)}`
-}
+let checkpointsTable = (sink, ~registry) =>
+  switch registry.checkpoints {
+  | Some(table) => table
+  | None =>
+    let table =
+      sink
+      ->ClickHouseSink.registerCheckpointsTable(checkpointColumnSpecs)
+      ->ClickHouseSink.makeTable(~name=InternalTable.Checkpoints.table.tableName)
+    registry.checkpoints = Some(table)
+    table
+  }
 
-// Generate CREATE VIEW query for entity current state
-let makeCreateViewQuery = (
-  ~entityConfig: Internal.entityConfig,
-  ~database: string,
-  ~onCluster: bool=false,
-) => {
-  let historyTableName = EntityHistory.historyTableName(
-    ~entityName=entityConfig.name,
-    ~entityIndex=entityConfig.index,
+let stageBuilders = (sink, ~table: ClickHouseSink.table, ~builders, ~rows) =>
+  sink->ClickHouseSink.stage(
+    ~table=table.handle,
+    ~rows,
+    ~columns=builders->Array.map(ClickHouseSink.builderPayload),
   )
 
-  // A per-chain entity's rows are only comparable within a chain, so the
-  // current-state dedup keys on (id, chain id).
-  let dedupKey =
-    switch entityConfig.table->Table.getChainIdField {
-    | Some(field) => [Table.idFieldName, field->Table.getClickHouseDbFieldName]
-    | None => [Table.idFieldName]
-    }
-    ->Array.map(name => `\`${name}\``)
-    ->Array.joinUnsafe(", ")
-
-  let checkpointsTableName = InternalTable.Checkpoints.table.tableName
-  let checkpointIdField = (#id: InternalTable.Checkpoints.field :> string)
-
-  let entityFields =
-    entityConfig.table.fields
-    ->Array.filterMap(field => {
-      switch field {
-      | Field(field) => {
-          let fieldName = field->Table.getClickHouseDbFieldName
-          Some(`\`${fieldName}\``)
+let stageCheckpointsOrThrow = (sink, ~registry, ~batch: Batch.t) => {
+  let rows = batch.checkpointIds->Array.length
+  if rows === 0 {
+    Null.null
+  } else {
+    let table = sink->checkpointsTable(~registry)
+    try {
+      // The table was registered from `checkpointColumns`, in this order.
+      let builders = table.columns->Array.map(ClickHouseSink.makeBuilder(_, ~rows))
+      builders->Array.forEachWithIndex((builder, index) => {
+        let columnValues = (checkpointColumns->Array.getUnsafe(index)).valuesOf(batch)
+        for row in 0 to rows - 1 {
+          builder->ClickHouseSink.writeValue(~row, columnValues->Array.getUnsafe(row))
         }
-      | DerivedFrom(_) => None
-      }
-    })
-    ->Array.joinUnsafe(", ")
-
-  `CREATE VIEW IF NOT EXISTS ${database}.\`${entityConfig.name}\`${onClusterClause(~onCluster)} AS
-SELECT ${entityFields}
-FROM (
-  SELECT ${entityFields}, \`${EntityHistory.changeFieldName}\`
-  FROM ${database}.\`${historyTableName}\`
-  WHERE \`${EntityHistory.checkpointIdFieldName}\` <= (SELECT max(${checkpointIdField}) FROM ${database}.\`${checkpointsTableName}\`)
-  ORDER BY \`${EntityHistory.checkpointIdFieldName}\` DESC
-  LIMIT 1 BY ${dedupKey}
-)
-WHERE \`${EntityHistory.changeFieldName}\` = '${(EntityHistory.RowAction.SET :> string)}'`
-}
-
-// Initialize ClickHouse tables for entities
-let initialize = async (
-  client,
-  ~database: string,
-  ~entities: array<Internal.entityConfig>,
-  ~enums as _: array<Table.enumConfig<Table.enum>>,
-  ~chainIdMode: ChainId.mode=Int32,
-) => {
-  try {
-    let databaseEngine = Env.ClickHouse.databaseEngine()
-    let databaseEngineClause = switch databaseEngine {
-    | Some(engine) => ` ENGINE = ${engine}`
-    | None => ""
-    }
-    let hasReplicatedDatabaseEngine = switch databaseEngine {
-    | Some(engine) => engine->databaseEngineName === "Replicated"
-    | None => false
-    }
-    let envReplicated = Env.ClickHouse.replicated()
-    // A Replicated database engine only replicates data when its tables use the
-    // ReplicatedMergeTree engine, so it implies replicated mode even when
-    // ENVIO_CLICKHOUSE_REPLICATED is unset.
-    let replicated = envReplicated || hasReplicatedDatabaseEngine
-    if hasReplicatedDatabaseEngine && !envReplicated {
-      Logging.info(
-        "ENVIO_CLICKHOUSE_DATABASE_ENGINE is Replicated; enabling replicated mode so tables use the ReplicatedMergeTree engine.",
+      })
+      Null.make(sink->stageBuilders(~table, ~builders, ~rows))
+    } catch {
+    | exn =>
+      throw(
+        Persistence.StorageError({
+          message: `Failed to convert checkpoints for ClickHouse table "${table.name}"`,
+          reason: exn->Utils.prettifyExn,
+        }),
       )
     }
-    let databaseOnClusterClause = onClusterClause(~onCluster=replicated)
-    // DDL that a Replicated database engine propagates itself must not carry
-    // ON CLUSTER on top of it — the clause is only for the plain-database case.
-    let ddlOnCluster = replicated && !hasReplicatedDatabaseEngine
+  }
+}
 
-    switch databaseEngine {
-    | Some(engineSpec) => {
-        let expectedEngineName = engineSpec->databaseEngineName
-        let existingResult = await client->query({
-          query: `SELECT engine FROM system.databases WHERE name = '${database}'`,
-        })
-        let rows = (await existingResult->json)["data"]
-        switch rows->Array.get(0) {
-        | Some(row) if row["engine"] !== expectedEngineName =>
-          JsError.throwWithMessage(
-            `ClickHouse database "${database}" exists with engine "${row["engine"]}" but ENVIO_CLICKHOUSE_DATABASE_ENGINE specifies "${expectedEngineName}". Drop the database manually to change its engine.`,
-          )
-        | _ => ()
-        }
+let stageUpdatesOrThrow = (
+  sink,
+  ~registry,
+  ~changes: array<Change.t<Internal.entity>>,
+  ~entityConfig: Internal.entityConfig,
+  ~scope: Internal.chainScope,
+) => {
+  let rows = changes->Array.length
+  if rows === 0 {
+    None
+  } else {
+    let table = sink->entityTable(~registry, ~entityConfig)
+    let tableName = table.name
+    let cacheKey = `${entityConfig.name}|${scope->Internal.chainScopeToString}`
+    try {
+      let converters = switch registry.converters->Utils.Dict.dangerouslyGetNonOption(cacheKey) {
+      | Some(cached) => cached
+      | None =>
+        let cached = makeConverters(~entityConfig, ~scope, ~table)
+        registry.converters->Dict.set(cacheKey, cached)
+        cached
       }
-    | None => ()
+      let builders = fillBuilders(~table, ~converters, ~changes)
+      Some(sink->stageBuilders(~table, ~builders, ~rows))
+    } catch {
+    | exn =>
+      throw(
+        Persistence.StorageError({
+          message: `Failed to convert items for ClickHouse table "${tableName}"`,
+          reason: exn->Utils.prettifyExn,
+        }),
+      )
     }
+  }
+}
 
-    if hasReplicatedDatabaseEngine {
-      // TRUNCATE DATABASE is unsupported on Replicated databases, so a reset
-      // has to DROP and recreate instead (plain databases keep the TRUNCATE
-      // fallback below). This requires the ClickHouse user to hold the DROP
-      // privilege; without it the reset fails here with ACCESS_DENIED. ON
-      // CLUSTER removes the database from every node — the engine's own log
-      // can't replicate the drop of the database it lives in — and SYNC waits
-      // for the drop to finish before the CREATE below.
-      await client->command({
-        query: `DROP DATABASE IF EXISTS ${database} ON CLUSTER '{cluster}' SYNC`,
-      })
-    } else {
-      await client->command({
-        query: `TRUNCATE DATABASE IF EXISTS ${database}${onClusterClause(~onCluster=ddlOnCluster)}`,
-      })
-    }
-    await client->command({
-      query: `CREATE DATABASE IF NOT EXISTS ${database}${databaseOnClusterClause}${databaseEngineClause}`,
-    })
+let writeStagedOrThrow = async (sink, ~entities, ~checkpoints) =>
+  try await sink->ClickHouseSink.writeBatch(~entities, ~checkpoints) catch {
+  | exn =>
+    throw(
+      Persistence.StorageError({
+        message: "Failed to write a batch to ClickHouse",
+        reason: exn->Utils.prettifyExn,
+      }),
+    )
+  }
 
-    await Promise.all(
-      entities->Array.map(entityConfig =>
-        client->command({
-          query: makeCreateHistoryTableQuery(
-            ~entityConfig,
-            ~database,
-            ~replicated,
-            ~onCluster=ddlOnCluster,
-            ~chainIdMode,
-          ),
-        })
-      ),
-    )->Utils.Promise.ignoreValue
-    await client->command({
-      query: makeCreateCheckpointsTableQuery(
-        ~database,
-        ~replicated,
-        ~onCluster=ddlOnCluster,
-        ~chainIdMode,
-      ),
+// Tables are not registered here: an indexer that finds an existing storage
+// never runs this, so the write path registers them on first use either way.
+let initialize = async (sink, ~entities: array<Internal.entityConfig>) => {
+  try {
+    await sink->ClickHouseSink.initialize({
+      entities: entities->Array.map(entityConfig => entitySpec(~entityConfig)),
+      checkpointColumns: checkpointColumnSpecs,
+      replicated: Env.ClickHouse.replicated(),
+      databaseEngine: ?Env.ClickHouse.databaseEngine(),
     })
 
-    // The client pools HTTP connections, so consecutive statements may reach
-    // different replicas, while a Replicated database applies DDL from its
-    // Keeper log asynchronously. A CREATE VIEW is analyzed against the node's
-    // local metadata and can land on a replica that hasn't applied the table
-    // creates yet, failing with UNKNOWN_TABLE. Block until every replica has
-    // caught up before creating the views. ON CLUSTER must precede the
-    // database name in this command's grammar.
-    if hasReplicatedDatabaseEngine {
-      await client->command({
-        query: `SYSTEM SYNC DATABASE REPLICA ON CLUSTER '{cluster}' ${database}`,
-      })
-    }
-
-    await Promise.all(
-      entities->Array.map(entityConfig =>
-        client->command({
-          query: makeCreateViewQuery(~entityConfig, ~database, ~onCluster=ddlOnCluster),
-        })
-      ),
-    )->Utils.Promise.ignoreValue
-
     Logging.trace("ClickHouse storage initialization completed successfully")
   } catch {
   | exn => {
       Logging.errorWithExn(exn, "Failed to initialize ClickHouse storage")
-      JsError.throwWithMessage("ClickHouse initialization failed")
+      throw(
+        Persistence.StorageError({
+          message: "Failed to initialize ClickHouse storage",
+          reason: exn->Utils.prettifyExn,
+        }),
+      )
     }
   }
 }
 
-// Resume ClickHouse sink after reorg by deleting rows with checkpoint IDs higher than target
-let resume = async (client, ~database: string, ~checkpointId: Internal.checkpointId) => {
-  try {
-    // Try to use the database - will throw if it doesn't exist
-    try {
-      await client->command({query: `USE ${database}`})
-    } catch {
-    | exn =>
-      Logging.errorWithExn(
-        exn,
-        `ClickHouse storage database "${database}" not found. Please run 'envio start -r' to reinitialize the indexer (it'll also drop Postgres database).`,
-      )
-      JsError.throwWithMessage("ClickHouse resume failed")
-    }
-
-    // Get all history tables
-    let tablesResult = await client->query({
-      query: `SHOW TABLES FROM ${database} LIKE '${EntityHistory.historyTablePrefix}%'`,
-    })
-    let tables = (await tablesResult->json)["data"]
-
-    // Delete rows with checkpoint IDs higher than the target for each history table
-    await Promise.all(
-      tables->Array.map(table => {
-        let tableName = table["name"]
-        client->command({
-          query: `ALTER TABLE ${database}.\`${tableName}\` DELETE WHERE \`${EntityHistory.checkpointIdFieldName}\` > ${checkpointId->BigInt.toString}`,
-        })
-      }),
-    )->Utils.Promise.ignoreValue
-
-    // Delete stale checkpoints
-    await client->command({
-      query: `DELETE FROM ${database}.\`${InternalTable.Checkpoints.table.tableName}\` WHERE \`${Table.idFieldName}\` > ${checkpointId->BigInt.toString}`,
-    })
-  } catch {
-  | Persistence.StorageError(_) as exn => throw(exn)
+let resume = async (
+  sink,
+  ~checkpointId: Internal.checkpointId,
+  ~chains: array<Persistence.initialChainState>,
+  ~entities: array<Internal.entityConfig>,
+) => {
+  let chainProgress = chains->Array.map(chain => {
+    ClickHouseSink.chainId: chain.id->ChainId.toString,
+    progressBlockNumber: chain.progressBlockNumber,
+  })
+  try await sink->ClickHouseSink.resume({
+    checkpointId: checkpointId->BigInt.toString,
+    chainProgress,
+    historyTables: entities->Array.map(entityConfig => entitySpec(~entityConfig).historyTable),
+    replicated: Env.ClickHouse.replicated(),
+    databaseEngine: ?Env.ClickHouse.databaseEngine(),
+  }) catch {
   | exn => {
       Logging.errorWithExn(exn, "Failed to resume ClickHouse storage")
-      JsError.throwWithMessage("ClickHouse resume failed")
+      throw(
+        Persistence.StorageError({
+          message: "Failed to resume ClickHouse storage",
+          reason: exn->Utils.prettifyExn,
+        }),
+      )
     }
   }
 }