envio 3.9.0 → 3.10.0

package/src/bindings/ClickHouse.res.mjs

@@ -7,533 +7,483 @@ import * as ChainId from "../ChainId.res.mjs";
 import * as Logging from "../Logging.res.mjs";
 import * as Internal from "../Internal.res.mjs";
 import * as Persistence from "../Persistence.res.mjs";
+import * as Stdlib_Dict from "@rescript/runtime/lib/es6/Stdlib_Dict.js";
 import * as Stdlib_Array from "@rescript/runtime/lib/es6/Stdlib_Array.js";
 import * as EntityHistory from "../db/EntityHistory.res.mjs";
 import * as InternalTable from "../db/InternalTable.res.mjs";
+import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js";
+import * as ClickHouseSink from "./ClickHouseSink.res.mjs";
 import * as Stdlib_JsError from "@rescript/runtime/lib/es6/Stdlib_JsError.js";
 import * as Primitive_option from "@rescript/runtime/lib/es6/Primitive_option.js";
 import * as S$RescriptSchema from "rescript-schema/src/S.res.mjs";
 import * as Primitive_exceptions from "@rescript/runtime/lib/es6/Primitive_exceptions.js";
 
-function getClickHouseFieldType(fieldType, isNullable, isArray, chainIdModeOpt) {
-  let chainIdMode = chainIdModeOpt !== undefined ? chainIdModeOpt : "int32";
-  let baseType;
+function clickHouseFieldSchema(f) {
+  let shaped = baseSchema => {
+    if (f.isNullable) {
+      return Utils.Schema.nullTolerant(baseSchema);
+    } else if (f.isArray) {
+      return S$RescriptSchema.array(baseSchema);
+    } else {
+      return baseSchema;
+    }
+  };
+  let match = f.fieldType;
+  if (typeof match === "object") {
+    return f.fieldSchema;
+  }
+  switch (match) {
+    case "Bytea" :
+      return shaped(S$RescriptSchema.json(false));
+    case "ChainId" :
+      return ChainId.schema;
+    case "Json" :
+      if (f.isArray) {
+        return f.fieldSchema;
+      } else {
+        return Utils.Schema.clickHouseJson;
+      }
+    case "Date" :
+      return shaped(Utils.Schema.clickHouseDate);
+    default:
+      return f.fieldSchema;
+  }
+}
+
+function makeClickHouseEntitySchema(table) {
+  return S$RescriptSchema.object(s => {
+    let dict = {};
+    table.fields.forEach(field => {
+      if (field.TAG !== "Field") {
+        return;
+      }
+      let f = field._0;
+      dict[Table.getApiFieldName(f)] = s.f(Table.getClickHouseDbFieldName(f), clickHouseFieldSchema(f));
+    });
+    return dict;
+  });
+}
+
+let logger = Logging.createChild({
+  context: "ClickHouse"
+});
+
+function makeSink(host, username, password, database, chainIdMode) {
+  return ClickHouseSink.make(host, username, password, database, chainIdMode, msg => Logging.childWarn(logger, {
+    msg: msg
+  }));
+}
+
+function makeColumnSpec(name, fieldNameOpt, fieldType, isNullableOpt, isArrayOpt) {
+  let fieldName = fieldNameOpt !== undefined ? fieldNameOpt : name;
+  let isNullable = isNullableOpt !== undefined ? isNullableOpt : false;
+  let isArray = isArrayOpt !== undefined ? isArrayOpt : false;
+  let tmp;
   if (typeof fieldType !== "object") {
     switch (fieldType) {
+      case "String" :
+        tmp = "String";
+        break;
       case "Boolean" :
-        baseType = "Bool";
+        tmp = "Boolean";
         break;
       case "Uint32" :
-        baseType = "UInt32";
+        tmp = "Uint32";
+        break;
+      case "UInt52" :
+        tmp = "UInt52";
         break;
       case "SmallInt" :
+        tmp = Stdlib_JsError.throwWithMessage("ClickHouse doesn't support the internal SmallInt column type");
+        break;
       case "Bytea" :
-        baseType = Stdlib_JsError.throwWithMessage("ClickHouse doesn't support the internal SmallInt and Bytea column types");
+        tmp = "Bytea";
         break;
-      case "UInt52" :
       case "UInt64" :
-        baseType = "UInt64";
+        tmp = "UInt64";
+        break;
+      case "Int32" :
+        tmp = "Int32";
         break;
       case "ChainId" :
-        baseType = chainIdMode === "int32" ? "Int32" : "UInt64";
+        tmp = "ChainId";
         break;
       case "Number" :
-        baseType = "Float64";
+        tmp = "Number";
         break;
-      case "Int32" :
       case "Serial" :
-        baseType = "Int32";
+        tmp = "Serial";
         break;
       case "BigSerial" :
-        baseType = "Int64";
+        tmp = "BigSerial";
         break;
-      case "String" :
       case "Json" :
-        baseType = "String";
+        tmp = "Json";
         break;
       case "Date" :
-        baseType = "DateTime64(3, 'UTC')";
+        tmp = "Date";
         break;
     }
   } else {
     switch (fieldType.type) {
       case "BigInt" :
-        let precision = fieldType.precision;
-        baseType = precision !== undefined && precision <= 38 ? `Decimal(` + precision.toString() + `,0)` : "String";
+        tmp = "BigInt";
         break;
       case "BigDecimal" :
-        let config = fieldType.config;
-        if (config !== undefined) {
-          let scale = config[1];
-          let precision$1 = config[0];
-          baseType = precision$1 > 38 || scale > precision$1 ? "String" : `Decimal(` + precision$1.toString() + `,` + scale.toString() + `)`;
-        } else {
-          baseType = "String";
-        }
+        tmp = "BigDecimal";
         break;
       case "Enum" :
-        let config$1 = fieldType.config;
-        let variantsLength = config$1.variants.length;
-        let enumType = variantsLength <= 127 ? "Enum8" : "Enum16";
-        let enumValues = config$1.variants.map(variant => `'` + variant + `'`).join(", ");
-        baseType = enumType + `(` + enumValues + `)`;
+        tmp = "Enum";
         break;
     }
   }
-  let baseType$1 = isArray ? `Array(` + baseType + `)` : baseType;
-  if (isNullable) {
-    return `Nullable(` + baseType$1 + `)`;
+  let tmp$1;
+  if (typeof fieldType !== "object") {
+    tmp$1 = undefined;
   } else {
-    return baseType$1;
+    switch (fieldType.type) {
+      case "BigInt" :
+        tmp$1 = fieldType.precision;
+        break;
+      case "BigDecimal" :
+        tmp$1 = Stdlib_Option.map(fieldType.config, param => param[0]);
+        break;
+      default:
+        tmp$1 = undefined;
+    }
   }
+  let tmp$2;
+  tmp$2 = typeof fieldType !== "object" || fieldType.type !== "BigDecimal" ? undefined : Stdlib_Option.map(fieldType.config, param => param[1]);
+  let tmp$3;
+  tmp$3 = typeof fieldType !== "object" || fieldType.type !== "Enum" ? undefined : fieldType.config.variants.map(variant => variant);
+  return {
+    name: name,
+    fieldName: fieldName,
+    fieldType: tmp,
+    isNullable: isNullable,
+    isArray: isArray,
+    precision: tmp$1,
+    scale: tmp$2,
+    enumVariants: tmp$3
+  };
 }
 
-function makeClickHouseEntitySchema(table) {
-  return S$RescriptSchema.object(s => {
-    let dict = {};
-    table.fields.forEach(field => {
+let checkpointColumns = Stdlib_Array.filterMap(InternalTable.Checkpoints.columns, param => {
+  let field = param.field;
+  if (field.TAG !== "Field") {
+    return;
+  }
+  let match = field._0;
+  return {
+    spec: makeColumnSpec(match.fieldName, undefined, param.clickHouseFieldType, match.isNullable, undefined),
+    valuesOf: param.valuesOf
+  };
+});
+
+let checkpointColumnSpecs = checkpointColumns.map(param => param.spec);
+
+function entitySpec(entityConfig) {
+  let options = entityConfig.storage.clickhouseOptions;
+  return {
+    name: entityConfig.name,
+    historyTable: EntityHistory.historyTableName(entityConfig.name, entityConfig.index),
+    columns: Stdlib_Array.filterMap(entityConfig.table.fields, field => {
       if (field.TAG !== "Field") {
         return;
       }
       let f = field._0;
-      let fieldName = Table.getClickHouseDbFieldName(f);
-      let match = f.fieldType;
-      let fieldSchema;
-      if (typeof match !== "object") {
-        switch (match) {
-          case "UInt52" :
-            let uint52Schema = S$RescriptSchema.preprocess(S$RescriptSchema.float, param => ({
-              p: unknown => parseFloat(unknown)
-            }));
-            fieldSchema = f.isNullable ? S$RescriptSchema.$$null(uint52Schema) : (
-                f.isArray ? S$RescriptSchema.array(uint52Schema) : uint52Schema
-              );
-            break;
-          case "ChainId" :
-            fieldSchema = ChainId.schema;
-            break;
-          case "Date" :
-            let dateSchema = Utils.Schema.clickHouseDate;
-            fieldSchema = f.isNullable ? S$RescriptSchema.$$null(dateSchema) : (
-                f.isArray ? S$RescriptSchema.array(dateSchema) : dateSchema
-              );
-            break;
-          default:
-            fieldSchema = f.fieldSchema;
-        }
+      return makeColumnSpec(Table.getClickHouseDbFieldName(f), f.fieldName, f.fieldType, f.isNullable, f.isArray);
+    }),
+    chainIdColumn: Stdlib_Option.map(Table.getChainIdField(entityConfig.table), Table.getClickHouseDbFieldName),
+    partitionBy: Stdlib_Option.flatMap(options, options => options.partitionBy),
+    orderBy: Stdlib_Option.flatMap(options, options => options.orderBy),
+    ttl: Stdlib_Option.flatMap(options, options => options.ttl),
+    skippingIndexes: Stdlib_Option.flatMap(options, options => Stdlib_Option.map(options.skippingIndexes, indexes => indexes.map(index => ({
+      name: index.name,
+      expr: index.expr,
+      indexType: index.type,
+      granularity: index.granularity
+    }))))
+  };
+}
+
+function makeRegistry() {
+  return {
+    entities: {},
+    checkpoints: undefined,
+    converters: {}
+  };
+}
+
+let absent = undefined;
+
+function compileSerializer(schema) {
+  return S$RescriptSchema.compile(schema, "Output", "Json", "Sync", false);
+}
+
+function makeConverters(entityConfig, scope, table) {
+  let columns = table.columns;
+  let tableName = table.name;
+  let sourceTable = entityConfig.table;
+  let match = Table.getChainIdField(sourceTable);
+  let match$1 = Internal.chainScopeChainId(scope);
+  let chainIdTag = match !== undefined && match$1 !== undefined ? [
+      Table.getClickHouseDbFieldName(match),
+      Primitive_option.valFromOption(match$1)
+    ] : undefined;
+  let fieldsByColumn = {};
+  sourceTable.fields.forEach(field => {
+    if (field.TAG !== "Field") {
+      return;
+    }
+    let f = field._0;
+    fieldsByColumn[Table.getClickHouseDbFieldName(f)] = f;
+  });
+  let setCells = [];
+  let deleteCells = [];
+  let registeredNames = new Set(columns.map(column => column.name));
+  columns.forEach(param => {
+    let name = param.name;
+    let match;
+    if (name === EntityHistory.checkpointIdFieldName) {
+      let cell = change => change.checkpointId;
+      match = [
+        cell,
+        cell
+      ];
+    } else if (name === EntityHistory.changeFieldName) {
+      match = [
+        param => "SET",
+        param => "DELETE"
+      ];
+    } else {
+      let match$1 = fieldsByColumn[name];
+      let exit = 0;
+      if (chainIdTag !== undefined && chainIdTag[0] === name) {
+        let chainId = chainIdTag[1];
+        match = [
+          param => chainId,
+          param => chainId
+        ];
       } else {
-        fieldSchema = f.fieldSchema;
+        exit = 1;
       }
-      dict[Table.getApiFieldName(f)] = s.f(fieldName, fieldSchema);
-    });
-    return dict;
+      if (exit === 1) {
+        if (match$1 !== undefined) {
+          let serialize = compileSerializer(clickHouseFieldSchema(match$1));
+          let apiName = Table.getApiFieldName(match$1);
+          if (apiName === Table.idFieldName) {
+            let cell$1 = change => serialize(change.entityId);
+            match = [
+              cell$1,
+              cell$1
+            ];
+          } else {
+            match = [
+              change => serialize(change.entity[apiName]),
+              param => absent
+            ];
+          }
+        } else {
+          match = Stdlib_JsError.throwWithMessage(`ClickHouse table "` + tableName + `" registered a column "` + name + `" that entity "` + entityConfig.name + `" has no value for`);
+        }
+      }
+    }
+    setCells.push(match[0]);
+    deleteCells.push(match[1]);
   });
+  Stdlib_Dict.forEachWithKey(fieldsByColumn, (f, column) => {
+    if (!registeredNames.has(column)) {
+      return Stdlib_JsError.throwWithMessage(`ClickHouse table "` + tableName + `" registered no column for field "` + f.fieldName + `" of entity "` + entityConfig.name + `"`);
+    }
+  });
+  return {
+    setCells: setCells,
+    deleteCells: deleteCells
+  };
 }
 
-let logger = Logging.createChild({
-  context: "ClickHouse"
-});
-
-async function insertWithRetry(client, table, values, format, retriesOpt) {
-  let retries = retriesOpt !== undefined ? retriesOpt : 8;
-  try {
-    return await client.insert({
-      table: table,
-      values: values,
-      format: format
-    });
-  } catch (raw_exn) {
-    let exn = Primitive_exceptions.internalToException(raw_exn);
-    if (retries > 0) {
-      let delayMs = Math.min(1000, 100 + ((900 * (8 - retries | 0) | 0) / 7 | 0) | 0);
-      if (values.length > 1) {
-        Logging.childWarn(logger, {
-          msg: "ClickHouse insert failed, splitting batch in half and retrying",
-          table: table,
-          batchSize: values.length,
-          retriesLeft: retries,
-          err: Utils.prettifyExn(exn)
-        });
-        await Utils.delay(delayMs);
-        let mid = (values.length >> 1);
-        let first = values.slice(0, mid);
-        let second = values.slice(mid);
-        await insertWithRetry(client, table, first, format, retries - 1 | 0);
-        return await insertWithRetry(client, table, second, format, retries - 1 | 0);
-      }
-      Logging.childWarn(logger, {
-        msg: "ClickHouse insert failed, retrying after delay",
-        table: table,
-        retriesLeft: retries,
-        err: Utils.prettifyExn(exn)
-      });
-      await Utils.delay(delayMs);
-      return await insertWithRetry(client, table, values, format, retries - 1 | 0);
+function fillBuilders(table, converters, changes) {
+  let rows = changes.length;
+  let builders = table.columns.map(__x => ClickHouseSink.makeBuilder(__x, rows));
+  let columns = builders.length;
+  for (let row = 0; row < rows; ++row) {
+    let change = changes[row];
+    let match;
+    match = change.type === "SET" ? [
+        converters.setCells,
+        ClickHouseSink.writeValue
+      ] : [
+        converters.deleteCells,
+        ClickHouseSink.writeDeletedValue
+      ];
+    let write = match[1];
+    let cells = match[0];
+    for (let column = 0; column < columns; ++column) {
+      write(builders[column], row, cells[column](change));
     }
-    throw exn;
   }
+  return builders;
 }
 
-async function setCheckpointsOrThrow(client, batch, database) {
-  let checkpointsCount = batch.checkpointIds.length;
-  if (checkpointsCount === 0) {
-    return;
+function entityTable(sink, registry, entityConfig) {
+  let table = registry.entities[entityConfig.name];
+  if (table !== undefined) {
+    return table;
   }
-  let checkpointRows = [];
-  for (let idx = 0; idx < checkpointsCount; ++idx) {
-    checkpointRows.push([
-      batch.checkpointIds[idx].toString(),
-      batch.checkpointChainIds[idx],
-      batch.checkpointBlockNumbers[idx],
-      batch.checkpointBlockHashes[idx],
-      batch.checkpointEventsProcessed[idx]
-    ]);
+  let spec = entitySpec(entityConfig);
+  let table$1 = ClickHouseSink.makeTable(spec.historyTable, sink.registerEntityTable(spec));
+  registry.entities[entityConfig.name] = table$1;
+  return table$1;
+}
+
+function checkpointsTable(sink, registry) {
+  let table = registry.checkpoints;
+  if (table !== undefined) {
+    return table;
   }
+  let table$1 = ClickHouseSink.makeTable(InternalTable.Checkpoints.table.tableName, sink.registerCheckpointsTable(checkpointColumnSpecs));
+  registry.checkpoints = table$1;
+  return table$1;
+}
+
+function stageBuilders(sink, table, builders, rows) {
+  return sink.stage(table.handle, rows, builders.map(ClickHouseSink.builderPayload));
+}
+
+function stageCheckpointsOrThrow(sink, registry, batch) {
+  let rows = batch.checkpointIds.length;
+  if (rows === 0) {
+    return null;
+  }
+  let table = checkpointsTable(sink, registry);
   try {
-    return await insertWithRetry(client, database + `.\`` + InternalTable.Checkpoints.table.tableName + `\``, checkpointRows, "JSONCompactEachRow", undefined);
+    let builders = table.columns.map(__x => ClickHouseSink.makeBuilder(__x, rows));
+    builders.forEach((builder, index) => {
+      let columnValues = checkpointColumns[index].valuesOf(batch);
+      for (let row = 0; row < rows; ++row) {
+        ClickHouseSink.writeValue(builder, row, columnValues[row]);
+      }
+    });
+    return stageBuilders(sink, table, builders, rows);
   } catch (raw_exn) {
     let exn = Primitive_exceptions.internalToException(raw_exn);
     throw {
       RE_EXN_ID: Persistence.StorageError,
-      message: `Failed to insert checkpoints into ClickHouse table "` + InternalTable.Checkpoints.table.tableName + `"`,
+      message: `Failed to convert checkpoints for ClickHouse table "` + table.name + `"`,
       reason: Utils.prettifyExn(exn),
       Error: new Error()
     };
   }
 }
 
-async function setUpdatesOrThrow(client, cache, changes, entityConfig, scope, database) {
-  if (changes.length === 0) {
+function stageUpdatesOrThrow(sink, registry, changes, entityConfig, scope) {
+  let rows = changes.length;
+  if (rows === 0) {
     return;
   }
-  let chainIdField = Table.getChainIdField(entityConfig.table);
-  let scopeChainId = Internal.chainScopeChainId(scope);
+  let table = entityTable(sink, registry, entityConfig);
+  let tableName = table.name;
   let cacheKey = entityConfig.name + `|` + Internal.chainScopeToString(scope);
-  let cached = cache[cacheKey];
-  let match;
-  if (cached !== undefined) {
-    match = cached;
-  } else {
-    let cached_tableName = database + `.\`` + EntityHistory.historyTableName(entityConfig.name, entityConfig.index) + `\``;
-    let cached_convertOrThrow = S$RescriptSchema.compile(S$RescriptSchema.array(S$RescriptSchema.union([
-      EntityHistory.makeSetUpdateSchema(Table.getIdSchema(entityConfig.table), makeClickHouseEntitySchema(entityConfig.table)),
-      S$RescriptSchema.object(s => {
-        s.tag(EntityHistory.changeFieldName, "DELETE");
-        if (chainIdField !== undefined && scopeChainId !== undefined) {
-          s.tag(Table.getClickHouseDbFieldName(chainIdField), Primitive_option.valFromOption(scopeChainId));
-        }
-        return {
-          type: "DELETE",
-          entityId: s.f(Table.idFieldName, Table.getIdSchema(entityConfig.table)),
-          checkpointId: s.f(EntityHistory.checkpointIdFieldName, EntityHistory.unsafeCheckpointIdSchema)
-        };
-      })
-    ])), "Output", "Json", "Sync", false);
-    let cached$1 = {
-      tableName: cached_tableName,
-      convertOrThrow: cached_convertOrThrow
-    };
-    cache[cacheKey] = cached$1;
-    match = cached$1;
-  }
-  let tableName = match.tableName;
-  let changes$1;
-  if (chainIdField !== undefined && scopeChainId !== undefined) {
-    let chainId = Primitive_option.valFromOption(scopeChainId);
-    changes$1 = changes.map(change => {
-      if (change.type === "SET") {
-        return {
-          type: "SET",
-          entityId: change.entityId,
-          entity: Internal.stampChainId(change.entity, chainIdField.fieldName, chainId),
-          checkpointId: change.checkpointId
-        };
-      } else {
-        return change;
-      }
-    });
-  } else {
-    changes$1 = changes;
-  }
   try {
-    let values = match.convertOrThrow(changes$1);
-    return await insertWithRetry(client, tableName, values, "JSONEachRow", undefined);
+    let cached = registry.converters[cacheKey];
+    let converters;
+    if (cached !== undefined) {
+      converters = cached;
+    } else {
+      let cached$1 = makeConverters(entityConfig, scope, table);
+      registry.converters[cacheKey] = cached$1;
+      converters = cached$1;
+    }
+    let builders = fillBuilders(table, converters, changes);
+    return stageBuilders(sink, table, builders, rows);
   } catch (raw_exn) {
     let exn = Primitive_exceptions.internalToException(raw_exn);
     throw {
       RE_EXN_ID: Persistence.StorageError,
-      message: `Failed to insert items into ClickHouse table "` + tableName + `"`,
+      message: `Failed to convert items for ClickHouse table "` + tableName + `"`,
       reason: Utils.prettifyExn(exn),
       Error: new Error()
     };
   }
 }
 
-function onClusterClause(onCluster) {
-  if (onCluster) {
-    return ` ON CLUSTER '{cluster}'`;
-  } else {
-    return "";
-  }
-}
-
-function replicatedTableSettingsClause(replicated) {
-  if (replicated) {
-    return "\nSETTINGS replicated_deduplication_window = 0";
-  } else {
-    return "";
-  }
-}
-
-function databaseEngineName(engineSpec) {
-  return engineSpec.trim().split("(")[0].split(" ")[0].trim();
-}
-
-function makeCreateHistoryTableQuery(entityConfig, database, replicatedOpt, onClusterOpt, chainIdModeOpt) {
-  let replicated = replicatedOpt !== undefined ? replicatedOpt : false;
-  let onCluster = onClusterOpt !== undefined ? onClusterOpt : false;
-  let chainIdMode = chainIdModeOpt !== undefined ? chainIdModeOpt : "int32";
-  let tableEngine = replicated ? "ReplicatedMergeTree" : "MergeTree()";
-  let fieldDefinitions = Stdlib_Array.filterMap(entityConfig.table.fields, field => {
-    if (field.TAG !== "Field") {
-      return;
-    }
-    let field$1 = field._0;
-    let fieldName = Table.getClickHouseDbFieldName(field$1);
-    let clickHouseType = getClickHouseFieldType(field$1.fieldType, field$1.isNullable, field$1.isArray, chainIdMode);
-    return `\`` + fieldName + `\` ` + clickHouseType;
-  });
-  let options = entityConfig.storage.clickhouseOptions;
-  let match = options !== undefined ? [
-      options.partitionBy,
-      options.orderBy,
-      options.ttl,
-      options.skippingIndexes
-    ] : [
-      undefined,
-      undefined,
-      undefined,
-      undefined
-    ];
-  let skippingIndexes = match[3];
-  let ttl = match[2];
-  let orderBy = match[1];
-  let partitionBy = match[0];
-  let columnByFieldName = {};
-  entityConfig.table.fields.forEach(field => {
-    if (field.TAG !== "Field") {
-      return;
-    }
-    let f = field._0;
-    columnByFieldName[f.fieldName] = Table.getClickHouseDbFieldName(f);
-  });
-  let orderByColumns;
-  if (orderBy !== undefined) {
-    let userColumns = orderBy.map(fieldName => {
-      let column = columnByFieldName[fieldName];
-      if (column !== undefined) {
-        return `\`` + column + `\``;
-      } else {
-        return Stdlib_JsError.throwWithMessage(`ClickHouse orderBy field "` + fieldName + `" is not defined on entity "` + entityConfig.name + `"`);
-      }
-    }).join(", ");
-    orderByColumns = userColumns + `, ` + EntityHistory.checkpointIdFieldName;
-  } else {
-    orderByColumns = Table.idFieldName + `, ` + EntityHistory.checkpointIdFieldName;
+async function writeStagedOrThrow(sink, entities, checkpoints) {
+  try {
+    return await sink.writeBatch(entities, checkpoints);
+  } catch (raw_exn) {
+    let exn = Primitive_exceptions.internalToException(raw_exn);
+    throw {
+      RE_EXN_ID: Persistence.StorageError,
+      message: "Failed to write a batch to ClickHouse",
+      reason: Utils.prettifyExn(exn),
+      Error: new Error()
+    };
   }
-  let resolveExpressionColumns = expression => expression.replace(/'[^']*'|`[^`]*`|[A-Za-z_][A-Za-z0-9_]*/g, (match, param, param$1) => {
-    let column = columnByFieldName[match];
-    if (column !== undefined) {
-      return `\`` + column + `\``;
-    } else {
-      return match;
-    }
-  });
-  let partitionByClause = partitionBy !== undefined ? `\nPARTITION BY ` + resolveExpressionColumns(partitionBy) : "";
-  let ttlClause = ttl !== undefined ? `\nTTL ` + resolveExpressionColumns(ttl) : "";
-  let skippingIndexDefinitions = skippingIndexes !== undefined ? skippingIndexes.map(index => {
-      let granularity = index.granularity;
-      let granularityClause = granularity !== undefined ? ` GRANULARITY ` + granularity.toString() : "";
-      return `,\n  INDEX \`` + index.name + `\` ` + resolveExpressionColumns(index.expr) + ` TYPE ` + index.type + granularityClause;
-    }).join("") : "";
-  return `CREATE TABLE IF NOT EXISTS ` + database + `.\`` + EntityHistory.historyTableName(entityConfig.name, entityConfig.index) + `\`` + (
-    onCluster ? ` ON CLUSTER '{cluster}'` : ""
-  ) + ` (
-  ` + fieldDefinitions.join(",\n  ") + `,
-  \`` + EntityHistory.checkpointIdFieldName + `\` ` + getClickHouseFieldType("UInt64", false, false, undefined) + `,
-  \`` + EntityHistory.changeFieldName + `\` ` + getClickHouseFieldType({
-    type: "Enum",
-    config: EntityHistory.RowAction.config
-  }, false, false, undefined) + skippingIndexDefinitions + `
-)
-ENGINE = ` + tableEngine + partitionByClause + `
-ORDER BY (` + orderByColumns + `)` + ttlClause + (
-    replicated ? "\nSETTINGS replicated_deduplication_window = 0" : ""
-  );
-}
-
-function makeCreateCheckpointsTableQuery(database, replicatedOpt, onClusterOpt, chainIdModeOpt) {
-  let replicated = replicatedOpt !== undefined ? replicatedOpt : false;
-  let onCluster = onClusterOpt !== undefined ? onClusterOpt : false;
-  let chainIdMode = chainIdModeOpt !== undefined ? chainIdModeOpt : "int32";
-  let tableEngine = replicated ? "ReplicatedMergeTree" : "MergeTree()";
-  return `CREATE TABLE IF NOT EXISTS ` + database + `.\`` + InternalTable.Checkpoints.table.tableName + `\`` + (
-    onCluster ? ` ON CLUSTER '{cluster}'` : ""
-  ) + ` (
-  \`` + "id" + `\` ` + getClickHouseFieldType("UInt64", false, false, undefined) + `,
-  \`` + "chain_id" + `\` ` + getClickHouseFieldType("ChainId", false, false, chainIdMode) + `,
-  \`` + "block_number" + `\` ` + getClickHouseFieldType("Int32", false, false, undefined) + `,
-  \`` + "block_hash" + `\` ` + getClickHouseFieldType("String", true, false, undefined) + `,
-  \`` + "events_processed" + `\` ` + getClickHouseFieldType("UInt64", false, false, undefined) + `
-)
-ENGINE = ` + tableEngine + `
-ORDER BY (` + "id" + `)` + (
-    replicated ? "\nSETTINGS replicated_deduplication_window = 0" : ""
-  );
 }
 
-function makeCreateViewQuery(entityConfig, database, onClusterOpt) {
-  let onCluster = onClusterOpt !== undefined ? onClusterOpt : false;
-  let historyTableName = EntityHistory.historyTableName(entityConfig.name, entityConfig.index);
-  let field = Table.getChainIdField(entityConfig.table);
-  let dedupKey = (
-    field !== undefined ? [
-        Table.idFieldName,
-        Table.getClickHouseDbFieldName(field)
-      ] : [Table.idFieldName]
-  ).map(name => `\`` + name + `\``).join(", ");
-  let checkpointsTableName = InternalTable.Checkpoints.table.tableName;
-  let entityFields = Stdlib_Array.filterMap(entityConfig.table.fields, field => {
-    if (field.TAG !== "Field") {
-      return;
-    }
-    let fieldName = Table.getClickHouseDbFieldName(field._0);
-    return `\`` + fieldName + `\``;
-  }).join(", ");
-  return `CREATE VIEW IF NOT EXISTS ` + database + `.\`` + entityConfig.name + `\`` + (
-    onCluster ? ` ON CLUSTER '{cluster}'` : ""
-  ) + ` AS
-SELECT ` + entityFields + `
-FROM (
-  SELECT ` + entityFields + `, \`` + EntityHistory.changeFieldName + `\`
-  FROM ` + database + `.\`` + historyTableName + `\`
-  WHERE \`` + EntityHistory.checkpointIdFieldName + `\` <= (SELECT max(` + "id" + `) FROM ` + database + `.\`` + checkpointsTableName + `\`)
-  ORDER BY \`` + EntityHistory.checkpointIdFieldName + `\` DESC
-  LIMIT 1 BY ` + dedupKey + `
-)
-WHERE \`` + EntityHistory.changeFieldName + `\` = '` + "SET" + `'`;
-}
-
-async function initialize(client, database, entities, param, chainIdModeOpt) {
-  let chainIdMode = chainIdModeOpt !== undefined ? chainIdModeOpt : "int32";
+async function initialize(sink, entities) {
   try {
-    let databaseEngine = Env.ClickHouse.databaseEngine();
-    let databaseEngineClause = databaseEngine !== undefined ? ` ENGINE = ` + databaseEngine : "";
-    let hasReplicatedDatabaseEngine = databaseEngine !== undefined ? databaseEngineName(databaseEngine) === "Replicated" : false;
-    let envReplicated = Env.ClickHouse.replicated();
-    let replicated = envReplicated || hasReplicatedDatabaseEngine;
-    if (hasReplicatedDatabaseEngine && !envReplicated) {
-      Logging.info("ENVIO_CLICKHOUSE_DATABASE_ENGINE is Replicated; enabling replicated mode so tables use the ReplicatedMergeTree engine.");
-    }
-    let databaseOnClusterClause = replicated ? ` ON CLUSTER '{cluster}'` : "";
-    let ddlOnCluster = replicated && !hasReplicatedDatabaseEngine;
-    if (databaseEngine !== undefined) {
-      let expectedEngineName = databaseEngineName(databaseEngine);
-      let existingResult = await client.query({
-        query: `SELECT engine FROM system.databases WHERE name = '` + database + `'`
-      });
-      let rows = (await existingResult.json()).data;
-      let row = rows[0];
-      if (row !== undefined) {
-        let row$1 = Primitive_option.valFromOption(row);
-        if (row$1.engine !== expectedEngineName) {
-          Stdlib_JsError.throwWithMessage(`ClickHouse database "` + database + `" exists with engine "` + row$1.engine + `" but ENVIO_CLICKHOUSE_DATABASE_ENGINE specifies "` + expectedEngineName + `". Drop the database manually to change its engine.`);
-        }
-      }
-    }
-    if (hasReplicatedDatabaseEngine) {
-      await client.command({
-        query: `DROP DATABASE IF EXISTS ` + database + ` ON CLUSTER '{cluster}' SYNC`
-      });
-    } else {
-      await client.command({
-        query: `TRUNCATE DATABASE IF EXISTS ` + database + (
-          ddlOnCluster ? ` ON CLUSTER '{cluster}'` : ""
-        )
-      });
-    }
-    await client.command({
-      query: `CREATE DATABASE IF NOT EXISTS ` + database + databaseOnClusterClause + databaseEngineClause
-    });
-    await Promise.all(entities.map(entityConfig => client.command({
-      query: makeCreateHistoryTableQuery(entityConfig, database, replicated, ddlOnCluster, chainIdMode)
-    })));
-    await client.command({
-      query: makeCreateCheckpointsTableQuery(database, replicated, ddlOnCluster, chainIdMode)
+    await sink.initialize({
+      entities: entities.map(entitySpec),
+      checkpointColumns: checkpointColumnSpecs,
+      replicated: Env.ClickHouse.replicated(),
+      databaseEngine: Env.ClickHouse.databaseEngine()
     });
-    if (hasReplicatedDatabaseEngine) {
-      await client.command({
-        query: `SYSTEM SYNC DATABASE REPLICA ON CLUSTER '{cluster}' ` + database
-      });
-    }
-    await Promise.all(entities.map(entityConfig => client.command({
-      query: makeCreateViewQuery(entityConfig, database, ddlOnCluster)
-    })));
     return Logging.trace("ClickHouse storage initialization completed successfully");
   } catch (raw_exn) {
     let exn = Primitive_exceptions.internalToException(raw_exn);
     Logging.errorWithExn(exn, "Failed to initialize ClickHouse storage");
-    return Stdlib_JsError.throwWithMessage("ClickHouse initialization failed");
+    throw {
+      RE_EXN_ID: Persistence.StorageError,
+      message: "Failed to initialize ClickHouse storage",
+      reason: Utils.prettifyExn(exn),
+      Error: new Error()
+    };
   }
 }
 
-async function resume(client, database, checkpointId) {
+async function resume(sink, checkpointId, chains, entities) {
+  let chainProgress = chains.map(chain => ({
+    chainId: ChainId.toString(chain.id),
+    progressBlockNumber: chain.progressBlockNumber
+  }));
   try {
-    try {
-      await client.command({
-        query: `USE ` + database
-      });
-    } catch (raw_exn) {
-      let exn = Primitive_exceptions.internalToException(raw_exn);
-      Logging.errorWithExn(exn, `ClickHouse storage database "` + database + `" not found. Please run 'envio start -r' to reinitialize the indexer (it'll also drop Postgres database).`);
-      Stdlib_JsError.throwWithMessage("ClickHouse resume failed");
-    }
-    let tablesResult = await client.query({
-      query: `SHOW TABLES FROM ` + database + ` LIKE '` + EntityHistory.historyTablePrefix + `%'`
-    });
-    let tables = (await tablesResult.json()).data;
-    await Promise.all(tables.map(table => {
-      let tableName = table.name;
-      return client.command({
-        query: `ALTER TABLE ` + database + `.\`` + tableName + `\` DELETE WHERE \`` + EntityHistory.checkpointIdFieldName + `\` > ` + checkpointId.toString()
-      });
-    }));
-    return await client.command({
-      query: `DELETE FROM ` + database + `.\`` + InternalTable.Checkpoints.table.tableName + `\` WHERE \`` + Table.idFieldName + `\` > ` + checkpointId.toString()
+    return await sink.resume({
+      checkpointId: checkpointId.toString(),
+      chainProgress: chainProgress,
+      historyTables: entities.map(entityConfig => entitySpec(entityConfig).historyTable),
+      replicated: Env.ClickHouse.replicated(),
+      databaseEngine: Env.ClickHouse.databaseEngine()
     });
-  } catch (raw_exn$1) {
-    let exn$1 = Primitive_exceptions.internalToException(raw_exn$1);
-    if (exn$1.RE_EXN_ID === Persistence.StorageError) {
-      throw exn$1;
-    }
-    Logging.errorWithExn(exn$1, "Failed to resume ClickHouse storage");
-    return Stdlib_JsError.throwWithMessage("ClickHouse resume failed");
+  } catch (raw_exn) {
+    let exn = Primitive_exceptions.internalToException(raw_exn);
+    Logging.errorWithExn(exn, "Failed to resume ClickHouse storage");
+    throw {
+      RE_EXN_ID: Persistence.StorageError,
+      message: "Failed to resume ClickHouse storage",
+      reason: Utils.prettifyExn(exn),
+      Error: new Error()
+    };
   }
 }
 
 export {
-  getClickHouseFieldType,
+  clickHouseFieldSchema,
   makeClickHouseEntitySchema,
   logger,
-  insertWithRetry,
-  setCheckpointsOrThrow,
-  setUpdatesOrThrow,
-  onClusterClause,
-  replicatedTableSettingsClause,
-  databaseEngineName,
-  makeCreateHistoryTableQuery,
-  makeCreateCheckpointsTableQuery,
-  makeCreateViewQuery,
+  makeSink,
+  makeColumnSpec,
+  checkpointColumns,
+  checkpointColumnSpecs,
+  entitySpec,
+  makeRegistry,
+  entityTable,
+  checkpointsTable,
+  stageBuilders,
+  stageCheckpointsOrThrow,
+  stageUpdatesOrThrow,
+  writeStagedOrThrow,
   initialize,
   resume,
 }