envio 3.7.0 → 3.9.0

package/src/TestIndexer.res.mjs

@@ -9,13 +9,14 @@ import * as Logging from "./Logging.res.mjs";
 import * as ChainMap from "./ChainMap.res.mjs";
 import * as EntityId from "./EntityId.res.mjs";
 import * as Internal from "./Internal.res.mjs";
+import * as ChainState from "./ChainState.res.mjs";
 import * as Stdlib_Int from "@rescript/runtime/lib/es6/Stdlib_Int.js";
+import * as AddressRows from "./AddressRows.res.mjs";
 import * as IndexerLoop from "./IndexerLoop.res.mjs";
 import * as Persistence from "./Persistence.res.mjs";
 import * as Stdlib_Dict from "@rescript/runtime/lib/es6/Stdlib_Dict.js";
 import * as EntityFilter from "./db/EntityFilter.res.mjs";
 import * as IndexerState from "./IndexerState.res.mjs";
-import * as Stdlib_Array from "@rescript/runtime/lib/es6/Stdlib_Array.js";
 import * as ErrorHandling from "./ErrorHandling.res.mjs";
 import * as HandlerLoader from "./HandlerLoader.res.mjs";
 import * as InternalTable from "./db/InternalTable.res.mjs";
@@ -24,37 +25,17 @@ import * as SimulateItems from "./SimulateItems.res.mjs";
 import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js";
 import * as Stdlib_JsError from "@rescript/runtime/lib/es6/Stdlib_JsError.js";
 import * as Stdlib_Promise from "@rescript/runtime/lib/es6/Stdlib_Promise.js";
+import * as ContractMapping from "./ContractMapping.res.mjs";
 import * as Primitive_option from "@rescript/runtime/lib/es6/Primitive_option.js";
 import * as S$RescriptSchema from "rescript-schema/src/S.res.mjs";
 import * as Primitive_exceptions from "@rescript/runtime/lib/es6/Primitive_exceptions.js";
 
-function toIndexingAddress(dc) {
-  return {
-    address: Config.EnvioAddresses.getAddress(dc),
-    contractName: dc.contract_name,
-    registrationBlock: dc.registration_block
-  };
+function addressRowsByChain(state) {
+  return AddressRows.Table.groupByChain(state.addresses);
 }
 
-function getIndexingAddressesByChain(state) {
-  let byChain = {};
-  let dcDict = state.entities[Config.EnvioAddresses.name];
-  if (dcDict !== undefined) {
-    Object.values(dcDict).forEach(entity => {
-      let chainIdStr = ChainId.toString(entity.chain_id);
-      let arr = byChain[chainIdStr];
-      let contracts;
-      if (arr !== undefined) {
-        contracts = arr;
-      } else {
-        let arr$1 = [];
-        byChain[chainIdStr] = arr$1;
-        contracts = arr$1;
-      }
-      contracts.push(toIndexingAddress(entity));
-    });
-  }
-  return byChain;
+function renderRows(rows, config) {
+  return AddressRows.render(rows, config.ecosystem.name, !config.lowercaseAddresses);
 }
 
 function rowKey(scope, entityId) {
@@ -88,8 +69,18 @@ function handleLoad(state, tableName, filter) {
   }
 }
 
-function handleWriteBatch(state, updatedEntities, checkpointIds, checkpointChainIds, checkpointBlockNumbers, checkpointEventsProcessed) {
+function handleWriteBatch(state, config, updatedEntities, registeredAddresses, checkpointIds, checkpointChainIds, checkpointBlockNumbers, checkpointEventsProcessed) {
   let changesByCheckpoint = {};
+  let addressesByCheckpoint = {};
+  let rendered = renderRows(registeredAddresses.map(param => param.row), config);
+  registeredAddresses.forEach((param, idx) => {
+    let row = param.row;
+    AddressRows.Table.insert(state.addresses, row);
+    Utils.Dict.push(addressesByCheckpoint, param.checkpointId.toString(), {
+      address: rendered[idx],
+      contract: ContractMapping.nameOfOrThrow(state.contractMapping, row.contractId)
+    });
+  });
   updatedEntities.forEach(param => {
     let scope = param.scope;
     let entityConfig = param.entityConfig;
@@ -151,40 +142,33 @@ function handleWriteBatch(state, updatedEntities, checkpointIds, checkpointChain
     change["chainId"] = checkpointChainIds[i];
     change["eventsProcessed"] = checkpointEventsProcessed[i];
     let checkpointKey = checkpointId.toString();
+    let rendered$1 = addressesByCheckpoint[checkpointKey];
+    if (rendered$1 !== undefined) {
+      let addressesObj = {};
+      addressesObj["sets"] = rendered$1;
+      change["addresses"] = addressesObj;
+    }
     let entityChanges = changesByCheckpoint[checkpointKey];
     if (entityChanges !== undefined) {
       Object.entries(entityChanges).forEach(param => {
         let match = param[1];
         let deleted = match.deleted;
         let sets = match.sets;
-        let entityName = param[0];
-        if (entityName === Config.EnvioAddresses.name) {
-          let entityObj = {};
-          if (sets.length !== 0) {
-            let simplifiedSets = sets.map(entity => ({
-              address: Config.EnvioAddresses.getAddress(entity),
-              contract: entity.contract_name
-            }));
-            entityObj["sets"] = simplifiedSets;
-          }
-          change["addresses"] = entityObj;
-          return;
-        }
-        let entityObj$1 = {};
+        let entityObj = {};
         if (sets.length !== 0) {
-          entityObj$1["sets"] = sets;
+          entityObj["sets"] = sets;
         }
         if (deleted.length !== 0) {
-          entityObj$1["deleted"] = deleted;
+          entityObj["deleted"] = deleted;
         }
-        change[Utils.$$String.capitalize(entityName)] = entityObj$1;
+        change[Utils.$$String.capitalize(param[0])] = entityObj;
       });
     }
     state.processChanges.push(change);
   }
 }
 
-function makeInitialState(config, processConfigChains, indexingAddressesByChain) {
+function makeInitialState(config, processConfigChains, addressRowsByChain, contractMapping) {
   let chainKeys = Object.keys(processConfigChains);
   let chains = chainKeys.map(chainIdStr => {
     let chain = ChainId.normalizeOrThrow(chainIdStr);
@@ -192,7 +176,7 @@ function makeInitialState(config, processConfigChains, indexingAddressesByChain)
       Stdlib_JsError.throwWithMessage(`Chain ` + chainIdStr + ` is not configured in config.yaml`);
     }
     let processChainConfig = processConfigChains[chainIdStr];
-    let indexingAddresses = Stdlib_Option.getOr(indexingAddressesByChain[chainIdStr], []);
+    let addressRows = Stdlib_Option.getOr(addressRowsByChain[chainIdStr], AddressRows.emptySeedRows());
     return {
       id: chain,
       startBlock: processChainConfig.startBlock,
@@ -202,17 +186,18 @@ function makeInitialState(config, processConfigChains, indexingAddressesByChain)
       numEventsProcessed: 0,
       firstEventBlockNumber: undefined,
       timestampCaughtUpToHeadOrEndblock: undefined,
-      indexingAddresses: indexingAddresses,
+      addressRows: addressRows,
       sourceBlockNumber: Stdlib_Option.getOr(processChainConfig.endBlock, 0)
     };
   });
   return {
     cleanRun: true,
+    contractMapping: contractMapping,
+    envioInfo: {},
     cache: {},
     chains: chains,
     checkpointId: InternalTable.Checkpoints.initialCheckpointId,
-    reorgCheckpoints: [],
-    envioInfo: Config.stripSensitiveData(Config.getPublicConfigJson())
+    reorgCheckpoints: []
   };
 }
 
@@ -230,22 +215,44 @@ function getSimulateEndBlock(simulateItems, config, startBlock) {
   let maxBlock = {
     contents: startBlock
   };
-  simulateItems.forEach(rawJson => {
-    let blockJson = rawJson.block;
-    if (blockJson == null) {
-      return;
-    }
-    let n = Stdlib_Option.flatMap(blockJson[config.ecosystem.blockNumberName], v => {
-      if (v == null) {
-        return;
-      } else {
-        return Primitive_option.some(v);
-      }
-    });
+  let match = config.ecosystem.name;
+  let blockNumberKey;
+  switch (match) {
+    case "evm" :
+    case "fuel" :
+      blockNumberKey = config.ecosystem.blockNumberName;
+      break;
+    case "svm" :
+      blockNumberKey = "slot";
+      break;
+  }
+  let bump = n => {
     if (n !== undefined && n > maxBlock.contents) {
       maxBlock.contents = n;
       return;
     }
+  };
+  let getInt = (d, key) => Stdlib_Option.flatMap(d[key], v => {
+    if (v == null) {
+      return;
+    } else {
+      return Primitive_option.some(v);
+    }
+  });
+  simulateItems.forEach(rawJson => {
+    let match = config.ecosystem.name;
+    switch (match) {
+      case "evm" :
+      case "fuel" :
+        break;
+      case "svm" :
+        bump(getInt(rawJson, "slot"));
+        break;
+    }
+    let blockJson = rawJson.block;
+    if (!(blockJson == null)) {
+      return bump(getInt(blockJson, blockNumberKey));
+    }
   });
   return maxBlock.contents;
 }
@@ -377,7 +384,7 @@ function makeInMemoryStorage(state) {
   return {
     name: "test-inmemory",
     isInitialized: async () => true,
-    initialize: async (param, param$1, param$2, param$3) => Stdlib_JsError.throwWithMessage("TestIndexer: initialize should not be called; the initial state is derived from config."),
+    initialize: async (param, param$1, param$2, param$3, param$4) => Stdlib_JsError.throwWithMessage("TestIndexer: initialize should not be called; the initial state is derived from config."),
     resumeInitialState: async () => Stdlib_JsError.throwWithMessage("TestIndexer: resumeInitialState should not be called; the initial state is derived from config."),
     loadOrThrow: async (filter, table) => handleLoad(state, table.tableName, filter),
     ensureQueryIndexes: async (param, param$1) => {},
@@ -391,7 +398,7 @@ function makeInMemoryStorage(state) {
     getRollbackTargetCheckpoint: async (param, param$1) => Stdlib_JsError.throwWithMessage("TestIndexer: Rollback is not supported. The runner forces rollbackOnReorg off, so this should be unreachable."),
     getRollbackProgressDiff: async param => Stdlib_JsError.throwWithMessage("TestIndexer: Rollback is not supported. The runner forces rollbackOnReorg off, so this should be unreachable."),
     getRollbackData: async (param, param$1) => Stdlib_JsError.throwWithMessage("TestIndexer: Rollback is not supported. The runner forces rollbackOnReorg off, so this should be unreachable."),
-    writeBatch: async (batch, param, param$1, param$2, param$3, param$4, updatedEntities, param$5, param$6) => handleWriteBatch(state, updatedEntities, batch.checkpointIds, batch.checkpointChainIds, batch.checkpointBlockNumbers, batch.checkpointEventsProcessed),
+    writeBatch: async (batch, param, param$1, config, param$2, param$3, updatedEntities, registeredAddresses, param$4, param$5) => handleWriteBatch(state, config, updatedEntities, registeredAddresses, batch.checkpointIds, batch.checkpointChainIds, batch.checkpointBlockNumbers, batch.checkpointEventsProcessed),
     close: async () => {}
   };
 }
@@ -424,36 +431,25 @@ function getRegistrations(config) {
 
 function createTestIndexer() {
   let config = Config.load();
-  let allEntities = config.allEntities;
+  let allEntities = config.userEntities;
   let entities = {};
   let entityConfigs = {};
   allEntities.forEach(entityConfig => {
     entities[entityConfig.name] = {};
     entityConfigs[entityConfig.name] = entityConfig;
   });
-  let envioAddressesDict = entities[Config.EnvioAddresses.name];
-  ChainMap.values(config.chainMap).forEach(chainConfig => {
-    chainConfig.contracts.forEach(contract => {
-      contract.addresses.forEach(address => {
-        let entity_id = Config.EnvioAddresses.makeId(chainConfig.id, address);
-        let entity_chain_id = chainConfig.id;
-        let entity_contract_name = contract.name;
-        let entity = {
-          id: entity_id,
-          chain_id: entity_chain_id,
-          registration_block: -1,
-          registration_log_index: -1,
-          contract_name: entity_contract_name
-        };
-        envioAddressesDict[entity_id] = entity;
-      });
-    });
-  });
+  let chainConfigs = ChainMap.values(config.chainMap);
+  let contractMapping = config.contractMapping;
+  let ecosystem = config.ecosystem.name;
+  let addresses = AddressRows.Table.make();
+  ChainState.seedConfigAddresses(addresses, chainConfigs, ecosystem, contractMapping);
   let state = {
     processInProgress: false,
     progressBlockByChain: {},
     entities: entities,
     entityConfigs: entityConfigs,
+    addresses: addresses,
+    contractMapping: contractMapping,
     processChanges: []
   };
   let storage = makeInMemoryStorage(state);
@@ -465,16 +461,13 @@ function createTestIndexer() {
   }
   let entityOpsDict = {};
   allEntities.forEach(entityConfig => {
-    if (entityConfig.name !== Config.EnvioAddresses.name) {
-      entityOpsDict[entityConfig.name] = {
-        get: makeEntityGet(state, entityConfig),
-        getAll: makeEntityGetAll(state, entityConfig),
-        getWhere: makeEntityGetWhere(state, entityConfig),
-        getOrThrow: makeEntityGetOrThrow(state, entityConfig),
-        set: makeEntitySet(state, entityConfig)
-      };
-      return;
-    }
+    entityOpsDict[entityConfig.name] = {
+      get: makeEntityGet(state, entityConfig),
+      getAll: makeEntityGetAll(state, entityConfig),
+      getWhere: makeEntityGetWhere(state, entityConfig),
+      getOrThrow: makeEntityGetOrThrow(state, entityConfig),
+      set: makeEntitySet(state, entityConfig)
+    };
   });
   let chainIds = [];
   let chains = Object.create(null);
@@ -512,11 +505,14 @@ function createTestIndexer() {
           if (state.processInProgress) {
             Stdlib_JsError.throwWithMessage(`Cannot access ` + contract.name + `.addresses while indexer.process() is running. ` + "Wait for process() to complete before reading contract addresses.");
           }
-          return Stdlib_Array.filterMap(Stdlib_Option.getOr(getIndexingAddressesByChain(state)[ChainId.toString(chainConfig.id)], []), ia => {
-            if (ia.contractName === contract.name) {
-              return Primitive_option.some(ia.address);
+          let contractId = ContractMapping.idOfOrThrow(state.contractMapping, contract.name, undefined);
+          return renderRows(AddressRows.Table.rows(state.addresses).filter(row => {
+            if (row.chainId === chainConfig.id) {
+              return row.contractId === contractId;
+            } else {
+              return false;
             }
-          });
+          }), config);
         }
       });
       Object.defineProperty(chainObj, contract.name, {
@@ -586,8 +582,7 @@ function createTestIndexer() {
       let chainIdStr = param[0];
       let chains = {};
       chains[chainIdStr] = processChainConfig;
-      let indexingAddressesByChain = getIndexingAddressesByChain(state);
-      let initialState = makeInitialState(config, chains, indexingAddressesByChain);
+      let initialState = makeInitialState(config, chains, AddressRows.Table.groupByChain(state.addresses), state.contractMapping);
       let exitAfterFirstEventBlock = Stdlib_Option.isNone(processChainConfig.endBlock);
       let resolvedChainDict = {};
       resolvedChainDict["startBlock"] = processChainConfig.startBlock;
@@ -674,8 +669,8 @@ function createTestIndexer() {
 }
 
 export {
-  toIndexingAddress,
-  getIndexingAddressesByChain,
+  addressRowsByChain,
+  renderRows,
   rowKey,
   readChainId,
   handleLoad,

package/src/Utils.res

@@ -199,6 +199,13 @@ module Dict = {
     }
   `)
 
+  let clearInPlace: dict<'a> => unit = %raw(`(dict) => {
+      for (const key in dict) {
+        delete dict[key];
+      }
+    }
+  `)
+
   let unsafeDeleteUndefinedFieldsInPlace: 'a => unit = %raw(`(dict) => {
       for (var key in dict) {
         if (dict[key] === undefined) {

package/src/Utils.res.mjs

@@ -149,6 +149,12 @@ let deleteInPlace = ((dict, key) => {
       delete dict[key];
     });
 
+let clearInPlace = ((dict) => {
+      for (const key in dict) {
+        delete dict[key];
+      }
+    });
+
 let unsafeDeleteUndefinedFieldsInPlace = ((dict) => {
       for (var key in dict) {
         if (dict[key] === undefined) {
@@ -175,6 +181,7 @@ let Dict = {
   size: size,
   isEmpty: isEmpty,
   deleteInPlace: deleteInPlace,
+  clearInPlace: clearInPlace,
   unsafeDeleteUndefinedFieldsInPlace: unsafeDeleteUndefinedFieldsInPlace,
   updateImmutable: updateImmutable,
   shallowCopy: shallowCopy
@@ -249,7 +256,7 @@ function mergeSorted(f, xs, ys) {
   return result;
 }
 
-let clearInPlace = ((arr) => {
+let clearInPlace$1 = ((arr) => {
     arr.length = 0
   });
 
@@ -387,7 +394,7 @@ let $$Array$1 = {
   unzip: Stdlib_Array.unzip,
   immutableEmpty: immutableEmpty,
   mergeSorted: mergeSorted,
-  clearInPlace: clearInPlace,
+  clearInPlace: clearInPlace$1,
   setIndexImmutable: setIndexImmutable,
   transposeResults: transposeResults,
   includes: includes,

package/src/Writing.res

@@ -120,6 +120,7 @@ let runOneWrite = async (state: IndexerState.t) => {
       }
     })
     let updatedEffectsCache = snapshotEffects(state, ~cache)
+    let registeredAddresses = batch.registeredAddresses
 
     let writtenEntityNames = Utils.Set.make()
     updatedEntities->Array.forEach(({entityConfig}) =>
@@ -145,6 +146,7 @@ let runOneWrite = async (state: IndexerState.t) => {
         ~config,
         ~allEntities=persistence.allEntities,
         ~updatedEntities,
+        ~registeredAddresses,
         ~updatedEffectsCache,
         ~chainMetaData,
         ~onWrite=(~storage, ~timeSeconds) =>

package/src/Writing.res.mjs

@@ -118,13 +118,14 @@ async function runOneWrite(state) {
     }
   });
   let updatedEffectsCache = snapshotEffects(state, cache);
+  let registeredAddresses = batch.registeredAddresses;
   let writtenEntityNames = new Set();
   updatedEntities.forEach(param => {
     writtenEntityNames.add(param.entityConfig.name);
   });
   let pruneTargets = PruneStaleHistory.select(state, writtenEntityNames, Stdlib_Option.isSome(rollback));
   await Promise.all([
-    persistence.storage.writeBatch(batch, rollback, batch.isInReorgThreshold, config, persistence.allEntities, updatedEffectsCache, updatedEntities, chainMetaData, (storage, timeSeconds) => IndexerState.recordStorageWrite(state, storage, timeSeconds)),
+    persistence.storage.writeBatch(batch, rollback, batch.isInReorgThreshold, config, persistence.allEntities, updatedEffectsCache, updatedEntities, registeredAddresses, chainMetaData, (storage, timeSeconds) => IndexerState.recordStorageWrite(state, storage, timeSeconds)),
     PruneStaleHistory.runConcurrent(state, pruneTargets)
   ]);
   IndexerState.markCommitted(state, upToCheckpointId);

package/src/bindings/ClickHouse.res

@@ -60,6 +60,12 @@ let getClickHouseFieldType = (
     }
   | Uint32 => "UInt32"
   | UInt52 => "UInt64"
+  // Internal-only column types, never on an entity the sink mirrors.
+  | SmallInt
+  | Bytea =>
+    JsError.throwWithMessage(
+      "ClickHouse doesn't support the internal SmallInt and Bytea column types",
+    )
   | UInt64 => "UInt64"
   | Serial => "Int32"
   | BigSerial => "Int64"
@@ -357,6 +363,17 @@ let setUpdatesOrThrow = async (
 // The '{cluster}' macro resolves to each node's configured cluster name.
 let onClusterClause = (~onCluster: bool) => onCluster ? ` ON CLUSTER '{cluster}'` : ""
 
+// ReplicatedMergeTree drops an insert whose block hash is already in Keeper, and
+// mutations don't clear those hashes. Crash recovery trims the history tail past
+// the committed Postgres checkpoint with ALTER ... DELETE and then replays it, so
+// an identical replayed block would be discarded while still reporting success —
+// a permanent gap that nothing surfaces. Trim-then-replay is what makes recovery
+// correct here, and the duplicates dedup would have caught are already collapsed
+// by the entity view's `LIMIT 1 BY`. Plain MergeTree goes by
+// non_replicated_deduplication_window (0 by default), so it needs no clause.
+let replicatedTableSettingsClause = (~replicated: bool) =>
+  replicated ? "\nSETTINGS replicated_deduplication_window = 0" : ""
+
 // Strip both engine arguments `(...)` and a trailing `SETTINGS ...` clause to
 // get the bare engine name, e.g. `Replicated('/p','{shard}','{replica}') SETTINGS x=1`
 // and `Replicated SETTINGS x=1` both yield `Replicated`.
@@ -395,9 +412,9 @@ let makeCreateHistoryTableQuery = (
     }
   })
 
-  let (partitionBy, orderBy, ttl) = switch entityConfig.storage.clickhouseOptions {
-  | Some(options) => (options.partitionBy, options.orderBy, options.ttl)
-  | None => (None, None, None)
+  let (partitionBy, orderBy, ttl, skippingIndexes) = switch entityConfig.storage.clickhouseOptions {
+  | Some(options) => (options.partitionBy, options.orderBy, options.ttl, options.skippingIndexes)
+  | None => (None, None, None, None)
   }
 
   // Schema field name -> ClickHouse column name, so @storage(clickhouse: {...})
@@ -461,6 +478,22 @@ let makeCreateHistoryTableQuery = (
   | None => ""
   }
 
+  // Data skipping indexes live inside the column list, after the last column.
+  // GRANULARITY is omitted when unset, leaving ClickHouse's default of 1.
+  let skippingIndexDefinitions = switch skippingIndexes {
+  | Some(skippingIndexes) =>
+    skippingIndexes
+    ->Array.map(index => {
+      let granularityClause = switch index.granularity {
+      | Some(granularity) => ` GRANULARITY ${granularity->Int.toString}`
+      | None => ""
+      }
+      `,\n  INDEX \`${index.name}\` ${index.expr->resolveExpressionColumns} TYPE ${index.type_}${granularityClause}`
+    })
+    ->Array.joinUnsafe("")
+  | None => ""
+  }
+
   `CREATE TABLE IF NOT EXISTS ${database}.\`${EntityHistory.historyTableName(
       ~entityName=entityConfig.name,
       ~entityIndex=entityConfig.index,
@@ -475,10 +508,10 @@ let makeCreateHistoryTableQuery = (
       ~fieldType=Enum({config: EntityHistory.RowAction.config->Table.fromGenericEnumConfig}),
       ~isNullable=false,
       ~isArray=false,
-    )}
+    )}${skippingIndexDefinitions}
 )
 ENGINE = ${tableEngine}${partitionByClause}
-ORDER BY (${orderByColumns})${ttlClause}`
+ORDER BY (${orderByColumns})${ttlClause}${replicatedTableSettingsClause(~replicated)}`
 }
 
 // Generate CREATE TABLE query for checkpoints
@@ -522,7 +555,7 @@ let makeCreateCheckpointsTableQuery = (
     )}
 )
 ENGINE = ${tableEngine}
-ORDER BY (${idField})`
+ORDER BY (${idField})${replicatedTableSettingsClause(~replicated)}`
 }
 
 // Generate CREATE VIEW query for entity current state

package/src/bindings/ClickHouse.res.mjs

@@ -26,6 +26,10 @@ function getClickHouseFieldType(fieldType, isNullable, isArray, chainIdModeOpt)
       case "Uint32" :
         baseType = "UInt32";
         break;
+      case "SmallInt" :
+      case "Bytea" :
+        baseType = Stdlib_JsError.throwWithMessage("ClickHouse doesn't support the internal SmallInt and Bytea column types");
+        break;
       case "UInt52" :
       case "UInt64" :
         baseType = "UInt64";
@@ -273,6 +277,14 @@ function onClusterClause(onCluster) {
   }
 }
 
+function replicatedTableSettingsClause(replicated) {
+  if (replicated) {
+    return "\nSETTINGS replicated_deduplication_window = 0";
+  } else {
+    return "";
+  }
+}
+
 function databaseEngineName(engineSpec) {
   return engineSpec.trim().split("(")[0].split(" ")[0].trim();
 }
@@ -295,12 +307,15 @@ function makeCreateHistoryTableQuery(entityConfig, database, replicatedOpt, onCl
   let match = options !== undefined ? [
       options.partitionBy,
       options.orderBy,
-      options.ttl
+      options.ttl,
+      options.skippingIndexes
     ] : [
+      undefined,
       undefined,
       undefined,
       undefined
     ];
+  let skippingIndexes = match[3];
   let ttl = match[2];
   let orderBy = match[1];
   let partitionBy = match[0];
@@ -336,6 +351,11 @@ function makeCreateHistoryTableQuery(entityConfig, database, replicatedOpt, onCl
   });
   let partitionByClause = partitionBy !== undefined ? `\nPARTITION BY ` + resolveExpressionColumns(partitionBy) : "";
   let ttlClause = ttl !== undefined ? `\nTTL ` + resolveExpressionColumns(ttl) : "";
+  let skippingIndexDefinitions = skippingIndexes !== undefined ? skippingIndexes.map(index => {
+      let granularity = index.granularity;
+      let granularityClause = granularity !== undefined ? ` GRANULARITY ` + granularity.toString() : "";
+      return `,\n  INDEX \`` + index.name + `\` ` + resolveExpressionColumns(index.expr) + ` TYPE ` + index.type + granularityClause;
+    }).join("") : "";
   return `CREATE TABLE IF NOT EXISTS ` + database + `.\`` + EntityHistory.historyTableName(entityConfig.name, entityConfig.index) + `\`` + (
     onCluster ? ` ON CLUSTER '{cluster}'` : ""
   ) + ` (
@@ -344,10 +364,12 @@ function makeCreateHistoryTableQuery(entityConfig, database, replicatedOpt, onCl
   \`` + EntityHistory.changeFieldName + `\` ` + getClickHouseFieldType({
     type: "Enum",
     config: EntityHistory.RowAction.config
-  }, false, false, undefined) + `
+  }, false, false, undefined) + skippingIndexDefinitions + `
 )
 ENGINE = ` + tableEngine + partitionByClause + `
-ORDER BY (` + orderByColumns + `)` + ttlClause;
+ORDER BY (` + orderByColumns + `)` + ttlClause + (
+    replicated ? "\nSETTINGS replicated_deduplication_window = 0" : ""
+  );
 }
 
 function makeCreateCheckpointsTableQuery(database, replicatedOpt, onClusterOpt, chainIdModeOpt) {
@@ -365,7 +387,9 @@ function makeCreateCheckpointsTableQuery(database, replicatedOpt, onClusterOpt,
   \`` + "events_processed" + `\` ` + getClickHouseFieldType("UInt64", false, false, undefined) + `
 )
 ENGINE = ` + tableEngine + `
-ORDER BY (` + "id" + `)`;
+ORDER BY (` + "id" + `)` + (
+    replicated ? "\nSETTINGS replicated_deduplication_window = 0" : ""
+  );
 }
 
 function makeCreateViewQuery(entityConfig, database, onClusterOpt) {
@@ -505,6 +529,7 @@ export {
   setCheckpointsOrThrow,
   setUpdatesOrThrow,
   onClusterClause,
+  replicatedTableSettingsClause,
   databaseEngineName,
   makeCreateHistoryTableQuery,
   makeCreateCheckpointsTableQuery,

package/src/bindings/NodeJs.res

@@ -66,6 +66,15 @@ module Process = {
   external getActiveResourcesInfo: unit => array<string> = "getActiveResourcesInfo"
 }
 
+module Buffer = {
+  type t
+  @val @scope("Buffer") external concat: array<t> => t = "concat"
+  @val @scope("Buffer") external alloc: int => t = "alloc"
+  @get external length: t => int = "length"
+  @send external toBase64: (t, @as("base64") _) => string = "toString"
+  let empty = alloc(0)
+}
+
 module V8 = {
   type heapSpaceStatistics = {
     @as("space_name") spaceName: string,

package/src/bindings/NodeJs.res.mjs

@@ -18,6 +18,12 @@ let Util$1 = {
 
 let Process = {};
 
+let empty = globalThis.Buffer.alloc(0);
+
+let Buffer = {
+  empty: empty
+};
+
 let V8 = {};
 
 let PerfHooks = {};
@@ -49,6 +55,7 @@ let Fs = {
 export {
   Util$1 as Util,
   Process,
+  Buffer,
   V8,
   PerfHooks,
   ChildProcess,
@@ -59,4 +66,4 @@ export {
   WorkerThreads,
   Fs,
 }
-/* url Not a pure module */
+/* empty Not a pure module */

package/src/bindings/Postgres.res

@@ -99,6 +99,13 @@ external makeSql: (~config: poolConfig) => sql = "default"
 // @send @variadic
 // external sql:  array<string>  => (sql, array<string>) => int = "sql"
 
+// postgres.js infers a Buffer array's type from its first element, so a bytea
+// column's array parameter comes out typed `bytea` and the server refuses the
+// cast. Naming the array type explicitly is what makes it bind. 1001 is
+// Postgres' `bytea[]` OID.
+let byteaArrayOid = 1001
+@send external typed: (sql, 'a, int) => unknown = "typed"
+
 @send external unsafe: (sql, string) => promise<'a> = "unsafe"
 @send external unpreparedUnsafe: (sql, string, unknown) => promise<'a> = "unsafe"
 @send
@@ -107,8 +114,10 @@ external preparedUnsafe: (sql, string, unknown, @as(json`{prepare: true}`) _) =>
 
 @unboxed
 type columnType =
+  | @as("SMALLINT") SmallInt
   | @as("INTEGER") Integer
   | @as("BIGINT") BigInt
+  | @as("BYTEA") Bytea
   | @as("BOOLEAN") Boolean
   | @as("NUMERIC") Numeric
   | @as("DOUBLE PRECISION") DoublePrecision