envio 3.9.0 → 3.10.0-subgraph

package/src/UserContext.res.mjs

@@ -1,6 +1,7 @@
 // Generated by ReScript, PLEASE EDIT WITH CARE
 
 import * as Utils from "./Utils.res.mjs";
+import * as ChainId from "./ChainId.res.mjs";
 import * as Logging from "./Logging.res.mjs";
 import * as Internal from "./Internal.res.mjs";
 import * as Ecosystem from "./Ecosystem.res.mjs";
@@ -12,8 +13,70 @@ import * as InMemoryStore from "./InMemoryStore.res.mjs";
 import * as InMemoryTable from "./InMemoryTable.res.mjs";
 import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js";
 import * as Stdlib_JsError from "@rescript/runtime/lib/es6/Stdlib_JsError.js";
+import * as Stdlib_Promise from "@rescript/runtime/lib/es6/Stdlib_Promise.js";
 import * as Primitive_option from "@rescript/runtime/lib/es6/Primitive_option.js";
 import * as S$RescriptSchema from "rescript-schema/src/S.res.mjs";
+import * as Primitive_exceptions from "@rescript/runtime/lib/es6/Primitive_exceptions.js";
+
+let Suspend = /* @__PURE__ */Primitive_exceptions.create("UserContext.Suspend");
+
+function isSuspend(exn) {
+  return exn.RE_EXN_ID === Suspend;
+}
+
+function makeSyncState() {
+  return {
+    status: "Active",
+    pending: undefined,
+    memo: undefined
+  };
+}
+
+function getPending(sync) {
+  let pending = sync.pending;
+  if (pending !== undefined) {
+    return pending;
+  }
+  let pending$1 = [];
+  sync.pending = pending$1;
+  return pending$1;
+}
+
+function getMemo(sync) {
+  let memo = sync.memo;
+  if (memo !== undefined) {
+    return memo;
+  }
+  let memo$1 = {};
+  sync.memo = memo$1;
+  return memo$1;
+}
+
+function checkStatusOrThrow(params, access) {
+  let exn = params.sync.status;
+  if (typeof exn !== "object") {
+    if (exn === "Active") {
+      return;
+    } else {
+      return ErrorHandling.mkLogAndRaise(Ecosystem.getItemLogger(params.item, params.config.ecosystem), undefined, new Error(`Impossible to access ` + access + ` after the handler is resolved. Make sure you didn't miss an await in the handler.`));
+    }
+  }
+  throw exn._0;
+}
+
+function scheduleAndSuspend(params, promise) {
+  getPending(params.sync).push(promise);
+  params.sync.status = {
+    TAG: "Aborted",
+    _0: {
+      RE_EXN_ID: Suspend
+    }
+  };
+  throw {
+    RE_EXN_ID: Suspend,
+    Error: new Error()
+  };
+}
 
 let paramsByThis = new WeakMap();
 
@@ -47,28 +110,69 @@ var EffectContext = function(params, chainId, effectName, defaultShouldCache, ca
 EffectContext.prototype = effectContextPrototype;
 ;
 
-function initEffect(params) {
-  let handlerChainId = Internal.getItemChainId(params.item);
+function prepareEffectCall(params, effect, input, caller) {
   let isCrossChain = effect => Stdlib_Option.getOr(effect.crossChain, params.config.defaultCrossChain);
-  let makeCaller = caller => ((effect, input) => {
-    let scope = isCrossChain(effect) ? "crossChain" : handlerChainId;
-    if (caller !== undefined && isCrossChain(caller) && !isCrossChain(effect)) {
-      Stdlib_JsError.throwWithMessage(`The cross-chain effect "` + caller.name + `" cannot call the chain-scoped effect "` + effect.name + `", because a cross-chain effect isn't tied to a single chain. Make "` + effect.name + `" cross-chain (\`crossChain: true\`), or make "` + caller.name + `" chain-scoped (\`crossChain: false\`).`);
+  let scope = isCrossChain(effect) ? "crossChain" : Internal.getItemChainId(params.item);
+  if (caller !== undefined && isCrossChain(caller) && !isCrossChain(effect)) {
+    Stdlib_JsError.throwWithMessage(`The cross-chain effect "` + caller.name + `" cannot call the chain-scoped effect "` + effect.name + `", because a cross-chain effect isn't tied to a single chain. Make "` + effect.name + `" cross-chain (\`crossChain: true\`), or make "` + caller.name + `" chain-scoped (\`crossChain: false\`).`);
+  }
+  let tmp;
+  tmp = scope === "crossChain" ? undefined : Primitive_option.some(scope);
+  let effectContext = new EffectContext(params, tmp, effect.name, effect.defaultShouldCache, (nested, nestedInput) => callEffectAsync(params, nested, nestedInput, effect));
+  let effectArgs_cacheKey = Utils.Hash.makeOrThrow(S$RescriptSchema.reverseConvertOrThrow(input, effect.input));
+  let effectArgs_checkpointId = params.checkpointId;
+  let effectArgs = {
+    input: input,
+    context: effectContext,
+    cacheKey: effectArgs_cacheKey,
+    checkpointId: effectArgs_checkpointId
+  };
+  return [
+    scope,
+    effectArgs
+  ];
+}
+
+function callEffectAsync(params, effect, input, caller) {
+  let match = prepareEffectCall(params, effect, input, caller);
+  return LoadLayer.loadEffect(params.loadManager, params.persistence, effect, match[1], match[0], params.indexerState, params.isPreload, params.item, params.config.ecosystem);
+}
+
+function initEffect(params) {
+  return (effect, input) => {
+    checkStatusOrThrow(params, "context.effect");
+    return callEffectAsync(params, effect, input, undefined);
+  };
+}
+
+function effectMemoKey(effect, scope, cacheKey) {
+  if (scope === "crossChain") {
+    return effect.name + `.` + cacheKey;
+  } else {
+    return effect.name + `.` + ChainId.toString(scope) + `.` + cacheKey;
+  }
+}
+
+function initEffectSync(params) {
+  return (effect, input) => {
+    checkStatusOrThrow(params, "context.effectSync");
+    let match = prepareEffectCall(params, effect, input, undefined);
+    let effectArgs = match[1];
+    let scope = match[0];
+    let memo = getMemo(params.sync);
+    let memoKey = effectMemoKey(effect, scope, effectArgs.cacheKey);
+    let output = memo[memoKey];
+    if (output !== undefined) {
+      return Primitive_option.valFromOption(output);
     }
-    let tmp;
-    tmp = scope === "crossChain" ? undefined : Primitive_option.some(scope);
-    let effectContext = new EffectContext(params, tmp, effect.name, effect.defaultShouldCache, makeCaller(effect));
-    let effectArgs_cacheKey = Utils.Hash.makeOrThrow(S$RescriptSchema.reverseConvertOrThrow(input, effect.input));
-    let effectArgs_checkpointId = params.checkpointId;
-    let effectArgs = {
-      input: input,
-      context: effectContext,
-      cacheKey: effectArgs_cacheKey,
-      checkpointId: effectArgs_checkpointId
-    };
-    return LoadLayer.loadEffect(params.loadManager, params.persistence, effect, effectArgs, scope, params.indexerState, params.isPreload, params.item, params.config.ecosystem);
-  });
-  return makeCaller(undefined);
+    let inMemTable = InMemoryStore.getEffectInMemTable(params.indexerState, effect, scope);
+    if (!InMemoryStore.hasEffectOutput(inMemTable, effectArgs.cacheKey)) {
+      return scheduleAndSuspend(params, LoadLayer.loadEffect(params.loadManager, params.persistence, effect, effectArgs, scope, params.indexerState, params.isPreload, params.item, params.config.ecosystem));
+    }
+    let output$1 = InMemoryStore.getEffectOutputUnsafe(inMemTable, effectArgs.cacheKey);
+    memo[memoKey] = output$1;
+    return output$1;
+  };
 }
 
 function entityScope(params) {
@@ -97,24 +201,83 @@ function throwClickHouseReadOnly(entityConfig, op) {
   return Stdlib_JsError.throwWithMessage(`context.` + entityConfig.name + `.` + op + `() is unavailable: ClickHouse storage is currently write-only. Follow Envio releases to be notified when ClickHouse supports both reads and writes from handlers.`);
 }
 
+function getSyncHandler(params, entityId) {
+  let inMemTable = InMemoryStore.getInMemTable(params.indexerState, params.entityConfig, entityScope(params));
+  if (entityId in inMemTable.latestEntityChangeById) {
+    return InMemoryTable.Entity.getUnsafe(inMemTable)(entityId);
+  } else {
+    return scheduleAndSuspend(params, LoadLayer.loadById(params.loadManager, params.persistence, params.entityConfig, entityScope(params), params.indexerState, params.isPreload, params.item, params.config.ecosystem, entityId));
+  }
+}
+
+function getWhereSyncHandler(params, filter) {
+  let entityConfig = params.entityConfig;
+  let inMemTable = InMemoryStore.getInMemTable(params.indexerState, entityConfig, entityScope(params));
+  let hasIndex = InMemoryTable.Entity.hasIndex(inMemTable);
+  let getOnIndex = InMemoryTable.Entity.getUnsafeOnIndex(inMemTable);
+  let filters = EntityFilter.parseGetWhereOrThrow(filter, entityConfig.name, entityConfig.table);
+  let missing = [];
+  let entities = [];
+  filters.forEach(filter => {
+    let filterKey = EntityFilter.toString(filter);
+    if (hasIndex(filterKey)) {
+      entities.push(...getOnIndex(filterKey));
+    } else {
+      missing.push(LoadLayer.loadByFilter(params.loadManager, params.persistence, entityConfig, entityScope(params), params.indexerState, params.isPreload, params.item, params.config.ecosystem, filter));
+    }
+  });
+  if (Utils.$$Array.notEmpty(missing)) {
+    let pending = getPending(params.sync);
+    missing.forEach(promise => {
+      pending.push(promise);
+    });
+    params.sync.status = {
+      TAG: "Aborted",
+      _0: {
+        RE_EXN_ID: Suspend
+      }
+    };
+    throw {
+      RE_EXN_ID: Suspend,
+      Error: new Error()
+    };
+  }
+  return entities;
+}
+
+function getInBlockSyncHandler(params, entityId) {
+  let inMemTable = InMemoryStore.getInMemTable(params.indexerState, params.entityConfig, entityScope(params));
+  let change = inMemTable.latestEntityChangeById[entityId];
+  if (change !== undefined && change.checkpointId === params.checkpointId) {
+    return InMemoryTable.Entity.mapChangeToEntity(change);
+  }
+}
+
 let entityTraps_get = (params, prop) => {
+  checkStatusOrThrow(params, `context.` + params.entityConfig.name + `.` + prop);
   let isClickHouseOnly = !params.entityConfig.storage.postgres;
-  let set = params.isPreload ? noopSet : entity => InMemoryTable.Entity.set(InMemoryStore.getInMemTable(params.indexerState, params.entityConfig, entityScope(params)), IndexerState.committedCheckpointId(params.indexerState), {
-      type: "SET",
-      entityId: entity.id,
-      entity: entity,
-      checkpointId: params.checkpointId
-    });
+  let set = params.isPreload ? noopSet : entity => {
+      checkStatusOrThrow(params, `context.` + params.entityConfig.name + `.set`);
+      InMemoryTable.Entity.set(InMemoryStore.getInMemTable(params.indexerState, params.entityConfig, entityScope(params)), IndexerState.committedCheckpointId(params.indexerState), {
+        type: "SET",
+        entityId: entity.id,
+        entity: entity,
+        checkpointId: params.checkpointId
+      });
+    };
   switch (prop) {
     case "deleteUnsafe" :
       if (params.isPreload) {
         return noopDeleteUnsafe;
       } else {
-        return entityId => InMemoryTable.Entity.set(InMemoryStore.getInMemTable(params.indexerState, params.entityConfig, entityScope(params)), IndexerState.committedCheckpointId(params.indexerState), {
-          type: "DELETE",
-          entityId: entityId,
-          checkpointId: params.checkpointId
-        });
+        return entityId => {
+          checkStatusOrThrow(params, `context.` + params.entityConfig.name + `.deleteUnsafe`);
+          InMemoryTable.Entity.set(InMemoryStore.getInMemTable(params.indexerState, params.entityConfig, entityScope(params)), IndexerState.committedCheckpointId(params.indexerState), {
+            type: "DELETE",
+            entityId: entityId,
+            checkpointId: params.checkpointId
+          });
+        };
       }
     case "get" :
       if (isClickHouseOnly) {
@@ -122,6 +285,8 @@ let entityTraps_get = (params, prop) => {
       } else {
         return entityId => LoadLayer.loadById(params.loadManager, params.persistence, params.entityConfig, entityScope(params), params.indexerState, params.isPreload, params.item, params.config.ecosystem, entityId);
       }
+    case "getInBlockSync" :
+      return entityId => getInBlockSyncHandler(params, entityId);
     case "getOrCreate" :
       if (isClickHouseOnly) {
         return _entity => throwClickHouseReadOnly(params.entityConfig, "getOrCreate");
@@ -147,12 +312,16 @@ let entityTraps_get = (params, prop) => {
           }
         });
       }
+    case "getSync" :
+      return entityId => getSyncHandler(params, entityId);
     case "getWhere" :
       if (isClickHouseOnly) {
         return _filter => throwClickHouseReadOnly(params.entityConfig, "getWhere");
       } else {
         return filter => getWhereHandler(params, filter);
       }
+    case "getWhereSync" :
+      return filter => getWhereSyncHandler(params, filter);
     case "set" :
       return set;
     default:
@@ -164,16 +333,63 @@ let entityTraps = {
   get: entityTraps_get
 };
 
-let handlerTraps_get = (params, prop) => {
-  if (params.isResolved) {
-    ErrorHandling.mkLogAndRaise(Ecosystem.getItemLogger(params.item, params.config.ecosystem), undefined, new Error(`Impossible to access context.` + prop + ` after the handler is resolved. Make sure you didn't miss an await in the handler.`));
+async function runSyncRound(params, fn, round) {
+  if (round > 10000) {
+    Stdlib_JsError.throwWithMessage(`The handler suspended on a synchronous read too many times: gave up after ` + (10000).toString() + ` rounds. This usually means the code isn't deterministic across reruns.`);
+  }
+  params.sync.status = "Active";
+  params.sync.pending = undefined;
+  let suspended;
+  try {
+    fn();
+    suspended = false;
+  } catch (raw_exn) {
+    let exn = Primitive_exceptions.internalToException(raw_exn);
+    if (exn.RE_EXN_ID === Suspend) {
+      suspended = true;
+    } else {
+      let pending = params.sync.pending;
+      if (pending !== undefined) {
+        params.sync.pending = undefined;
+        pending.forEach(promise => {
+          Utils.$$Promise.silentCatch(promise);
+        });
+      }
+      throw exn;
+    }
   }
+  let pending$1 = params.sync.pending;
+  if (pending$1 === undefined) {
+    return;
+  }
+  params.sync.pending = undefined;
+  if (suspended) {
+    let errors = [];
+    await Promise.all(pending$1.map(promise => Stdlib_Promise.$$catch(promise, exn => {
+      errors.push(exn);
+      return Promise.resolve();
+    })));
+    let exn$1 = errors[0];
+    if (exn$1 !== undefined) {
+      throw exn$1;
+    }
+    return await runSyncRound(params, fn, round + 1 | 0);
+  }
+  pending$1.forEach(promise => {
+    Utils.$$Promise.silentCatch(promise);
+  });
+}
+
+let handlerTraps_get = (params, prop) => {
+  checkStatusOrThrow(params, `context.` + prop);
   switch (prop) {
     case "chain" :
       let chainId = Internal.getItemChainId(params.item);
       return params.chains[chainId];
     case "effect" :
       return initEffect(params);
+    case "effectSync" :
+      return initEffectSync(params);
     case "isPreload" :
       return params.isPreload;
     case "log" :
@@ -182,6 +398,8 @@ let handlerTraps_get = (params, prop) => {
       } else {
         return Ecosystem.getItemUserLogger(params.item, params.config.ecosystem);
       }
+    case "runSync" :
+      return fn => runSyncRound(params, fn, 1);
     default:
       let entityConfig = params.config.userEntitiesByName[prop];
       if (entityConfig !== undefined) {
@@ -194,7 +412,7 @@ let handlerTraps_get = (params, prop) => {
           isPreload: params.isPreload,
           chains: params.chains,
           config: params.config,
-          isResolved: params.isResolved,
+          sync: params.sync,
           entityConfig: entityConfig
         }, entityTraps);
       } else {
@@ -211,16 +429,34 @@ function getHandlerContext(params) {
   return new Proxy(params, handlerTraps);
 }
 
+let maxSyncRounds = 10000;
+
 export {
+  Suspend,
+  isSuspend,
+  makeSyncState,
+  getPending,
+  getMemo,
+  checkStatusOrThrow,
+  scheduleAndSuspend,
   paramsByThis,
   effectContextPrototype,
+  prepareEffectCall,
+  callEffectAsync,
   initEffect,
+  effectMemoKey,
+  initEffectSync,
   entityScope,
   getWhereHandler,
   noopSet,
   noopDeleteUnsafe,
   throwClickHouseReadOnly,
+  getSyncHandler,
+  getWhereSyncHandler,
+  getInBlockSyncHandler,
   entityTraps,
+  maxSyncRounds,
+  runSyncRound,
   handlerTraps,
   getHandlerContext,
 }

package/src/Utils.res

@@ -18,6 +18,33 @@ external importPathWithJson: (
   "default": JSON.t,
 }> = "import"
 
+// Half the delay fixed, half spread across it. Every indexer pointed at one
+// provider loses its stream, or gives up on a quiet chain, in the same instant
+// that provider blinks, and putting them all back on the same schedule is how a
+// blip becomes a stampede.
+let jitter = delay => delay / 2 + (Math.random() *. (delay / 2)->Int.toFloat)->Float.toInt
+
+// Clearing a pending timer and forgetting it are one action. A ref left holding
+// a spent id reads as a timer still pending, and the next clear looks like it
+// did something.
+let clearTimeoutRef = timeoutId => {
+  switch timeoutId.contents {
+  | Some(id) => clearTimeout(id)
+  | None => ()
+  }
+  timeoutId := None
+}
+
+// Keeps the doubling from overflowing on something that has been failing for
+// days. Every caller's delay reaches its cap long before this.
+let maxBackoffExponent = 20
+
+let expBackoff = (~base, ~exp, ~maxMillis) =>
+  Pervasives.min(
+    base * Math.pow(2.0, ~exp=Pervasives.min(exp, maxBackoffExponent)->Int.toFloat)->Float.toInt,
+    maxMillis,
+  )
+
 let delay = milliseconds =>
   Promise.make((resolve, _) => {
     let _interval = setTimeout(_ => {
@@ -97,6 +124,15 @@ module Dict = {
   @get_index
   external dangerouslyGetNonOption: (dict<'a>, string) => option<'a> = ""
 
+  let incrementBy = (dict, key, count) =>
+    dict->Dict.set(
+      key,
+      switch dict->dangerouslyGetNonOption(key) {
+      | Some(current) => current + count
+      | None => count
+      },
+    )
+
   let getOrInsertEmptyDict = (dict, key) => {
     switch dict->dangerouslyGetNonOption(key) {
     | Some(d) => d
@@ -253,6 +289,10 @@ module Array = {
 
   let immutableEmpty: array<unknown> = []
 
+  /** Render names for an error message: `"a", "b"`. */
+  let quotedJoin = (names: array<string>) =>
+    names->Array.map(name => `"${name}"`)->Array.joinUnsafe(", ")
+
   @send
   external forEachAsync: (array<'a>, 'a => promise<unit>) => unit = "forEach"
 
@@ -491,9 +531,120 @@ let unwrapResultExn = res =>
 
 external queueMicrotask: (unit => unit) => unit = "queueMicrotask"
 
+module Bytes = {
+  let asUint8Array: unknown => option<Uint8Array.t> = %raw(`(value) =>
+    value instanceof Uint8Array
+      ? value.constructor === Uint8Array
+        ? value
+        : new Uint8Array(value.buffer, value.byteOffset, value.byteLength)
+      : undefined`)
+
+  let toHex = (bytes: Uint8Array.t) =>
+    NodeJs.Buffer.fromArrayBuffer(
+      bytes->TypedArray.buffer,
+      ~byteOffset=bytes->TypedArray.byteOffset,
+      ~length=bytes->TypedArray.byteLength,
+    )->NodeJs.Buffer.toHex
+
+  @get_index external byteAt: (Uint8Array.t, int) => int = ""
+
+  // A bytea[] parameter as the array literal Postgres parses itself. postgres.js
+  // serializes an array parameter with the element serializer of the type the
+  // server describes, and the bytea one expects a Uint8Array — yet it types the
+  // array after its first element, so an array of Uint8Arrays binds as a single
+  // bytea. Text binds untyped and reaches the server's array parser as it is.
+  //
+  // Nests, since an `in` filter over a list column carries one array per
+  // candidate value. A sub-array is written unquoted, which is how Postgres
+  // spells a dimension rather than an element.
+  let rec toPgArrayLiteral = (values: array<unknown>) =>
+    "{" ++
+    values
+    ->Array.map(value =>
+      switch value->(magic: unknown => Nullable.t<unknown>)->Nullable.toOption {
+      | None => "NULL"
+      | Some(value) =>
+        switch value->asUint8Array {
+        | Some(bytes) => `"\\\\x${bytes->toHex}"`
+        | None =>
+          if value->Array.isArray {
+            value->(magic: unknown => array<unknown>)->toPgArrayLiteral
+          } else {
+            JsError.throwWithMessage("Expected Uint8Array")
+          }
+        }
+      }
+    )
+    ->Array.join(",") ++ "}"
+
+  // Bytewise, the order Postgres sorts a bytea in.
+  let compare = (a: Uint8Array.t, b: Uint8Array.t) => {
+    let aLength = a->TypedArray.length
+    let bLength = b->TypedArray.length
+    let rec loop = index =>
+      if index === aLength || index === bLength {
+        Int.compare(aLength, bLength)
+      } else {
+        switch Int.compare(a->byteAt(index), b->byteAt(index)) {
+        | 0. => loop(index + 1)
+        | order => order
+        }
+      }
+    loop(0)
+  }
+}
+
 module Schema = {
   let variantTag = S.union([S.string, S.object(s => s.field("TAG", S.string))])
 
+  // A ReScript `option` is `undefined` at runtime, so `S.null`'s serializer
+  // treats only `undefined` as the empty case — a genuine `null`, which a JS
+  // handler writes for a field it doesn't set, reaches the value serializer
+  // and crashes it (`null.toString()` for a BigInt column). Collapse both to
+  // `None` before the value serializer sees them.
+  //
+  // Revisit on the Sury v11 migration: if its nullable handles a `null` value
+  // on the serialize side, this wrapper goes away.
+  let nullTolerant = schema =>
+    S.null(schema)->S.transform(_ => {
+      parser: value => value,
+      serializer: value => value->(magic: option<'a> => Nullable.t<'a>)->Nullable.toOption,
+    })
+
+  // Postgres hands a bytea back as a Buffer, while handlers are promised the
+  // plain Uint8Array of the entity type — Buffer's `slice` and `toString` behave
+  // differently.
+  let bytes = S.custom("Bytes", s => {
+    parser: unknown =>
+      switch unknown->Bytes.asUint8Array {
+      | Some(bytes) => bytes
+      | None => s.fail("Expected Uint8Array")
+      },
+    serializer: (bytes: Uint8Array.t) => bytes,
+  })
+
+  // A bytea[] value: a list column, or the values an `in` filter compares
+  // against — one array per candidate when the column is itself a list, which
+  // the literal writer nests. Only a column is ever read back, so the parser
+  // takes the one level a bytea[] column returns.
+  let bytesArray = S.custom("BytesArray", s => {
+    parser: unknown =>
+      if unknown->Array.isArray {
+        unknown
+        ->(magic: unknown => array<unknown>)
+        ->Array.map(item =>
+          switch item->Bytes.asUint8Array {
+          | Some(bytes) => bytes
+          | None => s.fail("Expected Uint8Array")
+          }
+        )
+      } else {
+        s.fail("Expected array of Uint8Array")
+      },
+    serializer: (values: array<Uint8Array.t>) =>
+      values->(magic: array<Uint8Array.t> => array<unknown>)->Bytes.toPgArrayLiteral,
+  })
+
   // Don't use S.unknown, since it's not serializable to json
   // In a nutshell, this is completely unsafe.
   let dbDate =
@@ -501,6 +652,19 @@ module Schema = {
     ->(magic: S.t<JSON.t> => S.t<Date.t>)
     ->S.preprocess(_ => {serializer: date => date->magic->Date.toISOString})
 
+  // JSON `null` is a document, and Postgres stores it as one. Reaching the
+  // ClickHouse sink as a JS `null` it would instead read as a field the handler
+  // never set, which a String column has no way to hold — so it travels as the
+  // text it would have been serialized to. Every other document is left for the
+  // sink to serialize.
+  let clickHouseJson = S.json(~validate=false)->S.preprocess(_ => {
+    serializer: value =>
+      switch value->(magic: unknown => Nullable.t<unknown>)->Nullable.toOption {
+      | None => "null"->magic
+      | Some(json) => json
+      },
+  })
+
   // ClickHouse expects timestamps as numbers (milliseconds), not ISO strings
   let clickHouseDate =
     S.json(~validate=false)
@@ -761,6 +925,12 @@ let prettifyExn = exn => {
   }
 }
 
+let exnMessage = exn =>
+  switch exn->JsExn.anyToExnInternal {
+  | JsExn(jsExn) => jsExn->JsExn.message
+  | _ => None
+  }
+
 module EnvioPackage = {
   type t = {version: string}