envio 3.15.0-subgraph → 3.16.0-subgraph

package/src/LoadLayer.res.mjs

@@ -1,5 +1,6 @@
 // Generated by ReScript, PLEASE EDIT WITH CARE
 
+import * as Clock from "./Clock.res.mjs";
 import * as Utils from "./Utils.res.mjs";
 import * as Logger from "./Logger.res.mjs";
 import * as ChainId from "./ChainId.res.mjs";
@@ -28,7 +29,8 @@ function scopeKeySuffix(scope) {
 
 function loadById(loadManager, persistence, entityConfig, scope, indexerState, shouldGroup, entityId) {
   let inMemTable = InMemoryStore.getInMemTable(indexerState, entityConfig, scope);
-  let change = inMemTable.latestEntityChangeById[entityId];
+  let entityIdToKey = inMemTable.entityIdToKey;
+  let change = inMemTable.latestEntityChangeById[entityIdToKey(entityId)];
   if (change !== undefined) {
     return Promise.resolve(InMemoryTable.Entity.mapChangeToEntity(change));
   }
@@ -51,13 +53,13 @@ function loadById(loadManager, persistence, entityConfig, scope, indexerState, s
     let entitiesMap = {};
     for (let idx = 0, idx_finish = dbEntities.length; idx < idx_finish; ++idx) {
       let entity = dbEntities[idx];
-      entitiesMap[entity.id] = entity;
+      entitiesMap[entityIdToKey(entity.id)] = entity;
     }
     let committedCheckpointId = IndexerState.committedCheckpointIdFor(indexerState, scope);
-    idsToLoad.forEach(entityId => InMemoryTable.Entity.initValue(inMemTable, committedCheckpointId, entityId, entitiesMap[entityId]));
+    idsToLoad.forEach(entityId => InMemoryTable.Entity.initValue(inMemTable, committedCheckpointId, entityId, entitiesMap[entityIdToKey(entityId)]));
     return IndexerState.endStorageLoad(indexerState, timerRef, storageName, key, idsToLoad.length, dbEntities.length);
   };
-  return LoadManager.call(loadManager, entityId, key, load, LoadManager.noopHasher, shouldGroup, hash => hash in inMemTable.latestEntityChangeById, InMemoryTable.Entity.getUnsafe(inMemTable));
+  return LoadManager.call(loadManager, entityId, key, load, entityIdToKey, shouldGroup, hash => hash in inMemTable.latestEntityChangeById, InMemoryTable.Entity.getUnsafe(inMemTable));
 }
 
 function callEffect(effect, arg, inMemTable, timerRef, onError) {
@@ -70,7 +72,7 @@ function executeWithRateLimit(effect, effectArgs, inMemTable, onError, isFromQue
   let promises = [];
   let state = inMemTable.rateLimitState;
   if (state !== undefined) {
-    let now = Date.now();
+    let now = state.clock.now();
     if (now >= state.windowStartTime + state.durationMs) {
       state.availableCalls = state.callsPerDuration;
       state.windowStartTime = now;
@@ -99,7 +101,7 @@ function executeWithRateLimit(effect, effectArgs, inMemTable, onError, isFromQue
         nextWindowPromise = p;
       } else {
         millisUntilReset.contents = state.windowStartTime + state.durationMs - now | 0;
-        let p$1 = Utils.delay(millisUntilReset.contents);
+        let p$1 = Clock.delay(state.clock, "effect rate limit window", millisUntilReset.contents);
         state.nextWindowPromise = p$1;
         nextWindowPromise = p$1;
       }
@@ -195,8 +197,9 @@ function indexFilter(inMemTable, filter, table) {
 
 function getByIdInMemory(entityConfig, scope, indexerState, entityId) {
   let inMemTable = InMemoryStore.getInMemTable(indexerState, entityConfig, scope);
-  if (entityId in inMemTable.latestEntityChangeById) {
-    return Primitive_option.some(InMemoryTable.Entity.getUnsafe(inMemTable)(entityId));
+  let entityKey = inMemTable.entityIdToKey(entityId);
+  if (entityKey in inMemTable.latestEntityChangeById) {
+    return Primitive_option.some(InMemoryTable.Entity.getUnsafe(inMemTable)(entityKey));
   }
 }
 

package/src/Main.res.mjs

@@ -414,7 +414,7 @@ async function startIndexer(config, logger, persistence, resetOpt, isTestOpt, ex
     Metrics.startRuntimeCollectors();
     Server.startServer(logger, getMetrics, envioVersion, () => Persistence.getInitializedStorageOrThrow(persistence$1).dumpEffectCache().then(prim => {}), Metrics.collectRuntime, isDevelopmentMode);
   }
-  let state = IndexerState.makeFromDbState(config$2, logger, persistence$1, Persistence.getInitializedState(persistence$1), registrationsByChainId, isDevelopmentMode, shouldUseTui, exitAfterFirstEventBlock, undefined, undefined, Stdlib_Option.mapOr(Worker.config, false, worker => worker.holdRealtime), onError, undefined);
+  let state = IndexerState.makeFromDbState(config$2, logger, persistence$1, Persistence.getInitializedState(persistence$1), registrationsByChainId, isDevelopmentMode, shouldUseTui, exitAfterFirstEventBlock, undefined, undefined, Stdlib_Option.mapOr(Worker.config, false, worker => worker.holdRealtime), onError, undefined, undefined);
   if (shouldUseTui) {
     Tui.start(config$2, logger, () => IndexerState.toMetrics(state));
   }

package/src/Persistence.res

@@ -117,8 +117,6 @@ type storage = {
   // carries a matching row — storage doesn't interpret it.
   initialize: (
     ~chainConfigs: array<Config.chain>=?,
-    ~entities: array<Internal.entityConfig>=?,
-    ~enums: array<Table.enumConfig<Table.enum>>=?,
     ~contractMapping: ContractMapping.t,
     ~envioInfo: JSON.t,
   ) => promise<initialState>,
@@ -129,16 +127,11 @@ type storage = {
   // Brings an initialized storage a chain it doesn't have yet: its row, its
   // partitions and its config addresses, as `initialize` would have created
   // them. The chains already there are left untouched.
-  addChain: (
-    ~chainConfig: Config.chain,
-    ~entities: array<Internal.entityConfig>,
-    ~contractMapping: ContractMapping.t,
-  ) => promise<unit>,
+  addChain: (~chainConfig: Config.chain, ~contractMapping: ContractMapping.t) => promise<unit>,
   // `chainIds` is what this run drives. An isolated run resumes a subset of
   // the stored chains, and ClickHouse's resume trims past each resumed chain's
   // checkpoint, so chains a sibling process is still writing must stay out.
   resumeInitialState: (
-    ~entities: array<Internal.entityConfig>,
     ~chainIds: array<ChainId.t>,
     ~contractMapping: ContractMapping.t,
   ) => promise<initialState>,
@@ -159,19 +152,12 @@ type storage = {
   // never runs `finalizeBackfill`: an index dropped or invalidated while it was
   // down would otherwise never be rebuilt. Best-effort, and safe to run with
   // indexing live.
-  ensureSchemaIndexes: (
-    ~entities: array<Internal.entityConfig>,
-    ~chainIds: array<ChainId.t>,
-  ) => promise<unit>,
+  ensureSchemaIndexes: (~chainIds: array<ChainId.t>) => promise<unit>,
   // Creates every index the schema promises for the given chains, then stamps
   // `ready_at` on them. Called once those chains finish backfill. The indexes
   // are committed one at a time so a failure part way through doesn't undo the
   // ones already built; `ready_at` is only written once they all verify.
-  finalizeBackfill: (
-    ~entities: array<Internal.entityConfig>,
-    ~chainIds: array<ChainId.t>,
-    ~readyAt: Date.t,
-  ) => promise<unit>,
+  finalizeBackfill: (~chainIds: array<ChainId.t>, ~readyAt: Date.t) => promise<unit>,
   // This is to download cache from the database to .envio/cache
   dumpEffectCache: unit => promise<unit>,
   reset: unit => promise<unit>,
@@ -190,7 +176,6 @@ type storage = {
   getRollbackTargetCheckpoint: (
     ~reorgChainId: ChainId.t,
     ~lastKnownValidBlockNumber: int,
-    ~sequence: CheckpointSequence.t,
   ) => promise<option<Internal.checkpointId>>,
   // Get rollback progress diff
   getRollbackProgressDiff: (
@@ -214,7 +199,6 @@ type storage = {
     ~batch: Batch.t,
     ~rollback: option<rollback>,
     ~config: Config.t,
-    ~allEntities: array<Internal.entityConfig>,
     ~updatedEffectsCache: array<updatedEffectCache>,
     ~updatedEntities: array<updatedEntity>,
     // Addresses this batch registered, with the checkpoint that covers them.
@@ -339,8 +323,6 @@ let init = {
             throw(Refused(Config.setUpMeanwhileMessage))
           }
           let initialState = await persistence.storage.initialize(
-            ~entities=persistence.allEntities,
-            ~enums=persistence.allEnums,
             ~chainConfigs,
             ~contractMapping,
             ~envioInfo,
@@ -397,13 +379,11 @@ let init = {
               )->Array.getUnsafe(0)
             await persistence.storage.addChain(
               ~chainConfig,
-              ~entities=persistence.allEntities,
               ~contractMapping=stored.contractMapping,
             )
           }
           if resume {
             let initialState = await persistence.storage.resumeInitialState(
-              ~entities=persistence.allEntities,
               ~chainIds=chainConfigs->Array.map(chain => chain.id),
               ~contractMapping=stored.contractMapping,
             )

package/src/Persistence.res.mjs

@@ -94,7 +94,7 @@ async function init(persistence, logger, chainConfigs, contractMapping, envioInf
           Error: new Error()
         };
       }
-      let initialState = await persistence.storage.initialize(chainConfigs$1, persistence.allEntities, persistence.allEnums, contractMapping, envioInfo);
+      let initialState = await persistence.storage.initialize(chainConfigs$1, contractMapping, envioInfo);
       Logger.info(logger, `The indexer storage is ready. Starting indexing!`, undefined);
       persistence.storageStatus = {
         TAG: "Ready",
@@ -136,11 +136,11 @@ async function init(persistence, logger, chainConfigs, contractMapping, envioInf
               chainId: chainConfig.id
             });
             let chainConfig$1 = (await StartBlockResolver.resolveAllOrThrow([chainConfig], logger, lowercaseAddresses, startBlockRetry, sourceTiming))[0];
-            await persistence.storage.addChain(chainConfig$1, persistence.allEntities, stored.contractMapping);
+            await persistence.storage.addChain(chainConfig$1, stored.contractMapping);
           }
         }
         if (resume) {
-          let initialState$1 = await persistence.storage.resumeInitialState(persistence.allEntities, chainConfigs.map(chain => chain.id), stored.contractMapping);
+          let initialState$1 = await persistence.storage.resumeInitialState(chainConfigs.map(chain => chain.id), stored.contractMapping);
           persistence.storageStatus = {
             TAG: "Ready",
             _0: initialState$1

package/src/PreloadLookahead.res

@@ -26,12 +26,12 @@
 type t = {
   items: array<Internal.item>,
   indexOf: Utils.Map.t<Internal.item, int>,
-  absentReaders: dict<array<int>>,
+  absentReaders: Map.t<string, array<int>>,
   dirty: array<bool>,
   // Milliseconds the item's preload rounds have run for, all runs together.
   preloadMs: array<float>,
   // By entity key, the writes items' preloads made: (item, entity or deleted).
-  overlay: dict<array<(int, option<Internal.entity>)>>,
+  overlay: Map.t<string, array<(int, option<Internal.entity>)>>,
   window: int,
   // The last item execute finished; reads and writes at or before it are history.
   mutable executed: int,
@@ -51,10 +51,10 @@ let start = (~items: array<Internal.item>) => {
       Some({
         items,
         indexOf,
-        absentReaders: Dict.make(),
+        absentReaders: Map.make(),
         dirty: Array.make(~length=items->Array.length, false),
         preloadMs: Array.make(~length=items->Array.length, 0.),
-        overlay: Dict.make(),
+        overlay: Map.make(),
         window,
         executed: -1,
       })
@@ -102,9 +102,9 @@ let recordAbsent = (~item: Internal.item, ~entityName, ~scope, ~entityId) =>
     switch t.indexOf->Utils.Map.get(item) {
     | Some(idx) if idx > t.executed =>
       let key = entityKey(~entityName, ~scope, ~entityId)
-      switch t.absentReaders->Utils.Dict.dangerouslyGetNonOption(key) {
+      switch t.absentReaders->Map.get(key) {
       | Some(readers) => readers->Array.push(idx)
-      | None => t.absentReaders->Dict.set(key, [idx])
+      | None => t.absentReaders->Map.set(key, [idx])
       }
     | _ => ()
     }
@@ -113,22 +113,42 @@ let recordAbsent = (~item: Internal.item, ~entityName, ~scope, ~entityId) =>
 let onSet = (~entityName, ~scope, ~entityId) =>
   switch active.contents {
   | None => ()
-  | Some(t) =>
+  // Every write execute makes lands here, and most batches have no absent read.
+  | Some(t) if t.absentReaders->Map.size > 0 =>
     let key = entityKey(~entityName, ~scope, ~entityId)
-    switch t.absentReaders->Utils.Dict.dangerouslyGetNonOption(key) {
+    switch t.absentReaders->Map.get(key) {
     | None => ()
     | Some(readers) =>
-      t.absentReaders->Utils.Dict.deleteInPlace(key)
+      t.absentReaders->Map.delete(key)->ignore
       readers->Array.forEach(idx =>
         if idx > t.executed {
           t.dirty->Array.setUnsafe(idx, true)
         }
       )
     }
+  | Some(_) => ()
   }
 
 let isActive = () => active.contents->Option.isSome
 
+// How many of `writes` (sorted by item) are by an item before `item`. An entity
+// every event of a batch touches (a popular holder) gathers a write per event,
+// and a scan of them per read was quadratic in the batch.
+let countWritesBeforeItem = (writes: array<(int, option<Internal.entity>)>, item) => {
+  let low = ref(0)
+  let high = ref(writes->Array.length)
+  while low.contents < high.contents {
+    let mid = (low.contents + high.contents) / 2
+    let (writer, _) = writes->Array.getUnsafe(mid)
+    if writer < item {
+      low := mid + 1
+    } else {
+      high := mid
+    }
+  }
+  low.contents
+}
+
 let overlaySet = (~item: Internal.item, ~entityName, ~scope, ~entityId, ~entity) =>
   switch active.contents {
   | None => ()
@@ -136,12 +156,15 @@ let overlaySet = (~item: Internal.item, ~entityName, ~scope, ~entityId, ~entity)
     switch t.indexOf->Utils.Map.get(item) {
     | Some(writer) =>
       let key = entityKey(~entityName, ~scope, ~entityId)
-      let writes = switch t.overlay->Utils.Dict.dangerouslyGetNonOption(key) {
-      | Some(writes) => writes->Array.filter(((idx, _)) => idx !== writer)
-      | None => []
+      switch t.overlay->Map.get(key) {
+      | Some(writes) =>
+        let at = writes->countWritesBeforeItem(writer)
+        switch writes->Array.get(at) {
+        | Some((idx, _)) if idx === writer => writes->Array.setUnsafe(at, (writer, entity))
+        | _ => writes->Array.splice(~start=at, ~remove=0, ~insert=[(writer, entity)])
+        }
+      | None => t.overlay->Map.set(key, [(writer, entity)])
       }
-      writes->Array.push((writer, entity))
-      t.overlay->Dict.set(key, writes)
     | None => ()
     }
   }
@@ -154,18 +177,13 @@ let overlayGet = (~item: Internal.item, ~entityName, ~scope, ~entityId) =>
   | Some(t) =>
     switch (
       t.indexOf->Utils.Map.get(item),
-      t.overlay->Utils.Dict.dangerouslyGetNonOption(entityKey(~entityName, ~scope, ~entityId)),
+      t.overlay->Map.get(entityKey(~entityName, ~scope, ~entityId)),
     ) {
     | (Some(reader), Some(writes)) =>
-      let found = ref(None)
-      let foundAt = ref(t.executed)
-      writes->Array.forEach(((writer, entity)) =>
-        if writer > foundAt.contents && writer < reader {
-          foundAt := writer
-          found := Some(entity)
-        }
-      )
-      found.contents
+      switch writes->Array.get(writes->countWritesBeforeItem(reader) - 1) {
+      | Some((writer, entity)) if writer > t.executed => Some(entity)
+      | _ => None
+      }
     | _ => None
     }
   }

package/src/PreloadLookahead.res.mjs

@@ -1,7 +1,6 @@
 // Generated by ReScript, PLEASE EDIT WITH CARE
 
 import * as Env from "./Env.res.mjs";
-import * as Utils from "./Utils.res.mjs";
 import * as ChainId from "./ChainId.res.mjs";
 import * as Performance from "./bindings/Performance.res.mjs";
 import * as Stdlib_Array from "@rescript/runtime/lib/es6/Stdlib_Array.js";
@@ -24,10 +23,10 @@ function start(items) {
   active.contents = {
     items: items,
     indexOf: indexOf,
-    absentReaders: {},
+    absentReaders: new Map(),
     dirty: Stdlib_Array.make(items.length, false),
     preloadMs: Stdlib_Array.make(items.length, 0),
-    overlay: {},
+    overlay: new Map(),
     window: Env.subgraphPreloadLookahead,
     executed: -1
   };
@@ -86,11 +85,11 @@ function recordAbsent(item, entityName, scope, entityId) {
     return;
   }
   let key = entityKey(entityName, scope, entityId);
-  let readers = t.absentReaders[key];
+  let readers = t.absentReaders.get(key);
   if (readers !== undefined) {
     readers.push(idx);
   } else {
-    t.absentReaders[key] = [idx];
+    t.absentReaders.set(key, [idx]);
   }
 }
 
@@ -99,10 +98,13 @@ function onSet(entityName, scope, entityId) {
   if (t === undefined) {
     return;
   }
+  if (t.absentReaders.size <= 0) {
+    return;
+  }
   let key = entityKey(entityName, scope, entityId);
-  let readers = t.absentReaders[key];
+  let readers = t.absentReaders.get(key);
   if (readers !== undefined) {
-    Utils.Dict.deleteInPlace(t.absentReaders, key);
+    t.absentReaders.delete(key);
     readers.forEach(idx => {
       if (idx > t.executed) {
         t.dirty[idx] = true;
@@ -117,6 +119,21 @@ function isActive() {
   return Stdlib_Option.isSome(active.contents);
 }
 
+function countWritesBeforeItem(writes, item) {
+  let low = 0;
+  let high = writes.length;
+  while (low < high) {
+    let mid = (low + high | 0) / 2 | 0;
+    let match = writes[mid];
+    if (match[0] < item) {
+      low = mid + 1 | 0;
+    } else {
+      high = mid;
+    }
+  };
+  return low;
+}
+
 function overlaySet(item, entityName, scope, entityId, entity) {
   let t = active.contents;
   if (t === undefined) {
@@ -127,13 +144,27 @@ function overlaySet(item, entityName, scope, entityId, entity) {
     return;
   }
   let key = entityKey(entityName, scope, entityId);
-  let writes = t.overlay[key];
-  let writes$1 = writes !== undefined ? writes.filter(param => param[0] !== writer) : [];
-  writes$1.push([
-    writer,
-    entity
-  ]);
-  t.overlay[key] = writes$1;
+  let writes = t.overlay.get(key);
+  if (writes !== undefined) {
+    let at = countWritesBeforeItem(writes, writer);
+    let match = writes[at];
+    if (match !== undefined && match[0] === writer) {
+      writes[at] = [
+        writer,
+        entity
+      ];
+      return;
+    }
+    writes.splice(at, 0, [
+      writer,
+      entity
+    ]);
+    return;
+  }
+  t.overlay.set(key, [[
+      writer,
+      entity
+    ]]);
 }
 
 function overlayGet(item, entityName, scope, entityId) {
@@ -142,28 +173,17 @@ function overlayGet(item, entityName, scope, entityId) {
     return;
   }
   let match = t.indexOf.get(item);
-  let match$1 = t.overlay[entityKey(entityName, scope, entityId)];
+  let match$1 = t.overlay.get(entityKey(entityName, scope, entityId));
   if (match === undefined) {
     return;
   }
   if (match$1 === undefined) {
     return;
   }
-  let found = {
-    contents: undefined
-  };
-  let foundAt = {
-    contents: t.executed
-  };
-  match$1.forEach(param => {
-    let writer = param[0];
-    if (writer > foundAt.contents && writer < match) {
-      foundAt.contents = writer;
-      found.contents = Primitive_option.some(param[1]);
-      return;
-    }
-  });
-  return found.contents;
+  let match$2 = match$1[countWritesBeforeItem(match$1, match) - 1 | 0];
+  if (match$2 !== undefined && match$2[0] > t.executed) {
+    return Primitive_option.some(match$2[1]);
+  }
 }
 
 function afterItem(t, index, preload) {
@@ -190,6 +210,7 @@ export {
   recordAbsent,
   onSet,
   isActive,
+  countWritesBeforeItem,
   overlaySet,
   overlayGet,
   afterItem,

package/src/Rollback.res

@@ -130,7 +130,6 @@ and executeRollback = async (
     switch await (state->IndexerState.persistence).storage.getRollbackTargetCheckpoint(
       ~reorgChainId=reorgChain,
       ~lastKnownValidBlockNumber=rollbackTargetBlockNumber,
-      ~sequence=(state->IndexerState.config).checkpointSequence,
     ) {
     | Some(checkpointId) => checkpointId
     | None => 0n

package/src/Rollback.res.mjs

@@ -42,7 +42,7 @@ async function executeRollback(state, reorgChain, rollbackTargetBlockNumber, sch
   Logger.info(logger, "Started rollback on reorg", undefined);
   ChainState.setRollbackTargetBlock(IndexerState.getChainState(state, reorgChain), rollbackTargetBlockNumber);
   await Writing.flush(state);
-  let checkpointId = await IndexerState.persistence(state).storage.getRollbackTargetCheckpoint(reorgChain, rollbackTargetBlockNumber, IndexerState.config(state).checkpointSequence);
+  let checkpointId = await IndexerState.persistence(state).storage.getRollbackTargetCheckpoint(reorgChain, rollbackTargetBlockNumber);
   let rollbackTargetCheckpointId = checkpointId !== undefined ? checkpointId : 0n;
   let config = IndexerState.config(state);
   let next = RollbackFloors.make(config.checkpointSequence, ChainMap.keys(config.chainMap), reorgChain, rollbackTargetCheckpointId, rollbackTargetBlockNumber);

package/src/SafeCheckpointTracking.res

@@ -16,9 +16,7 @@ let make = (
 ) => {
   if maxReorgDepth > 0 && shouldRollbackOnReorg {
     let checkpointIds = Utils.Array.jsArrayCreate(chainReorgCheckpoints->Array.length)
-    let checkpointBlockNumbers = Utils.Array.jsArrayCreate(
-      chainReorgCheckpoints->Array.length,
-    )
+    let checkpointBlockNumbers = Utils.Array.jsArrayCreate(chainReorgCheckpoints->Array.length)
     chainReorgCheckpoints->Array.forEachWithIndex((checkpoint, idx) => {
       checkpointIds->Array.setUnsafe(idx, checkpoint.checkpointId)
       checkpointBlockNumbers->Array.setUnsafe(idx, checkpoint.blockNumber)
@@ -36,7 +34,9 @@ let make = (
 let getSafeCheckpointId = (safeCheckpointTracking: t, ~sourceBlockNumber: int): option<
   Internal.checkpointId,
 > => {
-  let safeBlockNumber = sourceBlockNumber - safeCheckpointTracking.maxReorgDepth
+  // The first block reorg detection compares is itself reorgable, so the safe
+  // block is the one below it.
+  let safeBlockNumber = sourceBlockNumber - safeCheckpointTracking.maxReorgDepth - 1
 
   switch safeCheckpointTracking.checkpointIds {
   | [] => None

package/src/SafeCheckpointTracking.res.mjs

@@ -20,7 +20,7 @@ function make(maxReorgDepth, shouldRollbackOnReorg, chainReorgCheckpoints) {
 }
 
 function getSafeCheckpointId(safeCheckpointTracking, sourceBlockNumber) {
-  let safeBlockNumber = sourceBlockNumber - safeCheckpointTracking.maxReorgDepth | 0;
+  let safeBlockNumber = (sourceBlockNumber - safeCheckpointTracking.maxReorgDepth | 0) - 1 | 0;
   let match = safeCheckpointTracking.checkpointIds;
   if (match.length === 0) {
     return;

package/src/StagedRows.res

@@ -0,0 +1,106 @@
+// Values as the addon declares each staged field has to arrive, compiled once
+// per layout and written into a stage.
+
+@send external getTime: unknown => float = "getTime"
+@send external toFixed: unknown => string = "toFixed"
+@val @scope("Array") external isArray: unknown => bool = "isArray"
+
+// A list element as its JSON: the encoding a scalar of its type would take,
+// except that a boolean stays one and bytes become their values.
+%%private(
+  let elementOf = (encoding: Staging.encoding, ~column): (unknown => unknown) =>
+    switch encoding {
+    | Number =>
+      value =>
+        value->Staging.toNumber->Staging.finiteOrThrow(~column)->(Utils.magic: float => unknown)
+    | Date => value => value->getTime->(Utils.magic: float => unknown)
+    | Digits => value => value->Staging.stringOf->(Utils.magic: string => unknown)
+    | Decimal => value => value->toFixed->(Utils.magic: string => unknown)
+    | Document => value => value->Sql.canonicalDocument->(Utils.magic: string => unknown)
+    | Bytes =>
+      value =>
+        value
+        ->(Utils.magic: unknown => Array.arrayLike<int>)
+        ->Array.fromArrayLike
+        ->(Utils.magic: array<int> => unknown)
+    | Bool | Text | List => value => value
+    }
+)
+
+%%private(
+  let rec mapList = (value: unknown, element: unknown => unknown): unknown =>
+    if value->isArray {
+      value
+      ->(Utils.magic: unknown => array<unknown>)
+      ->Array.map(item => item->mapList(element))
+      ->(Utils.magic: array<unknown> => unknown)
+    } else if value === %raw(`undefined`) || value === %raw(`null`) {
+      %raw(`null`)
+    } else {
+      element(value)
+    }
+)
+
+// What a present value becomes before it is written to its slot.
+%%private(
+  let encoderOf = ({name, encoding, element}: Staging.stagedField): (unknown => unknown) =>
+    switch encoding {
+    | Bool =>
+      value => (value->(Utils.magic: unknown => bool) ? 1. : 0.)->(Utils.magic: float => unknown)
+    | Date => value => value->getTime->(Utils.magic: float => unknown)
+    | Decimal => value => value->toFixed->(Utils.magic: string => unknown)
+    | Document => value => value->Sql.canonicalDocument->(Utils.magic: string => unknown)
+    | List =>
+      let encode = elementOf(element, ~column=name)
+      // A document can be a list itself, so a list of them is one level deep.
+      let mapList = switch element {
+      | Document =>
+        (value, encode) =>
+          value
+          ->(Utils.magic: unknown => array<unknown>)
+          ->Array.map(item => item->Staging.isAbsent ? %raw(`null`) : encode(item))
+          ->(Utils.magic: array<unknown> => unknown)
+      | _ => mapList
+      }
+      value =>
+        value
+        ->mapList(encode)
+        ->(Utils.magic: unknown => JSON.t)
+        ->JSON.stringify
+        ->Utils.replaceLoneSurrogateEscapes
+        ->(Utils.magic: string => unknown)
+    | Number | Text | Digits | Bytes => value => value
+    }
+)
+
+// A layout's fields compiled once: the slot each takes and how a value of it
+// is written there.
+type writer = {field: Staging.stagedField, encode: unknown => unknown}
+
+let writersOf = (fields: array<Staging.stagedField>) =>
+  fields->Array.map(field => {field, encode: encoderOf(field)})
+
+let columnsOf = (writers: array<writer>): array<Staging.column> =>
+  writers->Array.map(({field}) => {
+    Staging.name: field.name,
+    kind: field.encoding->Staging.kindOf,
+    isNullable: field.isNullable,
+  })
+
+// JSON `null` is a document of its own, which a column that can't be NULL
+// stores as one. Left out, the value is no document and is refused like any
+// other.
+let writeField = (stage, ~writers: array<writer>, ~column, ~row, value: unknown) => {
+  let {field, encode} = writers->Array.getUnsafe(column)
+  if value->Staging.isAbsent {
+    stage->Staging.writeValue(
+      ~column,
+      ~row,
+      field.encoding === Staging.Document && !field.isNullable && value === %raw(`null`)
+        ? "null"->(Utils.magic: string => unknown)
+        : value,
+    )
+  } else {
+    stage->Staging.writeValue(~column, ~row, encode(value))
+  }
+}