envio 3.1.0-rc.1 → 3.1.0

package/src/InMemoryStore.res.mjs

@@ -1,13 +1,20 @@
 // Generated by ReScript, PLEASE EDIT WITH CARE
 
+import * as Env from "./Env.res.mjs";
 import * as Utils from "./Utils.res.mjs";
 import * as Config from "./Config.res.mjs";
+import * as Logging from "./Logging.res.mjs";
 import * as Internal from "./Internal.res.mjs";
+import * as Throttler from "./Throttler.res.mjs";
 import * as Prometheus from "./Prometheus.res.mjs";
 import * as Stdlib_Array from "@rescript/runtime/lib/es6/Stdlib_Array.js";
 import * as ErrorHandling from "./ErrorHandling.res.mjs";
 import * as InMemoryTable from "./InMemoryTable.res.mjs";
+import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js";
+import * as RollbackCommit from "./RollbackCommit.res.mjs";
 import * as Stdlib_JsError from "@rescript/runtime/lib/es6/Stdlib_JsError.js";
+import * as Primitive_object from "@rescript/runtime/lib/es6/Primitive_object.js";
+import * as Primitive_option from "@rescript/runtime/lib/es6/Primitive_option.js";
 import * as S$RescriptSchema from "rescript-schema/src/S.res.mjs";
 import * as Primitive_exceptions from "@rescript/runtime/lib/es6/Primitive_exceptions.js";
 
@@ -39,15 +46,30 @@ let EntityTables = {
   get: get
 };
 
-function make$1(entities, committedCheckpointIdOpt) {
+function make$1(entities, committedCheckpointIdOpt, persistence, config, onError) {
   let committedCheckpointId = committedCheckpointIdOpt !== undefined ? committedCheckpointIdOpt : Internal.initialCheckpointId;
+  let intervalMillis = Env.ThrottleWrites.chainMetadataIntervalMillis;
+  let chainMetaThrottler = Throttler.make(intervalMillis, Logging.createChild({
+    context: "Throttler for chain metadata writes",
+    intervalMillis: intervalMillis
+  }));
   return {
     allEntities: entities,
-    rawEvents: [],
     entities: make(entities),
     effects: {},
     rollback: undefined,
-    committedCheckpointId: committedCheckpointId
+    committedCheckpointId: committedCheckpointId,
+    processedCheckpointId: committedCheckpointId,
+    processedBatches: [],
+    writeFiber: undefined,
+    hasFailedWrite: false,
+    onError: onError,
+    commitWaiters: [],
+    persistence: persistence,
+    config: config,
+    chainMeta: {},
+    chainMetaDirty: false,
+    chainMetaThrottler: chainMetaThrottler
   };
 }
 
@@ -61,12 +83,65 @@ function getEffectInMemTable(inMemoryStore, effect) {
     idsToStore: [],
     invalidationsCount: 0,
     dict: {},
+    changesCount: 0,
     effect: effect
   };
   inMemoryStore.effects[key] = table$1;
   return table$1;
 }
 
+function getEffectOutput(inMemTable, key) {
+  let match = inMemTable.dict[key];
+  if (match !== undefined && match.type === "SET") {
+    return Primitive_option.some(match.entity);
+  }
+}
+
+function setEffectOutput(inMemTable, checkpointId, cacheKey, output, shouldCache) {
+  let match = inMemTable.dict[cacheKey];
+  if (match !== undefined) {
+    
+  } else {
+    inMemTable.changesCount = inMemTable.changesCount + 1;
+  }
+  inMemTable.dict[cacheKey] = {
+    type: "SET",
+    entityId: cacheKey,
+    entity: output,
+    checkpointId: checkpointId
+  };
+  if (shouldCache) {
+    inMemTable.idsToStore.push(cacheKey);
+    return;
+  }
+}
+
+function initEffectOutputFromDb(inMemTable, cacheKey, output) {
+  if (Stdlib_Option.isNone(inMemTable.dict[cacheKey])) {
+    inMemTable.changesCount = inMemTable.changesCount + 1;
+    inMemTable.dict[cacheKey] = {
+      type: "SET",
+      entityId: cacheKey,
+      entity: output,
+      checkpointId: Internal.loadedFromDbCheckpointId
+    };
+    return;
+  }
+}
+
+function dropCommittedEffects(inMemTable, committedCheckpointId, keepLoadedFromDb) {
+  let keysToDelete = [];
+  Utils.Dict.forEachWithKey(inMemTable.dict, (change, key) => {
+    let checkpointId = change.checkpointId;
+    if (checkpointId <= committedCheckpointId && !(keepLoadedFromDb && checkpointId === Internal.loadedFromDbCheckpointId)) {
+      keysToDelete.push(key);
+      return;
+    }
+  });
+  keysToDelete.forEach(key => Utils.Dict.deleteInPlace(inMemTable.dict, key));
+  inMemTable.changesCount = inMemTable.changesCount - keysToDelete.length;
+}
+
 function getInMemTable(inMemoryStore, entityConfig) {
   return get(inMemoryStore.entities, entityConfig.name);
 }
@@ -75,33 +150,145 @@ function isRollingBack(inMemoryStore) {
   return inMemoryStore.rollback !== undefined;
 }
 
-async function writeBatch(inMemoryStore, persistence, batch, config, isInReorgThreshold) {
-  let match = persistence.storageStatus;
-  if (typeof match !== "object") {
-    return Stdlib_JsError.throwWithMessage(`Failed to access the indexer storage. The Persistence layer is not initialized.`);
-  }
-  if (match.TAG === "Initializing") {
-    return Stdlib_JsError.throwWithMessage(`Failed to access the indexer storage. The Persistence layer is not initialized.`);
-  }
-  let cache = match._0.cache;
-  let committedCheckpointId = inMemoryStore.committedCheckpointId;
-  let totalChanges = {
+function getChangesCount(inMemoryStore) {
+  let total = {
+    contents: 0
+  };
+  inMemoryStore.allEntities.forEach(entityConfig => {
+    total.contents = total.contents + getInMemTable(inMemoryStore, entityConfig).changesCount;
+  });
+  Utils.Dict.forEach(inMemoryStore.effects, inMemTable => {
+    total.contents = total.contents + inMemTable.changesCount;
+  });
+  inMemoryStore.processedBatches.forEach(batch => {
+    total.contents = total.contents + batch.totalBatchSize;
+  });
+  return total.contents;
+}
+
+function wakeCommitWaiters(inMemoryStore) {
+  let waiters = inMemoryStore.commitWaiters;
+  inMemoryStore.commitWaiters = [];
+  waiters.forEach(resolve => resolve());
+}
+
+function waitForCommit(inMemoryStore) {
+  return new Promise((resolve, param) => {
+    inMemoryStore.commitWaiters.push(resolve);
+  });
+}
+
+function drainBatchRun(inMemoryStore) {
+  let all = inMemoryStore.processedBatches;
+  let isInReorgThreshold = all[0].isInReorgThreshold;
+  let rest = [];
+  let progressedChainsById = {};
+  let totalBatchSize = {
     contents: 0
   };
-  persistence.allEntities.forEach(entityConfig => {
-    totalChanges.contents = totalChanges.contents + getInMemTable(inMemoryStore, entityConfig).changesCount;
+  let items = [];
+  let checkpointIds = [];
+  let checkpointChainIds = [];
+  let checkpointBlockNumbers = [];
+  let checkpointBlockHashes = [];
+  let checkpointEventsProcessed = [];
+  all.forEach(batch => {
+    if (Utils.$$Array.isEmpty(rest) && batch.isInReorgThreshold === isInReorgThreshold) {
+      Utils.Dict.forEachWithKey(batch.progressedChainsById, (chainAfterBatch, key) => {
+        progressedChainsById[key] = chainAfterBatch;
+      });
+      totalBatchSize.contents = totalBatchSize.contents + batch.totalBatchSize | 0;
+      items.push(...batch.items);
+      checkpointIds.push(...batch.checkpointIds);
+      checkpointChainIds.push(...batch.checkpointChainIds);
+      checkpointBlockNumbers.push(...batch.checkpointBlockNumbers);
+      checkpointBlockHashes.push(...batch.checkpointBlockHashes);
+      checkpointEventsProcessed.push(...batch.checkpointEventsProcessed);
+    } else {
+      rest.push(batch);
+    }
+  });
+  inMemoryStore.processedBatches = rest;
+  return {
+    totalBatchSize: totalBatchSize.contents,
+    items: items,
+    progressedChainsById: progressedChainsById,
+    isInReorgThreshold: isInReorgThreshold,
+    checkpointIds: checkpointIds,
+    checkpointChainIds: checkpointChainIds,
+    checkpointBlockNumbers: checkpointBlockNumbers,
+    checkpointBlockHashes: checkpointBlockHashes,
+    checkpointEventsProcessed: checkpointEventsProcessed
+  };
+}
+
+function snapshotEffects(inMemoryStore, cache) {
+  let acc = [];
+  Utils.Dict.forEach(inMemoryStore.effects, inMemTable => {
+    let idsToStore = inMemTable.idsToStore;
+    let invalidationsCount = inMemTable.invalidationsCount;
+    let dict = inMemTable.dict;
+    let effect = inMemTable.effect;
+    if (idsToStore.length !== 0) {
+      let items = Stdlib_Array.filterMap(idsToStore, id => {
+        let match = dict[id];
+        if (match.type === "SET") {
+          return {
+            id: id,
+            output: match.entity
+          };
+        }
+      });
+      let effectName = effect.name;
+      let c = cache[effectName];
+      let effectCacheRecord;
+      if (c !== undefined) {
+        effectCacheRecord = c;
+      } else {
+        let c$1 = {
+          effectName: effectName,
+          count: 0
+        };
+        cache[effectName] = c$1;
+        effectCacheRecord = c$1;
+      }
+      let shouldInitialize = effectCacheRecord.count === 0;
+      effectCacheRecord.count = (effectCacheRecord.count + items.length | 0) - invalidationsCount | 0;
+      Prometheus.EffectCacheCount.set(effectCacheRecord.count, effectName);
+      acc.push({
+        effect: effect,
+        items: items,
+        shouldInitialize: shouldInitialize
+      });
+    }
+    inMemTable.idsToStore = [];
+    inMemTable.invalidationsCount = 0;
   });
-  let keepLatestChanges = totalChanges.contents < 50000;
+  return acc;
+}
+
+async function runOneWrite(inMemoryStore, persistence, config) {
+  let match = persistence.storageStatus;
+  let cache;
+  cache = typeof match !== "object" || match.TAG === "Initializing" ? Stdlib_JsError.throwWithMessage(`Failed to access the indexer storage. The Persistence layer is not initialized.`) : match._0.cache;
+  let chainMetaData = inMemoryStore.chainMetaDirty ? (inMemoryStore.chainMetaDirty = false, Utils.Dict.shallowCopy(inMemoryStore.chainMeta)) : undefined;
+  let match$1 = inMemoryStore.processedBatches;
+  if (match$1.length === 0) {
+    if (chainMetaData !== undefined) {
+      return await persistence.storage.setChainMeta(chainMetaData);
+    } else {
+      return;
+    }
+  }
+  let committedCheckpointId = inMemoryStore.committedCheckpointId;
+  let batch = drainBatchRun(inMemoryStore);
+  let checkpointId = Utils.$$Array.last(batch.checkpointIds);
+  let upToCheckpointId = checkpointId !== undefined ? checkpointId : committedCheckpointId;
+  let rollback = inMemoryStore.rollback;
+  inMemoryStore.rollback = undefined;
   let updatedEntities = Stdlib_Array.filterMap(persistence.allEntities, entityConfig => {
     let table = getInMemTable(inMemoryStore, entityConfig);
-    table.changesCount = table.changesCount - table.prevEntityChanges.length;
-    let changes = table.prevEntityChanges;
-    Utils.Dict.forEach(table.latestEntityChangeById, change => {
-      if (change.checkpointId > committedCheckpointId) {
-        changes.push(change);
-        return;
-      }
-    });
+    let changes = InMemoryTable.Entity.snapshotChanges(table, committedCheckpointId, upToCheckpointId);
     if (Utils.$$Array.isEmpty(changes)) {
       return;
     } else {
@@ -111,74 +298,120 @@ async function writeBatch(inMemoryStore, persistence, batch, config, isInReorgTh
       };
     }
   });
-  let acc = [];
-  await persistence.storage.writeBatch(batch, inMemoryStore.rawEvents, inMemoryStore.rollback, isInReorgThreshold, config, persistence.allEntities, (Utils.Dict.forEach(inMemoryStore.effects, inMemTable => {
-    let idsToStore = inMemTable.idsToStore;
-    let invalidationsCount = inMemTable.invalidationsCount;
-    let effect = inMemTable.effect;
-    let dict = inMemTable.dict;
-    if (idsToStore.length === 0) {
-      return;
+  let updatedEffectsCache = snapshotEffects(inMemoryStore, cache);
+  await persistence.storage.writeBatch(batch, rollback, batch.isInReorgThreshold, config, persistence.allEntities, updatedEffectsCache, updatedEntities, chainMetaData);
+  inMemoryStore.committedCheckpointId = upToCheckpointId;
+  if (rollback !== undefined && Utils.$$Array.notEmpty(RollbackCommit.callbacks)) {
+    return await RollbackCommit.fire(rollback.progressBlockNumberByChainId);
+  }
+}
+
+function hasPendingWrite(inMemoryStore) {
+  if (Utils.$$Array.notEmpty(inMemoryStore.processedBatches)) {
+    return true;
+  } else {
+    return inMemoryStore.chainMetaDirty;
+  }
+}
+
+async function runWriteLoop(inMemoryStore) {
+  while (hasPendingWrite(inMemoryStore) && !inMemoryStore.hasFailedWrite) {
+    try {
+      await runOneWrite(inMemoryStore, inMemoryStore.persistence, inMemoryStore.config);
+      wakeCommitWaiters(inMemoryStore);
+    } catch (raw_exn) {
+      let exn = Primitive_exceptions.internalToException(raw_exn);
+      inMemoryStore.hasFailedWrite = true;
+      inMemoryStore.onError(exn);
     }
-    let items = idsToStore.map(id => ({
-      id: id,
-      output: dict[id]
-    }));
-    let effectName = effect.name;
-    let c = cache[effectName];
-    let effectCacheRecord;
-    if (c !== undefined) {
-      effectCacheRecord = c;
-    } else {
-      let c$1 = {
-        effectName: effectName,
-        count: 0
-      };
-      cache[effectName] = c$1;
-      effectCacheRecord = c$1;
+  };
+  inMemoryStore.writeFiber = undefined;
+  return wakeCommitWaiters(inMemoryStore);
+}
+
+function kick(inMemoryStore) {
+  if (Stdlib_Option.isNone(inMemoryStore.writeFiber) && !inMemoryStore.hasFailedWrite && hasPendingWrite(inMemoryStore)) {
+    inMemoryStore.writeFiber = runWriteLoop(inMemoryStore);
+    return;
+  }
+}
+
+function metaFieldsEqual(a, b) {
+  if (Primitive_object.equal(a.first_event_block, b.first_event_block) && a.buffer_block === b.buffer_block && a._is_hyper_sync === b._is_hyper_sync) {
+    return Primitive_object.equal(Stdlib_Option.map(Primitive_option.fromNull(a.ready_at), prim => prim.getTime()), Stdlib_Option.map(Primitive_option.fromNull(b.ready_at), prim => prim.getTime()));
+  } else {
+    return false;
+  }
+}
+
+function setChainMeta(inMemoryStore, chainsData) {
+  Utils.Dict.forEachWithKey(chainsData, (meta, chainId) => {
+    let prev = inMemoryStore.chainMeta[chainId];
+    let changed = prev !== undefined ? !metaFieldsEqual(meta, prev) : true;
+    if (changed) {
+      inMemoryStore.chainMeta[chainId] = meta;
+      inMemoryStore.chainMetaDirty = true;
+      return;
     }
-    let shouldInitialize = effectCacheRecord.count === 0;
-    effectCacheRecord.count = (effectCacheRecord.count + items.length | 0) - invalidationsCount | 0;
-    Prometheus.EffectCacheCount.set(effectCacheRecord.count, effectName);
-    acc.push({
-      effect: effect,
-      items: items,
-      shouldInitialize: shouldInitialize
+  });
+  if (inMemoryStore.chainMetaDirty) {
+    return Throttler.schedule(inMemoryStore.chainMetaThrottler, () => {
+      kick(inMemoryStore);
+      return Promise.resolve();
     });
-  }), acc), updatedEntities);
-  inMemoryStore.rawEvents = [];
-  inMemoryStore.effects = {};
-  inMemoryStore.rollback = undefined;
+  }
+}
+
+function commitBatch(inMemoryStore, batch) {
+  inMemoryStore.processedBatches.push(batch);
   let checkpointId = Utils.$$Array.last(batch.checkpointIds);
-  inMemoryStore.committedCheckpointId = checkpointId !== undefined ? checkpointId : committedCheckpointId;
-  if (keepLatestChanges) {
-    persistence.allEntities.forEach(entityConfig => {
-      let table = getInMemTable(inMemoryStore, entityConfig);
-      inMemoryStore.entities[entityConfig.name] = InMemoryTable.Entity.resetButKeepLatestChanges(table);
-    });
+  if (checkpointId !== undefined) {
+    inMemoryStore.processedCheckpointId = checkpointId;
+  }
+  kick(inMemoryStore);
+}
+
+function dropCommitted(inMemoryStore, keepLoadedFromDb) {
+  let committedCheckpointId = inMemoryStore.committedCheckpointId;
+  inMemoryStore.allEntities.forEach(entityConfig => InMemoryTable.Entity.dropCommittedChanges(getInMemTable(inMemoryStore, entityConfig), committedCheckpointId, keepLoadedFromDb));
+  Utils.Dict.forEach(inMemoryStore.effects, inMemTable => dropCommittedEffects(inMemTable, committedCheckpointId, keepLoadedFromDb));
+}
+
+async function awaitCapacity(inMemoryStore) {
+  if (!inMemoryStore.hasFailedWrite && getChangesCount(inMemoryStore) >= Env.inMemoryObjectsTarget) {
+    dropCommitted(inMemoryStore, true);
+    if (getChangesCount(inMemoryStore) >= Env.inMemoryObjectsTarget) {
+      dropCommitted(inMemoryStore, false);
+    }
+    if (getChangesCount(inMemoryStore) >= Env.inMemoryObjectsTarget && Utils.$$Array.notEmpty(inMemoryStore.processedBatches)) {
+      kick(inMemoryStore);
+      await waitForCommit(inMemoryStore);
+      return await awaitCapacity(inMemoryStore);
+    } else {
+      return;
+    }
+  }
+}
+
+async function flush(inMemoryStore) {
+  if (inMemoryStore.hasFailedWrite) {
     return;
   }
-  let loadedFromDbCount = {
-    contents: 0
-  };
-  let resetTables = persistence.allEntities.map(entityConfig => {
-    let resetTable = InMemoryTable.Entity.resetButKeepLoadedFromDbChanges(getInMemTable(inMemoryStore, entityConfig));
-    loadedFromDbCount.contents = loadedFromDbCount.contents + resetTable.changesCount;
-    return resetTable;
-  });
-  let dropEverything = loadedFromDbCount.contents >= 50000;
-  persistence.allEntities.forEach((entityConfig, idx) => {
-    inMemoryStore.entities[entityConfig.name] = dropEverything ? InMemoryTable.Entity.make() : resetTables[idx];
-  });
+  kick(inMemoryStore);
+  let fiber = inMemoryStore.writeFiber;
+  if (fiber !== undefined) {
+    await fiber;
+    return await flush(inMemoryStore);
+  }
 }
 
-async function prepareRollbackDiff(inMemoryStore, persistence, rollbackTargetCheckpointId, rollbackDiffCheckpointId) {
-  inMemoryStore.rawEvents = [];
+async function prepareRollbackDiff(inMemoryStore, persistence, rollbackTargetCheckpointId, rollbackDiffCheckpointId, progressBlockNumberByChainId) {
   inMemoryStore.entities = make(inMemoryStore.allEntities);
   inMemoryStore.effects = {};
   inMemoryStore.rollback = {
     targetCheckpointId: rollbackTargetCheckpointId,
-    diffCheckpointId: rollbackDiffCheckpointId
+    diffCheckpointId: rollbackDiffCheckpointId,
+    progressBlockNumberByChainId: progressBlockNumberByChainId
   };
   let deletedEntities = {};
   let setEntities = {};
@@ -247,17 +480,35 @@ function setBatchDcs(inMemoryStore, batch) {
   }
 }
 
-let keepLatestChangesLimit = 50000;
+let keepLatestChangesLimit = Env.inMemoryObjectsTarget;
 
 export {
   EntityTables,
   make$1 as make,
   keepLatestChangesLimit,
   getEffectInMemTable,
+  getEffectOutput,
+  setEffectOutput,
+  initEffectOutputFromDb,
+  dropCommittedEffects,
   getInMemTable,
   isRollingBack,
-  writeBatch,
+  getChangesCount,
+  wakeCommitWaiters,
+  waitForCommit,
+  drainBatchRun,
+  snapshotEffects,
+  runOneWrite,
+  hasPendingWrite,
+  runWriteLoop,
+  kick,
+  metaFieldsEqual,
+  setChainMeta,
+  commitBatch,
+  dropCommitted,
+  awaitCapacity,
+  flush,
   prepareRollbackDiff,
   setBatchDcs,
 }
-/* Utils Not a pure module */
+/* Env Not a pure module */

package/src/InMemoryTable.res

@@ -9,13 +9,14 @@ module Entity = {
   type entityIndices = Utils.Set.t<TableIndices.Index.t>
   type t = {
     latestEntityChangeById: dict<Change.t<Internal.entity>>,
-    // Counts every recorded change (new latest ids and pushes to
-    // prevEntityChanges), kept in sync manually so InMemoryStore can gauge the
-    // store size without scanning every dict.
+    // Recorded changes (new latest ids + prevEntityChanges pushes), tracked
+    // manually so InMemoryStore can gauge size without scanning every dict.
     mutable changesCount: float,
-    prevEntityChanges: array<Change.t<Internal.entity>>,
-    indicesByEntityId: dict<entityIndices>,
-    fieldNameIndices: indexFieldNameToIndices,
+    // Swapped out when a write starts so processing keeps appending while the
+    // previous changes persist in the background.
+    mutable prevEntityChanges: array<Change.t<Internal.entity>>,
+    mutable indicesByEntityId: dict<entityIndices>,
+    mutable fieldNameIndices: indexFieldNameToIndices,
   }
 
   // Helper to extract entity ID from any entity
@@ -55,33 +56,51 @@ module Entity = {
     fieldNameIndices: Dict.make(),
   }
 
-  // Drops the per-batch index state and rollback history, but keeps the
-  // already committed entities so the next batch can read them without
-  // hitting the database.
-  let resetButKeepLatestChanges = (self: t): t => {
-    ...make(),
-    latestEntityChangeById: self.latestEntityChangeById,
-    // writeBatch already mutated this to subtract the dropped prevEntityChanges.
-    changesCount: self.changesCount,
+  // Changes to persist for checkpoints in (committedCheckpointId, upToCheckpointId].
+  // Those above upToCheckpointId stay in the table for a later write, while
+  // concurrent processing keeps accumulating.
+  let snapshotChanges = (self: t, ~committedCheckpointId, ~upToCheckpointId): array<
+    Change.t<Internal.entity>,
+  > => {
+    let changes = []
+    let keptPrev = []
+    self.prevEntityChanges->Array.forEach(change =>
+      if change->Change.getCheckpointId > upToCheckpointId {
+        keptPrev->Array.push(change)
+      } else {
+        changes->Array.push(change)
+      }
+    )
+    self.prevEntityChanges = keptPrev
+    self.changesCount = self.changesCount -. changes->Array.length->Int.toFloat
+    self.latestEntityChangeById->Utils.Dict.forEach(change => {
+      let checkpointId = change->Change.getCheckpointId
+      if checkpointId > committedCheckpointId && !(checkpointId > upToCheckpointId) {
+        changes->Array.push(change)
+      }
+    })
+    changes
   }
 
-  // Like resetButKeepLatestChanges, but only keeps entities loaded from the db
-  // (changes carrying loadedFromDbCheckpointId), dropping everything written in
-  // a batch. The kept count is exposed through the table's changesCount.
-  let resetButKeepLoadedFromDbChanges = (self: t): t => {
-    let latestEntityChangeById = Dict.make()
-    let keptCount = ref(0.)
-    self.latestEntityChangeById->Utils.Dict.forEachWithKey((change, key) =>
-      if change->Change.getCheckpointId === Internal.loadedFromDbCheckpointId {
-        latestEntityChangeById->Dict.set(key, change)
-        keptCount := keptCount.contents +. 1.
+  // Frees committed changes: drops latest entries at or below committedCheckpointId
+  // (re-readable from the db) and clears the per-batch indices (rebuilt on the next
+  // getWhere). Uncommitted changes are kept. With keepLoadedFromDb, entries seeded
+  // from a db read are spared so the cheaper-to-re-derive writes are dropped first.
+  let dropCommittedChanges = (self: t, ~committedCheckpointId, ~keepLoadedFromDb) => {
+    let keysToDelete = []
+    self.latestEntityChangeById->Utils.Dict.forEachWithKey((change, key) => {
+      let checkpointId = change->Change.getCheckpointId
+      if (
+        !(checkpointId > committedCheckpointId) &&
+        !(keepLoadedFromDb && checkpointId == Internal.loadedFromDbCheckpointId)
+      ) {
+        keysToDelete->Array.push(key)
       }
-    )
-    {
-      ...make(),
-      latestEntityChangeById,
-      changesCount: keptCount.contents,
-    }
+    })
+    keysToDelete->Array.forEach(key => self.latestEntityChangeById->Utils.Dict.deleteInPlace(key))
+    self.changesCount = self.changesCount -. keysToDelete->Array.length->Int.toFloat
+    self.indicesByEntityId = Dict.make()
+    self.fieldNameIndices = Dict.make()
   }
 
   let updateIndices = (self: t, ~entity: Internal.entity) => {