envio 3.12.0 → 3.13.0-alpha.0

package/src/IndexerState.res

@@ -115,6 +115,10 @@ type t = {
   // waitForNewBlock waiter is bound to the old, pre-realtime source). A fetch
   // response or waiter carrying an older epoch than this is discarded.
   mutable epoch: int,
+  // The loop's one door in from outside it: IndexerLoop owns scheduling and
+  // wires this when it starts, so an event the loop can't see for itself can
+  // still make it re-evaluate. A no-op before then.
+  mutable scheduleProcessing: unit => unit,
   // None off the simulate path.
   simulateDeadInputTracker: option<SimulateDeadInputTracker.t>,
   // --- Metric counters, rendered by Metrics at scrape time. ---
@@ -141,6 +145,7 @@ let make = (
   ~chainStates: dict<ChainState.t>,
   ~isRealtime: bool,
   ~targetBufferSize=CrossChainState.calculateTargetBufferSize(),
+  ~holdRealtime=false,
   ~committedFrontier=Frontier.empty(),
   ~isDevelopmentMode=false,
   ~shouldUseTui=false,
@@ -180,11 +185,17 @@ let make = (
     chainMetaDirty: false,
     chainMetaThrottler,
     isProcessing: false,
-    crossChainState: CrossChainState.make(~chainStates, ~isRealtime, ~targetBufferSize),
+    crossChainState: CrossChainState.make(
+      ~chainStates,
+      ~isRealtime,
+      ~targetBufferSize,
+      ~holdRealtime,
+    ),
     indexerStartTime: Date.make(),
     indexerStartTimeRef: Performance.now(),
     rollbackState: NoRollback,
     lastPrunedAtMillis: Dict.make(),
+    scheduleProcessing: () => (),
     loadManager: LoadManager.make(),
     keepProcessAlive: isDevelopmentMode || shouldUseTui,
     exitAfterFirstEventBlock,
@@ -227,6 +238,9 @@ let makeFromDbState = (
   ~exitAfterFirstEventBlock=false,
   ~reducedPollingInterval=?,
   ~targetBufferSize=CrossChainState.calculateTargetBufferSize(),
+  // A process driving part of a split run waits for its supervisor before
+  // entering the reorg threshold or switching to realtime.
+  ~holdRealtime=false,
   ~onError,
   ~onExit=?,
 ) => {
@@ -274,6 +288,7 @@ let makeFromDbState = (
     ~chainStates,
     ~isRealtime,
     ~targetBufferSize,
+    ~holdRealtime,
     ~committedFrontier=initialState.checkpointFrontier,
     ~isDevelopmentMode,
     ~shouldUseTui,
@@ -500,9 +515,33 @@ let isFinalizingIndexes = (state: t) =>
   state.crossChainState->CrossChainState.isCaughtUp &&
     !(state.crossChainState->CrossChainState.isRealtime)
 
+// The FinalizingIndexes phase is the transition a held process waits on: it
+// ends with `ready_at` committed and the indexer realtime.
+let shouldFinalizeIndexes = (state: t) =>
+  state->isFinalizingIndexes && !(state.crossChainState->CrossChainState.isHoldingRealtime)
+
 let markCaughtUpIfSettled = (state: t) =>
   state.crossChainState->CrossChainState.markCaughtUpIfSettled
 
+let isReadyToEnterReorgThreshold = (state: t, ~batch) =>
+  state.crossChainState->CrossChainState.isReadyToEnterReorgThreshold(~batch)
+
+let bindScheduleProcessing = (state: t, scheduleProcessing) =>
+  state.scheduleProcessing = scheduleProcessing
+
+// A process still waiting on its supervisor owes the schema the indexes its
+// chains deferred, so reaching every end block doesn't make it done.
+let isHoldingRealtime = (state: t) => state.crossChainState->CrossChainState.isHoldingRealtime
+
+let hasArrivedAtHead = (state: t) => state.crossChainState->CrossChainState.hasArrivedAtHead
+
+let releaseRealtime = (state: t) => {
+  state.crossChainState->CrossChainState.releaseRealtime
+  // Every chain is parked at the head with no batch coming, so nothing would
+  // notice the hold is gone without a pass through processing.
+  state.scheduleProcessing()
+}
+
 let markReady = (state: t, ~readyAt) => state.crossChainState->CrossChainState.markReady(~readyAt)
 
 let rollbackState = (state: t) => state.rollbackState
@@ -572,6 +611,7 @@ let toMetrics = (state: t): Metrics.t => {
     elapsedSeconds: state.indexerStartTimeRef->Performance.secondsSince,
     targetBufferSize: state.crossChainState->CrossChainState.targetBufferSize,
     isInReorgThreshold: state.crossChainState->CrossChainState.isInReorgThreshold,
+    hasArrivedAtHead: state.crossChainState->CrossChainState.hasArrivedAtHead,
     rollbackEnabled: state.config.shouldRollbackOnReorg,
     maxBatchSize: state.config.batchSize,
     preloadSeconds: state.preloadSeconds,

package/src/IndexerState.res.mjs

@@ -24,8 +24,9 @@ import * as Primitive_option from "@rescript/runtime/lib/es6/Primitive_option.js
 import * as CheckpointSequence from "./db/CheckpointSequence.res.mjs";
 import * as SimulateDeadInputTracker from "./SimulateDeadInputTracker.res.mjs";
 
-function make(config, persistence, chainStates, isRealtime, targetBufferSizeOpt, committedFrontierOpt, isDevelopmentModeOpt, shouldUseTuiOpt, exitAfterFirstEventBlockOpt, onError, onExit) {
+function make(config, persistence, chainStates, isRealtime, targetBufferSizeOpt, holdRealtimeOpt, committedFrontierOpt, isDevelopmentModeOpt, shouldUseTuiOpt, exitAfterFirstEventBlockOpt, onError, onExit) {
   let targetBufferSize = targetBufferSizeOpt !== undefined ? targetBufferSizeOpt : CrossChainState.calculateTargetBufferSize();
+  let holdRealtime = holdRealtimeOpt !== undefined ? holdRealtimeOpt : false;
   let committedFrontier = committedFrontierOpt !== undefined ? committedFrontierOpt : Frontier.empty();
   let isDevelopmentMode = isDevelopmentModeOpt !== undefined ? isDevelopmentModeOpt : false;
   let shouldUseTui = shouldUseTuiOpt !== undefined ? shouldUseTuiOpt : false;
@@ -54,7 +55,7 @@ function make(config, persistence, chainStates, isRealtime, targetBufferSizeOpt,
     chainMetaDirty: false,
     chainMetaThrottler: chainMetaThrottler,
     isProcessing: false,
-    crossChainState: CrossChainState.make(chainStates, isRealtime, targetBufferSize),
+    crossChainState: CrossChainState.make(chainStates, isRealtime, targetBufferSize, holdRealtime),
     rollbackState: "NoRollback",
     indexerStartTime: new Date(),
     indexerStartTimeRef: Performance.now(),
@@ -66,6 +67,7 @@ function make(config, persistence, chainStates, isRealtime, targetBufferSizeOpt,
     onExit: onExit,
     isStopped: false,
     epoch: 0,
+    scheduleProcessing: () => {},
     simulateDeadInputTracker: SimulateDeadInputTracker.makeFromConfig(config),
     preloadSeconds: 0,
     processingSeconds: 0,
@@ -82,11 +84,12 @@ function make(config, persistence, chainStates, isRealtime, targetBufferSizeOpt,
   };
 }
 
-function makeFromDbState(config, persistence, initialState, registrationsByChainId, isDevelopmentModeOpt, shouldUseTuiOpt, exitAfterFirstEventBlockOpt, reducedPollingInterval, targetBufferSizeOpt, onError, onExit) {
+function makeFromDbState(config, persistence, initialState, registrationsByChainId, isDevelopmentModeOpt, shouldUseTuiOpt, exitAfterFirstEventBlockOpt, reducedPollingInterval, targetBufferSizeOpt, holdRealtimeOpt, onError, onExit) {
   let isDevelopmentMode = isDevelopmentModeOpt !== undefined ? isDevelopmentModeOpt : false;
   let shouldUseTui = shouldUseTuiOpt !== undefined ? shouldUseTuiOpt : false;
   let exitAfterFirstEventBlock = exitAfterFirstEventBlockOpt !== undefined ? exitAfterFirstEventBlockOpt : false;
   let targetBufferSize = targetBufferSizeOpt !== undefined ? targetBufferSizeOpt : CrossChainState.calculateTargetBufferSize();
+  let holdRealtime = holdRealtimeOpt !== undefined ? holdRealtimeOpt : false;
   let isInReorgThreshold = initialState.cleanRun ? false : initialState.chains.some(resumedChainState => {
       let progressBlockNumber = resumedChainState.progressBlockNumber;
       let sourceBlockNumber = resumedChainState.sourceBlockNumber;
@@ -104,7 +107,7 @@ function makeFromDbState(config, persistence, initialState, registrationsByChain
     let chainConfig = ChainMap.get(config.chainMap, chainId);
     chainStates[resumedChainState.id] = ChainState.makeFromDbState(chainConfig, resumedChainState, initialState.reorgCheckpoints, isInReorgThreshold, isRealtime, config, initialState.contractMapping, registrationsByChainId, reducedPollingInterval);
   });
-  let state = make(config, persistence, chainStates, isRealtime, targetBufferSize, initialState.checkpointFrontier, isDevelopmentMode, shouldUseTui, exitAfterFirstEventBlock, onError, onExit);
+  let state = make(config, persistence, chainStates, isRealtime, targetBufferSize, holdRealtime, initialState.checkpointFrontier, isDevelopmentMode, shouldUseTui, exitAfterFirstEventBlock, onError, onExit);
   CrossChainState.markCaughtUpOnResume(state.crossChainState);
   Utils.Dict.forEach(initialState.cache, param => {
     let count = param.count;
@@ -350,10 +353,39 @@ function isFinalizingIndexes(state) {
   }
 }
 
+function shouldFinalizeIndexes(state) {
+  if (isFinalizingIndexes(state)) {
+    return !CrossChainState.isHoldingRealtime(state.crossChainState);
+  } else {
+    return false;
+  }
+}
+
 function markCaughtUpIfSettled(state) {
   CrossChainState.markCaughtUpIfSettled(state.crossChainState);
 }
 
+function isReadyToEnterReorgThreshold(state, batch) {
+  return CrossChainState.isReadyToEnterReorgThreshold(state.crossChainState, batch);
+}
+
+function bindScheduleProcessing(state, scheduleProcessing) {
+  state.scheduleProcessing = scheduleProcessing;
+}
+
+function isHoldingRealtime(state) {
+  return CrossChainState.isHoldingRealtime(state.crossChainState);
+}
+
+function hasArrivedAtHead(state) {
+  return CrossChainState.hasArrivedAtHead(state.crossChainState);
+}
+
+function releaseRealtime(state) {
+  CrossChainState.releaseRealtime(state.crossChainState);
+  state.scheduleProcessing();
+}
+
 function markReady(state, readyAt) {
   CrossChainState.markReady(state.crossChainState, readyAt);
 }
@@ -447,6 +479,7 @@ function toMetrics(state) {
     elapsedSeconds: Performance.secondsSince(state.indexerStartTimeRef),
     targetBufferSize: CrossChainState.targetBufferSize(state.crossChainState),
     isInReorgThreshold: CrossChainState.isInReorgThreshold(state.crossChainState),
+    hasArrivedAtHead: CrossChainState.hasArrivedAtHead(state.crossChainState),
     rollbackEnabled: state.config.shouldRollbackOnReorg,
     maxBatchSize: state.config.batchSize,
     preloadSeconds: state.preloadSeconds,
@@ -844,6 +877,12 @@ export {
   isRealtime,
   isFinalizingIndexes,
   markCaughtUpIfSettled,
+  isReadyToEnterReorgThreshold,
+  shouldFinalizeIndexes,
+  bindScheduleProcessing,
+  isHoldingRealtime,
+  hasArrivedAtHead,
+  releaseRealtime,
   markReady,
   rollbackState,
   indexerStartTime,

package/src/IndexerState.resi

@@ -16,6 +16,7 @@ let make: (
   ~chainStates: dict<ChainState.t>,
   ~isRealtime: bool,
   ~targetBufferSize: int=?,
+  ~holdRealtime: bool=?,
   ~committedFrontier: Frontier.t=?,
   ~isDevelopmentMode: bool=?,
   ~shouldUseTui: bool=?,
@@ -34,6 +35,7 @@ let makeFromDbState: (
   ~exitAfterFirstEventBlock: bool=?,
   ~reducedPollingInterval: int=?,
   ~targetBufferSize: int=?,
+  ~holdRealtime: bool=?,
   ~onError: ErrorHandling.t => unit,
   ~onExit: unit => unit=?,
 ) => t
@@ -93,6 +95,14 @@ let shouldSaveHistory: t => dict<bool>
 let isRealtime: t => bool
 let isFinalizingIndexes: t => bool
 let markCaughtUpIfSettled: t => unit
+let isReadyToEnterReorgThreshold: (t, ~batch: Batch.t) => bool
+let shouldFinalizeIndexes: t => bool
+// Wires the loop's way back in. IndexerLoop calls this as it starts.
+let bindScheduleProcessing: (t, unit => unit) => unit
+let isHoldingRealtime: t => bool
+let hasArrivedAtHead: t => bool
+// The supervisor's go-ahead for a process driving part of a split run.
+let releaseRealtime: t => unit
 let markReady: (t, ~readyAt: Date.t) => unit
 let rollbackState: t => rollbackState
 let indexerStartTime: t => Date.t

package/src/LoadLayer.res

@@ -6,23 +6,6 @@ let scopeKeySuffix = (scope: Internal.chainScope) =>
   | Chain(chainId) => `.${chainId->ChainId.toString}`
   }
 
-// Narrows a query to the scope's chain. Cross-chain entities have no chain-id
-// column, so their filter is left untouched.
-let scopeFilter = (filter: EntityFilter.t, ~table: Table.table, ~scope: Internal.chainScope) =>
-  switch (scope, table->Table.getChainIdField) {
-  | (Chain(chainId), Some(field)) =>
-    EntityFilter.And({
-      filters: [
-        filter,
-        Eq({
-          fieldName: field.fieldName,
-          fieldValue: chainId->(Utils.magic: ChainId.t => unknown),
-        }),
-      ],
-    })
-  | _ => filter
-  }
-
 let loadById = (
   ~loadManager,
   ~persistence: Persistence.t,
@@ -48,10 +31,10 @@ let loadById = (
       (
         await storage.loadOrThrow(
           ~table=entityConfig.table,
-          ~filter=EntityFilter.In({
-            fieldName: Table.idFieldName,
-            fieldValue: idsToLoad->(Utils.magic: array<string> => array<unknown>),
-          })->scopeFilter(~table=entityConfig.table, ~scope),
+          ~filter=EntityFilter.byIds(idsToLoad)->EntityFilter.scoped(
+            ~table=entityConfig.table,
+            ~scope,
+          ),
         )
       )->(Utils.magic: array<unknown> => array<Internal.entity>)
     } catch {
@@ -273,15 +256,9 @@ let loadEffect = (
       let {outputSchema} = effect.storageMeta
 
       let dbEntities = try {
-        (
-          await storage.loadOrThrow(
-            ~table,
-            ~filter=EntityFilter.In({
-              fieldName: Table.idFieldName,
-              fieldValue: idsToLoad->(Utils.magic: array<string> => array<unknown>),
-            }),
-          )
-        )->(Utils.magic: array<unknown> => array<Internal.effectCacheItem>)
+        (await storage.loadOrThrow(~table, ~filter=EntityFilter.byIds(idsToLoad)))->(
+          Utils.magic: array<unknown> => array<Internal.effectCacheItem>
+        )
       } catch {
       | exn =>
         Ecosystem.getItemLogger(item, ~ecosystem)->Logging.childWarn({
@@ -355,7 +332,7 @@ let loadEffect = (
   )
 }
 
-let loadByFilter = (
+let loadByParsedFilter = (
   ~loadManager,
   ~persistence: Persistence.t,
   ~entityConfig: Internal.entityConfig,
@@ -378,7 +355,9 @@ let loadByFilter = (
 
     let size = ref(0)
 
-    filters->Array.forEach(filter => inMemTable->InMemoryTable.Entity.addEmptyIndex(~filter))
+    filters->Array.forEach(filter =>
+      inMemTable->InMemoryTable.Entity.addEmptyIndex(~filter, ~table=entityConfig.table)
+    )
 
     // Any non-derived field can be filtered on, so the columns this query reads
     // are indexed on demand before it runs rather than promised by the schema.
@@ -399,7 +378,7 @@ let loadByFilter = (
           (
             await storage.loadOrThrow(
               ~table=entityConfig.table,
-              ~filter=filter->scopeFilter(~table=entityConfig.table, ~scope),
+              ~filter=filter->EntityFilter.scoped(~table=entityConfig.table, ~scope),
             )
           )->(Utils.magic: array<unknown> => array<Internal.entity>)
 
@@ -412,6 +391,10 @@ let loadByFilter = (
           )
         })
 
+        // Every row this filter's values could match is now in the table, so a
+        // later getWhere naming any of them needs no round trip of its own.
+        inMemTable->InMemoryTable.Entity.recordLoadedValues(~filter, ~table=entityConfig.table)
+
         size := size.contents + entities->Array.length
       } catch {
       | Persistence.StorageError({message, reason}) =>
@@ -442,13 +425,53 @@ let loadByFilter = (
     )
   }
 
+  // Keying an _in walks every value, so it's computed once here and handed to
+  // the load manager rather than recomputed by the hasher.
+  let filterKey = filter->EntityFilter.toString(~table=entityConfig.table)
+
+  if !(inMemTable->InMemoryTable.Entity.hasIndex)(filterKey) {
+    inMemTable->InMemoryTable.Entity.tryIndexFromLoadedValues(~filter, ~table=entityConfig.table)
+  }
+
   loadManager->LoadManager.call(
     ~key,
     ~load,
     ~input=filter,
     ~shouldGroup,
-    ~hasher=EntityFilter.toString,
+    ~hasher=_ => filterKey,
     ~getUnsafeInMemory=inMemTable->InMemoryTable.Entity.getUnsafeOnIndex,
     ~hasInMemory=inMemTable->InMemoryTable.Entity.hasIndex,
   )
 }
+
+let loadByFilter = (
+  ~loadManager,
+  ~persistence,
+  ~entityConfig: Internal.entityConfig,
+  ~scope,
+  ~indexerState,
+  ~shouldGroup,
+  ~item,
+  ~ecosystem,
+  ~filter: dict<dict<unknown>>,
+) =>
+  // Rejecting rather than throwing keeps a bad filter failing only its own
+  // call, even when the caller batches several with Promise.all.
+  try {
+    loadByParsedFilter(
+      ~loadManager,
+      ~persistence,
+      ~entityConfig,
+      ~scope,
+      ~indexerState,
+      ~shouldGroup,
+      ~item,
+      ~ecosystem,
+      ~filter=filter->EntityFilter.parseOrThrow(
+        ~entityName=entityConfig.name,
+        ~table=entityConfig.table,
+      ),
+    )
+  } catch {
+  | exn => Promise.reject(exn->Utils.prettifyExn)
+  }

package/src/LoadLayer.res.mjs

@@ -1,6 +1,5 @@
 // Generated by ReScript, PLEASE EDIT WITH CARE
 
-import * as Table from "./db/Table.res.mjs";
 import * as Utils from "./Utils.res.mjs";
 import * as ChainId from "./ChainId.res.mjs";
 import * as Logging from "./Logging.res.mjs";
@@ -27,25 +26,6 @@ function scopeKeySuffix(scope) {
   }
 }
 
-function scopeFilter(filter, table, scope) {
-  let match = Table.getChainIdField(table);
-  if (scope === "crossChain" || match === undefined) {
-    return filter;
-  } else {
-    return {
-      operator: "and",
-      filters: [
-        filter,
-        {
-          operator: "=",
-          fieldName: match.fieldName,
-          fieldValue: scope
-        }
-      ]
-    };
-  }
-}
-
 function loadById(loadManager, persistence, entityConfig, scope, indexerState, shouldGroup, item, ecosystem, entityId) {
   let key = entityConfig.name + `.get` + scopeKeySuffix(scope);
   let inMemTable = InMemoryStore.getInMemTable(indexerState, entityConfig, scope);
@@ -54,11 +34,7 @@ function loadById(loadManager, persistence, entityConfig, scope, indexerState, s
     let timerRef = IndexerState.startStorageLoad(indexerState, storage.name, key);
     let dbEntities;
     try {
-      dbEntities = await storage.loadOrThrow(scopeFilter({
-        operator: "in",
-        fieldName: Table.idFieldName,
-        fieldValue: idsToLoad
-      }, entityConfig.table, scope), entityConfig.table);
+      dbEntities = await storage.loadOrThrow(EntityFilter.scoped(EntityFilter.byIds(idsToLoad), entityConfig.table, scope), entityConfig.table);
     } catch (raw_exn) {
       let exn = Primitive_exceptions.internalToException(raw_exn);
       if (exn.RE_EXN_ID === Persistence.StorageError) {
@@ -157,11 +133,7 @@ function loadEffect(loadManager, persistence, effect, effectArgs, scope, indexer
       let outputSchema = match$1.outputSchema;
       let dbEntities;
       try {
-        dbEntities = await storage.loadOrThrow({
-          operator: "in",
-          fieldName: Table.idFieldName,
-          fieldValue: idsToLoad
-        }, table);
+        dbEntities = await storage.loadOrThrow(EntityFilter.byIds(idsToLoad), table);
       } catch (raw_exn) {
         let exn = Primitive_exceptions.internalToException(raw_exn);
         Logging.childWarn(Ecosystem.getItemLogger(item, ecosystem), {
@@ -211,38 +183,49 @@ function loadEffect(loadManager, persistence, effect, effectArgs, scope, indexer
 }
 
 function loadByFilter(loadManager, persistence, entityConfig, scope, indexerState, shouldGroup, item, ecosystem, filter) {
-  let key = EntityFilter.toOperationKey(filter, entityConfig.name) + scopeKeySuffix(scope);
-  let inMemTable = InMemoryStore.getInMemTable(indexerState, entityConfig, scope);
-  let load = async (filters, param) => {
-    let storage = Persistence.getInitializedStorageOrThrow(persistence);
-    let timerRef = IndexerState.startStorageLoad(indexerState, storage.name, key);
-    let size = {
-      contents: 0
-    };
-    filters.forEach(filter => InMemoryTable.Entity.addEmptyIndex(inMemTable, filter));
-    await storage.ensureQueryIndexes(entityConfig, scope, filters);
-    let queries = EntityFilter.merge(filters);
-    await Promise.all(queries.map(async filter => {
-      try {
-        let entities = await storage.loadOrThrow(scopeFilter(filter, entityConfig.table, scope), entityConfig.table);
-        let committedCheckpointId = IndexerState.committedCheckpointIdFor(indexerState, scope);
-        entities.forEach(entity => InMemoryTable.Entity.initValue(inMemTable, committedCheckpointId, entity.id, entity));
-        size.contents = size.contents + entities.length | 0;
-        return;
-      } catch (raw_exn) {
-        let exn = Primitive_exceptions.internalToException(raw_exn);
-        if (exn.RE_EXN_ID === Persistence.StorageError) {
-          return ErrorHandling.mkLogAndRaise(Logging.createChildFrom(Ecosystem.getItemLogger(item, ecosystem), {
-            operation: key,
-            params: EntityFilter.getParams(filter)
-          }), exn.message, exn.reason);
+  try {
+    let filter$1 = EntityFilter.parseOrThrow(filter, entityConfig.name, entityConfig.table);
+    let key = EntityFilter.toOperationKey(filter$1, entityConfig.name) + scopeKeySuffix(scope);
+    let inMemTable = InMemoryStore.getInMemTable(indexerState, entityConfig, scope);
+    let load = async (filters, param) => {
+      let storage = Persistence.getInitializedStorageOrThrow(persistence);
+      let timerRef = IndexerState.startStorageLoad(indexerState, storage.name, key);
+      let size = {
+        contents: 0
+      };
+      filters.forEach(filter => InMemoryTable.Entity.addEmptyIndex(inMemTable, filter, entityConfig.table));
+      await storage.ensureQueryIndexes(entityConfig, scope, filters);
+      let queries = EntityFilter.merge(filters);
+      await Promise.all(queries.map(async filter => {
+        try {
+          let entities = await storage.loadOrThrow(EntityFilter.scoped(filter, entityConfig.table, scope), entityConfig.table);
+          let committedCheckpointId = IndexerState.committedCheckpointIdFor(indexerState, scope);
+          entities.forEach(entity => InMemoryTable.Entity.initValue(inMemTable, committedCheckpointId, entity.id, entity));
+          InMemoryTable.Entity.recordLoadedValues(inMemTable, filter, entityConfig.table);
+          size.contents = size.contents + entities.length | 0;
+          return;
+        } catch (raw_exn) {
+          let exn = Primitive_exceptions.internalToException(raw_exn);
+          if (exn.RE_EXN_ID === Persistence.StorageError) {
+            return ErrorHandling.mkLogAndRaise(Logging.createChildFrom(Ecosystem.getItemLogger(item, ecosystem), {
+              operation: key,
+              params: EntityFilter.getParams(filter)
+            }), exn.message, exn.reason);
+          }
+          throw exn;
         }
-        throw exn;
-      }
-    }));
-    return IndexerState.endStorageLoad(indexerState, timerRef, storage.name, key, Stdlib_Array.reduce(queries, 0, (acc, query) => acc + EntityFilter.valuesCount(query) | 0), size.contents);
-  };
-  return LoadManager.call(loadManager, filter, key, load, EntityFilter.toString, shouldGroup, InMemoryTable.Entity.hasIndex(inMemTable), InMemoryTable.Entity.getUnsafeOnIndex(inMemTable));
+      }));
+      return IndexerState.endStorageLoad(indexerState, timerRef, storage.name, key, Stdlib_Array.reduce(queries, 0, (acc, query) => acc + EntityFilter.valuesCount(query) | 0), size.contents);
+    };
+    let filterKey = EntityFilter.toString(filter$1, entityConfig.table);
+    if (!InMemoryTable.Entity.hasIndex(inMemTable)(filterKey)) {
+      InMemoryTable.Entity.tryIndexFromLoadedValues(inMemTable, filter$1, entityConfig.table);
+    }
+    return LoadManager.call(loadManager, filter$1, key, load, param => filterKey, shouldGroup, InMemoryTable.Entity.hasIndex(inMemTable), InMemoryTable.Entity.getUnsafeOnIndex(inMemTable));
+  } catch (raw_exn) {
+    let exn = Primitive_exceptions.internalToException(raw_exn);
+    return Promise.reject(Utils.prettifyExn(exn));
+  }
 }
 
 export {
@@ -250,4 +233,4 @@ export {
   loadByFilter,
   loadEffect,
 }
-/* Table Not a pure module */
+/* Utils Not a pure module */

package/src/LoadLayer.resi

@@ -19,7 +19,7 @@ let loadByFilter: (
   ~shouldGroup: bool,
   ~item: Internal.item,
   ~ecosystem: Ecosystem.t,
-  ~filter: EntityFilter.t,
+  ~filter: dict<dict<unknown>>,
 ) => promise<array<Internal.entity>>
 
 let loadEffect: (

package/src/Logging.res

@@ -26,6 +26,30 @@ let logLevels = [
 
 %%private(let logger = ref(None))
 
+// Fields every line this process logs carries. Merged into each line rather
+// than bound to a child logger: pino writes a child's bindings and the line's
+// own fields side by side, so a line that names the same key would carry it
+// twice. A fresh object per line, since pino merges the line's fields into
+// whatever this returns.
+%%private(let context: ref<dict<JSON.t>> = ref(Dict.make()))
+%%private(let mixin = () => JSON.Object(context.contents->Dict.copy))
+
+// A child logger that binds a field the process already carries would have pino
+// write it twice: a child's bindings and the context are concatenated into the
+// line, not merged. The context is the one place a line names it, which it can
+// be because a process only takes one when its every line is about that chain.
+%%private(
+  let withoutContext = (params: 'a) =>
+    switch context.contents->Dict.keysToArray {
+    | [] => params
+    | keys => {
+        let narrowed = params->(Utils.magic: 'a => dict<JSON.t>)->Dict.copy
+        keys->Array.forEach(key => narrowed->Dict.delete(key))
+        narrowed->(Utils.magic: dict<JSON.t> => 'a)
+      }
+    }
+)
+
 let makeLogger = (~logStrategy, ~logFilePath, ~defaultFileLogLevel, ~userLogLevel) => {
   // Currently unused - useful if using multiple transports.
   // let pinoRaw = {"target": "pino/file", "level": Config.userLogLevel}
@@ -56,17 +80,19 @@ let makeLogger = (~logStrategy, ~logFilePath, ~defaultFileLogLevel, ~userLogLeve
         ...Pino.ECS.make(),
         customLevels: logLevels,
         base,
+        mixin,
       },
       Transport.make(pinoFile),
     )
   | EcsConsoleMultistream =>
-    makeMultiStreamLogger(~logFile=None, ~options=Some({...Pino.ECS.make(), base}))
+    makeMultiStreamLogger(~logFile=None, ~options=Some({...Pino.ECS.make(), base, mixin}))
   | EcsConsole =>
     make({
       ...Pino.ECS.make(),
       level: userLogLevel,
       customLevels: logLevels,
       base,
+      mixin,
     })
   | FileOnly =>
     makeWithOptionsAndTransport(
@@ -74,12 +100,13 @@ let makeLogger = (~logStrategy, ~logFilePath, ~defaultFileLogLevel, ~userLogLeve
         customLevels: logLevels,
         level: defaultFileLogLevel,
         base,
+        mixin,
       },
       Transport.make(pinoFile),
     )
-  | ConsoleRaw => makeMultiStreamLogger(~logFile=None, ~options=Some({base: base}))
-  | ConsolePretty => makeMultiStreamLogger(~logFile=None, ~options=Some({base: base}))
-  | Both => makeMultiStreamLogger(~logFile=Some(logFilePath), ~options=Some({base: base}))
+  | ConsoleRaw => makeMultiStreamLogger(~logFile=None, ~options=Some({base, mixin}))
+  | ConsolePretty => makeMultiStreamLogger(~logFile=None, ~options=Some({base, mixin}))
+  | Both => makeMultiStreamLogger(~logFile=Some(logFilePath), ~options=Some({base, mixin}))
   }
 }
 
@@ -149,10 +176,15 @@ let childFatal = (logger, params: 'a) => {
 }
 
 let createChild = (~params: 'a) => {
-  getLogger()->child(params->createChildParams)
+  getLogger()->child(params->withoutContext->createChildParams)
 }
+
+// What belongs on every line is the run's to decide; the logger only carries
+// what it is handed. A line that names one of these fields itself wins.
+let setContext = (fields: dict<JSON.t>) => context := fields
+
 let createChildFrom = (~logger: t, ~params: 'a) => {
-  logger->child(params->createChildParams)
+  logger->child(params->withoutContext->createChildParams)
 }
 
 @inline