envio 3.11.0 → 3.12.1

package/src/LoadLayer.res

@@ -6,23 +6,6 @@ let scopeKeySuffix = (scope: Internal.chainScope) =>
   | Chain(chainId) => `.${chainId->ChainId.toString}`
   }
 
-// Narrows a query to the scope's chain. Cross-chain entities have no chain-id
-// column, so their filter is left untouched.
-let scopeFilter = (filter: EntityFilter.t, ~table: Table.table, ~scope: Internal.chainScope) =>
-  switch (scope, table->Table.getChainIdField) {
-  | (Chain(chainId), Some(field)) =>
-    EntityFilter.And({
-      filters: [
-        filter,
-        Eq({
-          fieldName: field.fieldName,
-          fieldValue: chainId->(Utils.magic: ChainId.t => unknown),
-        }),
-      ],
-    })
-  | _ => filter
-  }
-
 let loadById = (
   ~loadManager,
   ~persistence: Persistence.t,
@@ -48,10 +31,10 @@ let loadById = (
       (
         await storage.loadOrThrow(
           ~table=entityConfig.table,
-          ~filter=EntityFilter.In({
-            fieldName: Table.idFieldName,
-            fieldValue: idsToLoad->(Utils.magic: array<string> => array<unknown>),
-          })->scopeFilter(~table=entityConfig.table, ~scope),
+          ~filter=EntityFilter.byIds(idsToLoad)->EntityFilter.scoped(
+            ~table=entityConfig.table,
+            ~scope,
+          ),
         )
       )->(Utils.magic: array<unknown> => array<Internal.entity>)
     } catch {
@@ -69,9 +52,10 @@ let loadById = (
       let entity = dbEntities->Array.getUnsafe(idx)
       entitiesMap->Dict.set(entity.id, entity)
     }
+    let committedCheckpointId = indexerState->IndexerState.committedCheckpointIdFor(~scope)
     idsToLoad->Array.forEach(entityId => {
       inMemTable->InMemoryTable.Entity.initValue(
-        ~committedCheckpointId=indexerState->IndexerState.committedCheckpointId,
+        ~committedCheckpointId,
         ~key=entityId,
         ~entity=entitiesMap->Utils.Dict.dangerouslyGetNonOption(entityId),
       )
@@ -109,6 +93,7 @@ let callEffect = (
   effect.handler(arg)
   ->Promise.thenResolve(output => {
     inMemTable->InMemoryStore.setEffectOutput(
+      ~chainId=arg.chainId,
       ~checkpointId=arg.checkpointId,
       ~cacheKey=arg.cacheKey,
       ~output,
@@ -271,15 +256,9 @@ let loadEffect = (
       let {outputSchema} = effect.storageMeta
 
       let dbEntities = try {
-        (
-          await storage.loadOrThrow(
-            ~table,
-            ~filter=EntityFilter.In({
-              fieldName: Table.idFieldName,
-              fieldValue: idsToLoad->(Utils.magic: array<string> => array<unknown>),
-            }),
-          )
-        )->(Utils.magic: array<unknown> => array<Internal.effectCacheItem>)
+        (await storage.loadOrThrow(~table, ~filter=EntityFilter.byIds(idsToLoad)))->(
+          Utils.magic: array<unknown> => array<Internal.effectCacheItem>
+        )
       } catch {
       | exn =>
         Ecosystem.getItemLogger(item, ~ecosystem)->Logging.childWarn({
@@ -294,7 +273,11 @@ let loadEffect = (
         try {
           let output = dbEntity.output->S.parseOrThrow(outputSchema)
           idsFromCache->Utils.Set.add(dbEntity.id)->ignore
-          inMemTable->InMemoryStore.initEffectOutputFromDb(~cacheKey=dbEntity.id, ~output)
+          inMemTable->InMemoryStore.initEffectOutputFromDb(
+            ~chainId=item->Internal.getItemChainId,
+            ~cacheKey=dbEntity.id,
+            ~output,
+          )
         } catch {
         | S.Raised(error) =>
           inMemTable->EffectState.recordInvalidation
@@ -349,7 +332,7 @@ let loadEffect = (
   )
 }
 
-let loadByFilter = (
+let loadByParsedFilter = (
   ~loadManager,
   ~persistence: Persistence.t,
   ~entityConfig: Internal.entityConfig,
@@ -372,14 +355,16 @@ let loadByFilter = (
 
     let size = ref(0)
 
-    filters->Array.forEach(filter => inMemTable->InMemoryTable.Entity.addEmptyIndex(~filter))
+    filters->Array.forEach(filter =>
+      inMemTable->InMemoryTable.Entity.addEmptyIndex(~filter, ~table=entityConfig.table)
+    )
 
     // Any non-derived field can be filtered on, so the columns this query reads
     // are indexed on demand before it runs rather than promised by the schema.
     // Inside the load timing: waiting on the build is time the handler spends
     // waiting for this operation, and it's the only thing that explains an
     // occasional very slow getWhere.
-    await storage.ensureQueryIndexes(~table=entityConfig.table, ~filters)
+    await storage.ensureQueryIndexes(~entityConfig, ~scope, ~filters)
 
     // Loading a superset of rows via a merged query is safe: every loaded
     // entity is matched against all registered indexes, not only the
@@ -393,18 +378,23 @@ let loadByFilter = (
           (
             await storage.loadOrThrow(
               ~table=entityConfig.table,
-              ~filter=filter->scopeFilter(~table=entityConfig.table, ~scope),
+              ~filter=filter->EntityFilter.scoped(~table=entityConfig.table, ~scope),
             )
           )->(Utils.magic: array<unknown> => array<Internal.entity>)
 
+        let committedCheckpointId = indexerState->IndexerState.committedCheckpointIdFor(~scope)
         entities->Array.forEach(entity => {
           inMemTable->InMemoryTable.Entity.initValue(
-            ~committedCheckpointId=indexerState->IndexerState.committedCheckpointId,
+            ~committedCheckpointId,
             ~key=entity.id,
             ~entity=Some(entity),
           )
         })
 
+        // Every row this filter's values could match is now in the table, so a
+        // later getWhere naming any of them needs no round trip of its own.
+        inMemTable->InMemoryTable.Entity.recordLoadedValues(~filter, ~table=entityConfig.table)
+
         size := size.contents + entities->Array.length
       } catch {
       | Persistence.StorageError({message, reason}) =>
@@ -435,13 +425,53 @@ let loadByFilter = (
     )
   }
 
+  // Keying an _in walks every value, so it's computed once here and handed to
+  // the load manager rather than recomputed by the hasher.
+  let filterKey = filter->EntityFilter.toString(~table=entityConfig.table)
+
+  if !(inMemTable->InMemoryTable.Entity.hasIndex)(filterKey) {
+    inMemTable->InMemoryTable.Entity.tryIndexFromLoadedValues(~filter, ~table=entityConfig.table)
+  }
+
   loadManager->LoadManager.call(
     ~key,
     ~load,
     ~input=filter,
     ~shouldGroup,
-    ~hasher=EntityFilter.toString,
+    ~hasher=_ => filterKey,
     ~getUnsafeInMemory=inMemTable->InMemoryTable.Entity.getUnsafeOnIndex,
     ~hasInMemory=inMemTable->InMemoryTable.Entity.hasIndex,
   )
 }
+
+let loadByFilter = (
+  ~loadManager,
+  ~persistence,
+  ~entityConfig: Internal.entityConfig,
+  ~scope,
+  ~indexerState,
+  ~shouldGroup,
+  ~item,
+  ~ecosystem,
+  ~filter: dict<dict<unknown>>,
+) =>
+  // Rejecting rather than throwing keeps a bad filter failing only its own
+  // call, even when the caller batches several with Promise.all.
+  try {
+    loadByParsedFilter(
+      ~loadManager,
+      ~persistence,
+      ~entityConfig,
+      ~scope,
+      ~indexerState,
+      ~shouldGroup,
+      ~item,
+      ~ecosystem,
+      ~filter=filter->EntityFilter.parseOrThrow(
+        ~entityName=entityConfig.name,
+        ~table=entityConfig.table,
+      ),
+    )
+  } catch {
+  | exn => Promise.reject(exn->Utils.prettifyExn)
+  }

package/src/LoadLayer.res.mjs

@@ -1,9 +1,9 @@
 // Generated by ReScript, PLEASE EDIT WITH CARE
 
-import * as Table from "./db/Table.res.mjs";
 import * as Utils from "./Utils.res.mjs";
 import * as ChainId from "./ChainId.res.mjs";
 import * as Logging from "./Logging.res.mjs";
+import * as Internal from "./Internal.res.mjs";
 import * as Ecosystem from "./Ecosystem.res.mjs";
 import * as EffectState from "./EffectState.res.mjs";
 import * as LoadManager from "./LoadManager.res.mjs";
@@ -26,25 +26,6 @@ function scopeKeySuffix(scope) {
   }
 }
 
-function scopeFilter(filter, table, scope) {
-  let match = Table.getChainIdField(table);
-  if (scope === "crossChain" || match === undefined) {
-    return filter;
-  } else {
-    return {
-      operator: "and",
-      filters: [
-        filter,
-        {
-          operator: "=",
-          fieldName: match.fieldName,
-          fieldValue: scope
-        }
-      ]
-    };
-  }
-}
-
 function loadById(loadManager, persistence, entityConfig, scope, indexerState, shouldGroup, item, ecosystem, entityId) {
   let key = entityConfig.name + `.get` + scopeKeySuffix(scope);
   let inMemTable = InMemoryStore.getInMemTable(indexerState, entityConfig, scope);
@@ -53,11 +34,7 @@ function loadById(loadManager, persistence, entityConfig, scope, indexerState, s
     let timerRef = IndexerState.startStorageLoad(indexerState, storage.name, key);
     let dbEntities;
     try {
-      dbEntities = await storage.loadOrThrow(scopeFilter({
-        operator: "in",
-        fieldName: Table.idFieldName,
-        fieldValue: idsToLoad
-      }, entityConfig.table, scope), entityConfig.table);
+      dbEntities = await storage.loadOrThrow(EntityFilter.scoped(EntityFilter.byIds(idsToLoad), entityConfig.table, scope), entityConfig.table);
     } catch (raw_exn) {
       let exn = Primitive_exceptions.internalToException(raw_exn);
       if (exn.RE_EXN_ID === Persistence.StorageError) {
@@ -71,7 +48,8 @@ function loadById(loadManager, persistence, entityConfig, scope, indexerState, s
       let entity = dbEntities[idx];
       entitiesMap[entity.id] = entity;
     }
-    idsToLoad.forEach(entityId => InMemoryTable.Entity.initValue(inMemTable, IndexerState.committedCheckpointId(indexerState), entityId, entitiesMap[entityId]));
+    let committedCheckpointId = IndexerState.committedCheckpointIdFor(indexerState, scope);
+    idsToLoad.forEach(entityId => InMemoryTable.Entity.initValue(inMemTable, committedCheckpointId, entityId, entitiesMap[entityId]));
     return IndexerState.endStorageLoad(indexerState, timerRef, storage.name, key, idsToLoad.length, dbEntities.length);
   };
   return LoadManager.call(loadManager, entityId, key, load, LoadManager.noopHasher, shouldGroup, hash => hash in inMemTable.latestEntityChangeById, InMemoryTable.Entity.getUnsafe(inMemTable));
@@ -79,7 +57,7 @@ function loadById(loadManager, persistence, entityConfig, scope, indexerState, s
 
 function callEffect(effect, arg, inMemTable, timerRef, onError) {
   EffectState.startCall(inMemTable.stats, timerRef);
-  return effect.handler(arg).then(output => InMemoryStore.setEffectOutput(inMemTable, arg.checkpointId, arg.cacheKey, output, arg.context.cache)).catch(exn => onError(arg.cacheKey, exn)).finally(() => EffectState.endCall(inMemTable.stats, timerRef));
+  return effect.handler(arg).then(output => InMemoryStore.setEffectOutput(inMemTable, arg.chainId, arg.checkpointId, arg.cacheKey, output, arg.context.cache)).catch(exn => onError(arg.cacheKey, exn)).finally(() => EffectState.endCall(inMemTable.stats, timerRef));
 }
 
 function executeWithRateLimit(effect, effectArgs, inMemTable, onError, isFromQueue) {
@@ -155,11 +133,7 @@ function loadEffect(loadManager, persistence, effect, effectArgs, scope, indexer
       let outputSchema = match$1.outputSchema;
       let dbEntities;
       try {
-        dbEntities = await storage.loadOrThrow({
-          operator: "in",
-          fieldName: Table.idFieldName,
-          fieldValue: idsToLoad
-        }, table);
+        dbEntities = await storage.loadOrThrow(EntityFilter.byIds(idsToLoad), table);
       } catch (raw_exn) {
         let exn = Primitive_exceptions.internalToException(raw_exn);
         Logging.childWarn(Ecosystem.getItemLogger(item, ecosystem), {
@@ -173,7 +147,7 @@ function loadEffect(loadManager, persistence, effect, effectArgs, scope, indexer
         try {
           let output = S$RescriptSchema.parseOrThrow(dbEntity.output, outputSchema);
           idsFromCache.add(dbEntity.id);
-          return InMemoryStore.initEffectOutputFromDb(inMemTable, dbEntity.id, output);
+          return InMemoryStore.initEffectOutputFromDb(inMemTable, Internal.getItemChainId(item), dbEntity.id, output);
         } catch (raw_error) {
           let error = Primitive_exceptions.internalToException(raw_error);
           if (error.RE_EXN_ID === S$RescriptSchema.Raised) {
@@ -209,37 +183,49 @@ function loadEffect(loadManager, persistence, effect, effectArgs, scope, indexer
 }
 
 function loadByFilter(loadManager, persistence, entityConfig, scope, indexerState, shouldGroup, item, ecosystem, filter) {
-  let key = EntityFilter.toOperationKey(filter, entityConfig.name) + scopeKeySuffix(scope);
-  let inMemTable = InMemoryStore.getInMemTable(indexerState, entityConfig, scope);
-  let load = async (filters, param) => {
-    let storage = Persistence.getInitializedStorageOrThrow(persistence);
-    let timerRef = IndexerState.startStorageLoad(indexerState, storage.name, key);
-    let size = {
-      contents: 0
-    };
-    filters.forEach(filter => InMemoryTable.Entity.addEmptyIndex(inMemTable, filter));
-    await storage.ensureQueryIndexes(entityConfig.table, filters);
-    let queries = EntityFilter.merge(filters);
-    await Promise.all(queries.map(async filter => {
-      try {
-        let entities = await storage.loadOrThrow(scopeFilter(filter, entityConfig.table, scope), entityConfig.table);
-        entities.forEach(entity => InMemoryTable.Entity.initValue(inMemTable, IndexerState.committedCheckpointId(indexerState), entity.id, entity));
-        size.contents = size.contents + entities.length | 0;
-        return;
-      } catch (raw_exn) {
-        let exn = Primitive_exceptions.internalToException(raw_exn);
-        if (exn.RE_EXN_ID === Persistence.StorageError) {
-          return ErrorHandling.mkLogAndRaise(Logging.createChildFrom(Ecosystem.getItemLogger(item, ecosystem), {
-            operation: key,
-            params: EntityFilter.getParams(filter)
-          }), exn.message, exn.reason);
+  try {
+    let filter$1 = EntityFilter.parseOrThrow(filter, entityConfig.name, entityConfig.table);
+    let key = EntityFilter.toOperationKey(filter$1, entityConfig.name) + scopeKeySuffix(scope);
+    let inMemTable = InMemoryStore.getInMemTable(indexerState, entityConfig, scope);
+    let load = async (filters, param) => {
+      let storage = Persistence.getInitializedStorageOrThrow(persistence);
+      let timerRef = IndexerState.startStorageLoad(indexerState, storage.name, key);
+      let size = {
+        contents: 0
+      };
+      filters.forEach(filter => InMemoryTable.Entity.addEmptyIndex(inMemTable, filter, entityConfig.table));
+      await storage.ensureQueryIndexes(entityConfig, scope, filters);
+      let queries = EntityFilter.merge(filters);
+      await Promise.all(queries.map(async filter => {
+        try {
+          let entities = await storage.loadOrThrow(EntityFilter.scoped(filter, entityConfig.table, scope), entityConfig.table);
+          let committedCheckpointId = IndexerState.committedCheckpointIdFor(indexerState, scope);
+          entities.forEach(entity => InMemoryTable.Entity.initValue(inMemTable, committedCheckpointId, entity.id, entity));
+          InMemoryTable.Entity.recordLoadedValues(inMemTable, filter, entityConfig.table);
+          size.contents = size.contents + entities.length | 0;
+          return;
+        } catch (raw_exn) {
+          let exn = Primitive_exceptions.internalToException(raw_exn);
+          if (exn.RE_EXN_ID === Persistence.StorageError) {
+            return ErrorHandling.mkLogAndRaise(Logging.createChildFrom(Ecosystem.getItemLogger(item, ecosystem), {
+              operation: key,
+              params: EntityFilter.getParams(filter)
+            }), exn.message, exn.reason);
+          }
+          throw exn;
         }
-        throw exn;
-      }
-    }));
-    return IndexerState.endStorageLoad(indexerState, timerRef, storage.name, key, Stdlib_Array.reduce(queries, 0, (acc, query) => acc + EntityFilter.valuesCount(query) | 0), size.contents);
-  };
-  return LoadManager.call(loadManager, filter, key, load, EntityFilter.toString, shouldGroup, InMemoryTable.Entity.hasIndex(inMemTable), InMemoryTable.Entity.getUnsafeOnIndex(inMemTable));
+      }));
+      return IndexerState.endStorageLoad(indexerState, timerRef, storage.name, key, Stdlib_Array.reduce(queries, 0, (acc, query) => acc + EntityFilter.valuesCount(query) | 0), size.contents);
+    };
+    let filterKey = EntityFilter.toString(filter$1, entityConfig.table);
+    if (!InMemoryTable.Entity.hasIndex(inMemTable)(filterKey)) {
+      InMemoryTable.Entity.tryIndexFromLoadedValues(inMemTable, filter$1, entityConfig.table);
+    }
+    return LoadManager.call(loadManager, filter$1, key, load, param => filterKey, shouldGroup, InMemoryTable.Entity.hasIndex(inMemTable), InMemoryTable.Entity.getUnsafeOnIndex(inMemTable));
+  } catch (raw_exn) {
+    let exn = Primitive_exceptions.internalToException(raw_exn);
+    return Promise.reject(Utils.prettifyExn(exn));
+  }
 }
 
 export {
@@ -247,4 +233,4 @@ export {
   loadByFilter,
   loadEffect,
 }
-/* Table Not a pure module */
+/* Utils Not a pure module */

package/src/LoadLayer.resi

@@ -19,7 +19,7 @@ let loadByFilter: (
   ~shouldGroup: bool,
   ~item: Internal.item,
   ~ecosystem: Ecosystem.t,
-  ~filter: EntityFilter.t,
+  ~filter: dict<dict<unknown>>,
 ) => promise<array<Internal.entity>>
 
 let loadEffect: (

package/src/Main.res

@@ -488,7 +488,12 @@ let getGlobalIndexer = (): 'indexer => {
   Utils.Proxy.make(Utils.Object.createNullObject(), traps)->(Utils.magic: {..} => 'indexer)
 }
 
-let startServer = (~getState, ~persistence: Persistence.t, ~isDevelopmentMode: bool) => {
+let startServer = (
+  ~getMetrics: unit => option<Metrics.t>,
+  ~envioVersion: string,
+  ~persistence: Persistence.t,
+  ~isDevelopmentMode: bool,
+) => {
   open Express
 
   let app = make()
@@ -520,10 +525,20 @@ let startServer = (~getState, ~persistence: Persistence.t, ~isDevelopmentMode: b
   })
 
   app->get("/console/state", (_req, res) => {
-    let state = if isDevelopmentMode {
-      getState()
-    } else {
+    let state = if !isDevelopmentMode {
       Disabled({})
+    } else {
+      switch getMetrics() {
+      | None => Initializing({})
+      | Some(metrics) =>
+        Active({
+          envioVersion,
+          chains: metrics.chains->Array.map(toChainData),
+          indexerStartTime: metrics.startTime,
+          isPreRegisteringDynamicContracts: false,
+          rollbackOnReorg: metrics.rollbackEnabled,
+        })
+      }
     }
 
     res->json(state->S.reverseConvertToJsonOrThrow(stateSchema))
@@ -543,10 +558,7 @@ let startServer = (~getState, ~persistence: Persistence.t, ~isDevelopmentMode: b
 
   app->get("/metrics", (_req, res) => {
     res->set("Content-Type", Metrics.contentType)
-    let _ =
-      res->endWithData(
-        Metrics.collect(~metrics=getIndexerState()->Option.map(IndexerState.toMetrics)),
-      )
+    let _ = res->endWithData(Metrics.collect(~metrics=getMetrics()))
   })
 
   app->get("/metrics/runtime", (_req, res) => {
@@ -648,6 +660,7 @@ let start = async (
     ~resetCommand=isDevelopmentMode ? "envio dev -r" : "envio start -r",
     ~runCommand=Some(isDevelopmentMode ? "envio dev" : "envio start"),
     ~lowercaseAddresses=config.lowercaseAddresses,
+    ~requireInitialized=config.isolated,
   )
 
   // Loads user handler files, which register handler/contractRegister/where
@@ -683,22 +696,10 @@ let start = async (
   }
   let envioVersion = Utils.EnvioPackage.value.version
 
+  let getMetrics = () => getIndexerState()->Option.map(IndexerState.toMetrics)
+
   if !isTest {
-    startServer(~persistence, ~isDevelopmentMode, ~getState=() =>
-      switch getIndexerState() {
-      | None => Initializing({})
-      | Some(state) => {
-          let chains = (state->IndexerState.toMetrics).chains->Array.map(toChainData)
-          Active({
-            envioVersion,
-            chains,
-            indexerStartTime: state->IndexerState.indexerStartTime,
-            isPreRegisteringDynamicContracts: false,
-            rollbackOnReorg: config.shouldRollbackOnReorg,
-          })
-        }
-      }
-    )
+    startServer(~persistence, ~isDevelopmentMode, ~envioVersion, ~getMetrics)
   }
 
   let state = IndexerState.makeFromDbState(
@@ -712,7 +713,7 @@ let start = async (
     ~onError,
   )
   if shouldUseTui {
-    let _rerender = Tui.start(~getState=() => state)
+    let _rerender = Tui.start(~config, ~getMetrics=() => state->IndexerState.toMetrics)
   }
   setIndexerState(state)
   state->IndexerLoop.start

package/src/Main.res.mjs

@@ -402,7 +402,7 @@ function getGlobalIndexer() {
   return new Proxy(Object.create(null), traps);
 }
 
-function startServer(getState, persistence, isDevelopmentMode) {
+function startServer(getMetrics, envioVersion, persistence, isDevelopmentMode) {
   let app = Express();
   let consoleCorsMiddleware = (req, res, next) => {
     let origin = req.headers["origin"];
@@ -425,9 +425,24 @@ function startServer(getState, persistence, isDevelopmentMode) {
     res.sendStatus(200);
   });
   app.get("/console/state", (_req, res) => {
-    let state = isDevelopmentMode ? getState() : ({
+    let state;
+    if (isDevelopmentMode) {
+      let metrics = getMetrics();
+      state = metrics !== undefined ? ({
+          status: "active",
+          envioVersion: envioVersion,
+          chains: metrics.chains.map(toChainData),
+          indexerStartTime: metrics.startTime,
+          isPreRegisteringDynamicContracts: false,
+          rollbackOnReorg: metrics.rollbackEnabled
+        }) : ({
+          status: "initializing"
+        });
+    } else {
+      state = {
         status: "disabled"
-      });
+      };
+    }
     res.json(S$RescriptSchema.reverseConvertToJsonOrThrow(state, stateSchema));
   });
   app.post("/console/syncCache", (_req, res) => {
@@ -442,7 +457,7 @@ function startServer(getState, persistence, isDevelopmentMode) {
   Metrics.startRuntimeCollectors();
   app.get("/metrics", (_req, res) => {
     res.set("Content-Type", Metrics.contentType);
-    res.end(Metrics.collect(Stdlib_Option.map(EnvioGlobal.value.indexerState, IndexerState.toMetrics)));
+    res.end(Metrics.collect(getMetrics()));
   });
   app.get("/metrics/runtime", (_req, res) => {
     res.set("Content-Type", Metrics.contentType);
@@ -467,7 +482,7 @@ function getEnvioInfo() {
 async function migrate(reset) {
   let config = Config.load();
   let persistence = PgStorage.makePersistenceFromConfig(config, undefined);
-  await Persistence.init(persistence, ChainMap.values(config.chainMap), config.contractMapping, Config.stripSensitiveData(Config.getPublicConfigJson()), "envio local db-migrate setup", undefined, reset, config.lowercaseAddresses, "Once");
+  await Persistence.init(persistence, ChainMap.values(config.chainMap), config.contractMapping, Config.stripSensitiveData(Config.getPublicConfigJson()), "envio local db-migrate setup", undefined, reset, config.lowercaseAddresses, undefined, "Once");
   return await persistence.storage.close();
 }
 
@@ -494,7 +509,7 @@ async function start(persistence, resetOpt, isTestOpt, exitAfterFirstEventBlockO
   let isDevelopmentMode = !isTest && config.isDev;
   let persistence$1 = persistence !== undefined ? persistence : PgStorage.makePersistenceFromConfig(config, undefined);
   EnvioGlobal.value.persistence = Primitive_option.some(persistence$1);
-  await Persistence.init(persistence$1, ChainMap.values(config.chainMap), config.contractMapping, Config.stripSensitiveData(Config.getPublicConfigJson()), isDevelopmentMode ? "envio dev -r" : "envio start -r", isDevelopmentMode ? "envio dev" : "envio start", reset, config.lowercaseAddresses, undefined);
+  await Persistence.init(persistence$1, ChainMap.values(config.chainMap), config.contractMapping, Config.stripSensitiveData(Config.getPublicConfigJson()), isDevelopmentMode ? "envio dev -r" : "envio start -r", isDevelopmentMode ? "envio dev" : "envio start", reset, config.lowercaseAddresses, config.isolated, undefined);
   let registrationsByChainId = await HandlerLoader.registerAllHandlers(config);
   let config$1;
   if (isTest) {
@@ -519,29 +534,13 @@ async function start(persistence, resetOpt, isTestOpt, exitAfterFirstEventBlockO
     });
   };
   let envioVersion = Utils.EnvioPackage.value.version;
+  let getMetrics = () => Stdlib_Option.map(EnvioGlobal.value.indexerState, IndexerState.toMetrics);
   if (!isTest) {
-    startServer(() => {
-      let state = EnvioGlobal.value.indexerState;
-      if (state === undefined) {
-        return {
-          status: "initializing"
-        };
-      }
-      let state$1 = Primitive_option.valFromOption(state);
-      let chains = IndexerState.toMetrics(state$1).chains.map(toChainData);
-      return {
-        status: "active",
-        envioVersion: envioVersion,
-        chains: chains,
-        indexerStartTime: IndexerState.indexerStartTime(state$1),
-        isPreRegisteringDynamicContracts: false,
-        rollbackOnReorg: config$2.shouldRollbackOnReorg
-      };
-    }, persistence$1, isDevelopmentMode);
+    startServer(getMetrics, envioVersion, persistence$1, isDevelopmentMode);
   }
   let state = IndexerState.makeFromDbState(config$2, persistence$1, Persistence.getInitializedState(persistence$1), registrationsByChainId, isDevelopmentMode, shouldUseTui, exitAfterFirstEventBlock, undefined, undefined, onError, undefined);
   if (shouldUseTui) {
-    Tui.start(() => state);
+    Tui.start(config$2, () => IndexerState.toMetrics(state));
   }
   EnvioGlobal.value.indexerState = Primitive_option.some(state);
   IndexerLoop.start(state);

package/src/Metrics.res

@@ -20,6 +20,7 @@ type chainMetrics = {
   // Raw committed progress (may be -1), unlike the optional latestProcessedBlock.
   progressBlockNumber: int,
   progressLatencyMs: option<int>,
+  progressBlockTime: option<int>,
   concurrency: int,
   partitionsCount: int,
   bufferSize: int,
@@ -36,8 +37,18 @@ type chainMetrics = {
   reorgCount: int,
   reorgDetectedBlock: option<int>,
   rollbackTargetBlock: option<int>,
+  rateLimitTimeMs: float,
+  rateLimitResetInMs: option<float>,
 }
 
+// Mirrors `ChainState.hasProcessedToEndblock`, which is what clamps
+// `knownHeight` to the end block — the two must agree.
+let hasProcessedToEndblock = (m: chainMetrics) =>
+  switch m.endBlock {
+  | Some(endBlock) => m.progressBlockNumber >= endBlock
+  | None => false
+  }
+
 type handlerMetrics = {
   contract: string,
   event: string,
@@ -90,6 +101,10 @@ type sourceRequestMetrics = {
   method: string,
   count: int,
   seconds: float,
+  // None for methods whose responses aren't measured in blocks; they render no
+  // block series at all.
+  responseBlocks: option<int>,
+  emptyResponseCount: int,
 }
 
 type sourceHeightMetrics = {
@@ -249,7 +264,16 @@ let renderMetrics = (b: builder, metrics: t) => {
       | Some(existing) =>
         byLabels->Dict.set(
           labels,
-          {...existing, count: existing.count + s.count, seconds: existing.seconds +. s.seconds},
+          {
+            ...existing,
+            count: existing.count + s.count,
+            seconds: existing.seconds +. s.seconds,
+            responseBlocks: switch (existing.responseBlocks, s.responseBlocks) {
+            | (Some(a), Some(b)) => Some(a + b)
+            | (some, None) | (None, some) => some
+            },
+            emptyResponseCount: existing.emptyResponseCount + s.emptyResponseCount,
+          },
         )
       | None => byLabels->Dict.set(labels, s)
       }
@@ -509,6 +533,23 @@ let renderMetrics = (b: builder, metrics: t) => {
     ~entries=sourceRequests,
     ~value=s => s.seconds !== 0. ? Some(s.seconds) : None,
   )
+  b->seriesOpt(
+    ~name="envio_source_response_blocks_total",
+    ~help="The number of blocks a data source returned, summed over its responses. Counted as the source sent them, before the indexer drops the blocks no event needs. Only sources whose responses are measured in blocks report it.",
+    ~kind="counter",
+    ~entries=sourceRequests,
+    ~value=s => s.responseBlocks->Option.map(Int.toFloat),
+  )
+  // Rendered flat at zero for any method that reports blocks: a chain scanning
+  // ranges it finds nothing in is the reading this exists for, and a series
+  // that only appears once the first empty response lands can't be alerted on.
+  b->seriesOpt(
+    ~name="envio_source_response_empty_total",
+    ~help="The number of responses that came back with no blocks at all — a range the source scanned and matched nothing in. Compare against envio_source_request_total for the share of requests that returned nothing.",
+    ~kind="counter",
+    ~entries=sourceRequests,
+    ~value=s => s.responseBlocks->Option.map(_ => s.emptyResponseCount->Int.toFloat),
+  )
   if heightStreamConnects->Array.length > 0 {
     b->series(
       ~name="envio_source_height_stream_connects_total",
@@ -619,6 +660,13 @@ let renderMetrics = (b: builder, metrics: t) => {
     ~entries=chains,
     ~value=m => m.numEventsProcessed,
   )
+  b->seriesOpt(
+    ~name="envio_progress_block_time_seconds",
+    ~help="Unix timestamp of the block the chain has processed up to. Subtract it from the scrape time for how far behind chain time the indexer is, which stays honest when the data source itself is behind the chain. Best effort in realtime mode, absent during backfill.",
+    ~kind="gauge",
+    ~entries=chains,
+    ~value=m => m.progressBlockTime->Option.map(Int.toFloat),
+  )
   b->seriesOpt(
     ~name="envio_progress_latency",
     ~help="The latency in milliseconds between the latest processed event creation and the time it was written to storage.",