envio 3.12.0 → 3.12.1

package/src/db/Table.res

@@ -331,18 +331,31 @@ let encodeIdsToJson = (table, ids: array<EntityId.t>): JSON.t =>
 
 // TODO: Test whether it should be passed via args and match the column type
 
-let getFieldByApiName = (table, apiFieldName) =>
-  table.fields->Array.find(field =>
-    switch field {
-    | Field(f) => f->getApiFieldName
-    | DerivedFrom({fieldName}) => fieldName
-    } === apiFieldName
+// Resolved once per table, because a getWhere looks up every field it filters
+// on and scanning the array made that cost grow with the entity's width.
+let fieldsByApiName: table => dict<fieldOrDerived> = Utils.WeakMap.memoize(table => {
+  let byApiName = Dict.make()
+  table.fields->Array.forEach(field =>
+    byApiName->Dict.set(
+      switch field {
+      | Field(f) => f->getApiFieldName
+      | DerivedFrom({fieldName}) => fieldName
+      },
+      field,
+    )
   )
+  byApiName
+})
+
+let getFieldByApiName = (table, apiFieldName) =>
+  table->fieldsByApiName->Utils.Dict.dangerouslyGetNonOption(apiFieldName)
 
 // Both schema instances are created once per field: rescript-schema compiles
 // and caches operations on the schema instance, so building S.array(fieldSchema)
 // per query would recompile the serializer on every call.
 type queryField = {
+  fieldType: fieldType,
+  isArray: bool,
   fieldSchema: S.t<unknown>,
   // Serializes the values array of an "in" filter
   arrayFieldSchema: S.t<unknown>,
@@ -363,6 +376,8 @@ let queryFields: table => dict<queryField> = Utils.WeakMap.memoize(table => {
       dict->Dict.set(
         field->getApiFieldName,
         {
+          fieldType: field.fieldType,
+          isArray: field.isArray,
           fieldSchema: field.fieldSchema,
           arrayFieldSchema: switch field.fieldType {
           | Bytea => Utils.Schema.bytesArray->S.toUnknown

package/src/db/Table.res.mjs

@@ -296,12 +296,18 @@ function encodeIdsToJson(table, ids) {
   return S$RescriptSchema.reverseConvertToJsonOrThrow(ids, idsArraySchema(table));
 }
 
-function getFieldByApiName(table, apiFieldName) {
-  return table.fields.find(field => {
+let fieldsByApiName = Utils.$$WeakMap.memoize(table => {
+  let byApiName = {};
+  table.fields.forEach(field => {
     let tmp;
     tmp = field.TAG === "Field" ? getApiFieldName(field._0) : field._0.fieldName;
-    return tmp === apiFieldName;
+    byApiName[tmp] = field;
   });
+  return byApiName;
+});
+
+function getFieldByApiName(table, apiFieldName) {
+  return fieldsByApiName(table)[apiFieldName];
 }
 
 let queryFields = Utils.$$WeakMap.memoize(table => {
@@ -315,6 +321,8 @@ let queryFields = Utils.$$WeakMap.memoize(table => {
     let tmp;
     tmp = typeof match !== "object" && match === "Bytea" ? Utils.Schema.bytesArray : S$RescriptSchema.array(field$1.fieldSchema);
     dict[getApiFieldName(field$1)] = {
+      fieldType: field$1.fieldType,
+      isArray: field$1.isArray,
       fieldSchema: field$1.fieldSchema,
       arrayFieldSchema: tmp,
       pgDbFieldName: getPgDbFieldName(field$1),
@@ -420,7 +428,7 @@ function toSqlParams(table, schema, pgSchema, chainIdModeOpt) {
           }
         }
       };
-      let field = getFieldByApiName(table, location);
+      let field = fieldsByApiName(table)[location];
       let field$1;
       if (field !== undefined) {
         field$1 = field;
@@ -533,6 +541,7 @@ export {
   getIdSchema,
   idsArraySchema,
   encodeIdsToJson,
+  fieldsByApiName,
   getFieldByApiName,
   queryFields,
   makeRowsSchema,

package/src/sources/BlockStore.res

@@ -142,8 +142,13 @@ external materialize: (
 // Hash of a stored block, if the store still holds it.
 @send external getHash: (t, int) => Null.t<string> = "getHash"
 
-// Unix timestamp of a stored block - exactly that block, on every ecosystem.
-@send external getTimestamp: (t, int) => Null.t<int> = "getTimestamp"
+// Unix timestamp of a stored block - exactly that block, except for an SVM slot
+// that produced none: with `allowSkippedSlot` the last real slot below it
+// answers instead, which is what chain time is at a skipped slot. Only set it
+// where the query covered every slot in its range, or a slot that was never
+// asked about reads as one the chain skipped.
+@send
+external getTimestamp: (t, int, ~allowSkippedSlot: bool) => Null.t<int> = "getTimestamp"
 
 // Block numbers in `[fromBlock, belowBlock)` with a stored hash, ascending.
 @send

package/src/sources/EvmHyperSyncSource.res

@@ -84,6 +84,7 @@ let make = (
     ~fromBlock,
     ~toBlock,
     ~addressSet,
+    ~includeAllBlocks,
     ~knownHeight,
     ~partitionId as _,
     ~selection: FetchState.selection,
@@ -104,6 +105,7 @@ let make = (
       ~registrationIndexes=selection.onEventRegistrations->Array.map(reg => reg.index),
       ~addressSet,
       ~clientFilteredContracts=selection.clientFilteredContracts,
+      ~includeAllBlocks,
     ) catch {
     | HyperSync.GetLogs.Error(WrongInstance) =>
       throw(Source.SourceBehindHead({blockNumber: fromBlock, requestStats: []}))

package/src/sources/EvmHyperSyncSource.res.mjs

@@ -44,12 +44,12 @@ function make(param) {
       }
     };
   };
-  let getItemsOrThrow = async (fromBlock, toBlock, addressSet, knownHeight, param, selection, itemsTarget, retry, param$1) => {
+  let getItemsOrThrow = async (fromBlock, toBlock, addressSet, includeAllBlocks, knownHeight, param, selection, itemsTarget, retry, param$1) => {
     let totalTimeRef = Performance.now();
     let startFetchingBatchTimeRef = Performance.now();
     let pageUnsafe;
     try {
-      pageUnsafe = await HyperSync.GetLogs.query(client, fromBlock, toBlock, itemsTarget, selection.onEventRegistrations.map(reg => reg.index), addressSet, selection.clientFilteredContracts);
+      pageUnsafe = await HyperSync.GetLogs.query(client, fromBlock, toBlock, itemsTarget, selection.onEventRegistrations.map(reg => reg.index), addressSet, selection.clientFilteredContracts, includeAllBlocks);
     } catch (raw_exn) {
       let exn = Primitive_exceptions.internalToException(raw_exn);
       if (exn.RE_EXN_ID === HyperSync.GetLogs.$$Error) {

package/src/sources/FuelHyperSyncSource.res

@@ -35,6 +35,7 @@ let make = ({chainId, endpointUrl, apiToken, onEventRegistrations, addressStore}
     ~fromBlock,
     ~toBlock,
     ~addressSet,
+    ~includeAllBlocks as _,
     ~knownHeight,
     ~partitionId as _,
     ~selection: FetchState.selection,

package/src/sources/FuelHyperSyncSource.res.mjs

@@ -28,7 +28,7 @@ function make(param) {
     let exn = Primitive_exceptions.internalToException(raw_exn);
     client = ErrorHandling.mkLogAndRaise(undefined, "Failed to instantiate the HyperFuel client", exn);
   }
-  let getItemsOrThrow = async (fromBlock, toBlock, addressSet, knownHeight, param, selection, param$1, retry, logger) => {
+  let getItemsOrThrow = async (fromBlock, toBlock, addressSet, param, knownHeight, param$1, selection, param$2, retry, logger) => {
     let totalTimeRef = Performance.now();
     let startFetchingBatchTimeRef = Performance.now();
     let pageUnsafe;

package/src/sources/HyperSync.res

@@ -137,6 +137,7 @@ module GetLogs = {
     ~registrationIndexes,
     ~addressSet,
     ~clientFilteredContracts,
+    ~includeAllBlocks,
   ): logsQueryPage => {
     let query: HyperSyncClient.EventItems.query = {
       fromBlock,
@@ -144,6 +145,9 @@ module GetLogs = {
       ?maxNumLogs,
       registrationIndexes,
       clientFilteredContracts,
+      // Absent rather than false, so a range that wants only the blocks its
+      // logs came from sends the query it always sent.
+      includeAllBlocks: ?(includeAllBlocks ? Some(true) : None),
     }
 
     let (res, transactionStore, blockStore) = switch await client.getEventItems(

package/src/sources/HyperSync.res.mjs

@@ -137,13 +137,15 @@ function extractMissingParams(exn) {
   }
 }
 
-async function query(client, fromBlock, toBlock, maxNumLogs, registrationIndexes, addressSet, clientFilteredContracts) {
+async function query(client, fromBlock, toBlock, maxNumLogs, registrationIndexes, addressSet, clientFilteredContracts, includeAllBlocks) {
+  let query_includeAllBlocks = includeAllBlocks ? true : undefined;
   let query$1 = {
     fromBlock: fromBlock,
     toBlock: toBlock,
     maxNumLogs: maxNumLogs,
     registrationIndexes: registrationIndexes,
-    clientFilteredContracts: clientFilteredContracts
+    clientFilteredContracts: clientFilteredContracts,
+    includeAllBlocks: query_includeAllBlocks
   };
   let match;
   try {

package/src/sources/HyperSync.resi

@@ -57,5 +57,6 @@ module GetLogs: {
     ~registrationIndexes: array<int>,
     ~addressSet: AddressSet.t,
     ~clientFilteredContracts: option<array<string>>,
+    ~includeAllBlocks: bool,
   ) => promise<logsQueryPage>
 }

package/src/sources/HyperSyncClient.res

@@ -262,6 +262,9 @@ module EventItems = {
     // depend on addresses (client-side filtering). None/empty means
     // every address-dependent contract is filtered server-side.
     clientFilteredContracts: option<array<string>>,
+    // Return a header for every block in the range, not only the ones a log
+    // landed on. Absent means only the blocks logs came from.
+    includeAllBlocks?: bool,
   }
 
   type item = {

package/src/sources/HyperSyncSSE.res

@@ -25,7 +25,7 @@ let subscribe = (~hyperSyncUrl, ~apiToken, ~onHeight, ~onStatus) =>
             args.headers
             ->Option.getOr(Dict.make())
             ->Utils.Dict.merge(
-              Dict.fromArray([("Authorization", `Bearer ${apiToken}`), ("User-Agent", userAgent)]),
+              dict{"Authorization": `Bearer ${apiToken}`, "User-Agent": userAgent},
             )
           EventSource.Fetch.fetch(url, ~args={...args, headers: headers})
         },

package/src/sources/HyperSyncSSE.res.mjs

@@ -35,16 +35,10 @@ function subscribe(hyperSyncUrl, apiToken, onHeight, onStatus) {
     let userAgent = `hyperindex/` + Utils.EnvioPackage.value.version;
     let es = new Eventsource.EventSource(hyperSyncUrl + `/height/sse`, {
       fetch: (url, args) => {
-        let headers = Utils.Dict.merge(Stdlib_Option.getOr(args.headers, {}), Object.fromEntries([
-          [
-            "Authorization",
-            `Bearer ` + apiToken
-          ],
-          [
-            "User-Agent",
-            userAgent
-          ]
-        ]));
+        let headers = Utils.Dict.merge(Stdlib_Option.getOr(args.headers, {}), {
+          Authorization: `Bearer ` + apiToken,
+          "User-Agent": userAgent
+        });
         let newrecord = {...args};
         return fetch(url, (newrecord.headers = headers, newrecord));
       }

package/src/sources/RpcSource.res

@@ -877,6 +877,7 @@ let make = (
     ~fromBlock,
     ~toBlock,
     ~addressSet,
+    ~includeAllBlocks as _,
     ~knownHeight,
     ~partitionId,
     ~selection: FetchState.selection,

package/src/sources/RpcSource.res.mjs

@@ -874,7 +874,7 @@ function make(param) {
       Error: new Error()
     };
   }, lowercaseAddresses);
-  let getItemsOrThrow = async (fromBlock, toBlock, addressSet, knownHeight, partitionId, selection, param, retry, param$1) => {
+  let getItemsOrThrow = async (fromBlock, toBlock, addressSet, param, knownHeight, partitionId, selection, param$1, retry, param$2) => {
     let startFetchingBatchTimeRef = Performance.now();
     let toBlock$1 = toBlock !== undefined ? Primitive_int.min(toBlock, knownHeight) : knownHeight;
     let firstBlockPromise = fromBlock > 0 && fromBlock <= toBlock$1 ? LazyLoader.get(blockLoader.contents, fromBlock).then(json => parseBlockInfo(json)) : Promise.resolve(undefined);

package/src/sources/SimulateSource.res

@@ -29,6 +29,7 @@ let make = (
       ~fromBlock,
       ~toBlock,
       ~addressSet,
+      ~includeAllBlocks as _,
       ~knownHeight as _,
       ~partitionId as _,
       ~selection: FetchState.selection,

package/src/sources/SimulateSource.res.mjs

@@ -27,7 +27,7 @@ function make(items, endBlock, chainId, addressStore, ecosystemOpt, transactionS
       height: reportedHeight,
       requestStats: []
     }),
-    getItemsOrThrow: (fromBlock, toBlock, addressSet, param, param$1, selection, param$2, param$3, param$4) => {
+    getItemsOrThrow: (fromBlock, toBlock, addressSet, param, param$1, param$2, selection, param$3, param$4, param$5) => {
       let toBlockQueried = toBlock !== undefined ? toBlock : reportedHeight;
       let selectionRegistrationIndexes = new Set();
       selection.onEventRegistrations.forEach(reg => {

package/src/sources/Source.res

@@ -197,6 +197,13 @@ type t = {
     // straight to its Rust client, which builds the query's address filter from
     // it and gates every returned item against the chain-wide store.
     ~addressSet: AddressSet.t,
+    // Return a header for every block in the range, not only the ones an item
+    // landed on. The progress block's timestamp is what measures how far behind
+    // chain time the indexer is, and at the head that block often carries no
+    // item of its own. Only set once the chain is at the head, where the range
+    // is a handful of blocks; over a backfill range it would be a header per
+    // block for no gain.
+    ~includeAllBlocks: bool,
     ~knownHeight: int,
     ~partitionId: string,
     ~selection: FetchState.selection,

package/src/sources/SourceManager.res

@@ -705,7 +705,7 @@ let waitForNewBlock = (sourceManager: t, ~knownHeight, ~isRealtime, ~reducedPoll
     logger->Logging.childTrace(
       reducedPolling
         ? `Waiting for new blocks with reduced polling (${(sourceManager.reducedPollingInterval / 1000)
-              ->Int.toString}s). Chain is caught up, waiting for other chains to backfill.`
+              ->Int.toString}s) until the indexer enters realtime mode.`
         : "Initiating check for new blocks.",
     )
     sourceManager.waitingLogged = true
@@ -717,8 +717,8 @@ let waitForNewBlock = (sourceManager: t, ~knownHeight, ~isRealtime, ~reducedPoll
   // cadence the sources poll at and the level the closing line is logged at.
   let stalled = ref(false)
 
-  // Use a much longer stall timeout when reduced polling is active
-  // to avoid spurious stall warnings while waiting for other chains to backfill
+  // Use a much longer stall timeout when reduced polling is active, so a chain
+  // deliberately asking rarely doesn't report itself stalled between polls.
   let stallTimeout = if reducedPolling {
     sourceManager.reducedPollingInterval * 2
   } else if isRealtime {
@@ -948,6 +948,7 @@ let executeQuery = async (
         ~fromBlock=query.fromBlock,
         ~toBlock,
         ~addressSet=query.addresses,
+        ~includeAllBlocks=isRealtime,
         ~partitionId=query.partitionId,
         ~knownHeight,
         ~selection=query.selection->FetchState.narrowSelectionToRange(~toBlock),

package/src/sources/SourceManager.res.mjs

@@ -537,7 +537,7 @@ function waitForNewBlock(sourceManager, knownHeight, isRealtime, reducedPolling)
     knownHeight: knownHeight
   });
   if (!sourceManager.waitingLogged) {
-    Logging.childTrace(logger, reducedPolling ? `Waiting for new blocks with reduced polling (` + (sourceManager.reducedPollingInterval / 1000 | 0).toString() + `s). Chain is caught up, waiting for other chains to backfill.` : "Initiating check for new blocks.");
+    Logging.childTrace(logger, reducedPolling ? `Waiting for new blocks with reduced polling (` + (sourceManager.reducedPollingInterval / 1000 | 0).toString() + `s) until the indexer enters realtime mode.` : "Initiating check for new blocks.");
     sourceManager.waitingLogged = true;
   }
   let mainSources = getNextSources(sourceManager, isRealtime, undefined);
@@ -691,7 +691,7 @@ async function executeQuery(sourceManager, query, knownHeight, isRealtime) {
       retry: retry
     });
     try {
-      let response = await source.getItemsOrThrow(query.fromBlock, toBlock, query.addresses, knownHeight, query.partitionId, FetchState.narrowSelectionToRange(query.selection, toBlock), query.itemsTarget, retry, logger$2);
+      let response = await source.getItemsOrThrow(query.fromBlock, toBlock, query.addresses, isRealtime, knownHeight, query.partitionId, FetchState.narrowSelectionToRange(query.selection, toBlock), query.itemsTarget, retry, logger$2);
       recordStatsInto(sourceState.requestStats, response.requestStats);
       validateResponseBlockStore("getItems", response.blockStore, undefined);
       sourceState.lastFailedAt = undefined;

package/src/sources/SvmHyperSyncClient.res

@@ -143,6 +143,11 @@ module EventItems = {
     // depend on addresses (client-side filtering). None/empty means every
     // address-dependent program is filtered server-side.
     clientFilteredContracts: option<array<string>>,
+    // Return a block for every slot in the range, not only the ones an
+    // instruction landed on. Absent means only the slots instructions came
+    // from, which leaves a skipped slot indistinguishable from an unfetched
+    // one.
+    includeAllBlocks?: bool,
   }
 
   // NAPI encodes Rust `None` as `null`, never `undefined`, so an unselected

package/src/sources/SvmHyperSyncSource.res

@@ -147,6 +147,7 @@ let make = (
     ~fromBlock,
     ~toBlock,
     ~addressSet,
+    ~includeAllBlocks,
     ~knownHeight,
     ~partitionId as _,
     ~selection: FetchState.selection,
@@ -163,6 +164,8 @@ let make = (
       maxNumInstructions: ?itemsTarget,
       registrationIndexes: selection.onEventRegistrations->Array.map(reg => reg.index),
       clientFilteredContracts: selection.clientFilteredContracts,
+      // Absent rather than false, so a backfill query is the one it always was.
+      includeAllBlocks: ?(includeAllBlocks ? Some(true) : None),
     }
 
     let (resp, transactionStore, blockStore) = try await client.getEventItems(

package/src/sources/SvmHyperSyncSource.res.mjs

@@ -106,17 +106,19 @@ function make(param) {
   let chainId = param.chainId;
   let apiToken = HyperSync.requireApiToken(param.apiToken);
   let client = SvmHyperSyncClient.make(endpointUrl, apiToken, param.clientTimeoutMillis, undefined, undefined, SvmHyperSyncClient.Registration.fromOnEventRegistrations(onEventRegistrations), param.addressStore);
-  let getItemsOrThrow = async (fromBlock, toBlock, addressSet, knownHeight, param, selection, itemsTarget, retry, param$1) => {
+  let getItemsOrThrow = async (fromBlock, toBlock, addressSet, includeAllBlocks, knownHeight, param, selection, itemsTarget, retry, param$1) => {
     let totalTimeRef = Performance.now();
     let pageFetchRef = Performance.now();
     let query_registrationIndexes = selection.onEventRegistrations.map(reg => reg.index);
     let query_clientFilteredContracts = selection.clientFilteredContracts;
+    let query_includeAllBlocks = includeAllBlocks ? true : undefined;
     let query = {
       fromSlot: fromBlock,
       toSlot: toBlock,
       maxNumInstructions: itemsTarget,
       registrationIndexes: query_registrationIndexes,
-      clientFilteredContracts: query_clientFilteredContracts
+      clientFilteredContracts: query_clientFilteredContracts,
+      includeAllBlocks: query_includeAllBlocks
     };
     let match;
     try {