envio 3.5.0-alpha.1 → 3.5.0-alpha.3

package/src/ChainState.resi

@@ -9,7 +9,7 @@ let make: (
   ~chainConfig: Config.chain,
   ~fetchState: FetchState.t,
   ~onEventRegistrations: array<Internal.onEventRegistration>=?,
-  ~indexingAddresses: IndexingAddresses.t,
+  ~addressStore: AddressStore.t,
   ~sourceManager: SourceManager.t,
   ~reorgDetection: ReorgDetection.t,
   ~committedProgressBlockNumber: int,
@@ -104,10 +104,6 @@ let isReady: t => bool
 let isFetchingAtHead: t => bool
 let isAtHeadWithoutEndBlock: t => bool
 
-// Drop over-fetched events an address-param filter rejects, before contract
-// registration runs on them. Uses the chain's current indexing addresses.
-let filterByClientAddress: (t, array<Internal.item>) => array<Internal.item>
-
 // State transitions. The chain state is mutated only through these.
 let handleQueryResult: (
   t,
@@ -141,7 +137,7 @@ let setEndBlockToFirstEvent: (t, ~blockNumber: int) => unit
 let enterReorgThreshold: t => unit
 let advanceAfterBatch: (t, ~batch: Batch.t, ~enteringReorgThreshold: bool) => unit
 let applyBatchProgress: (t, ~batch: Batch.t, ~blockTimestampName: string) => unit
-let markReady: t => unit
+let markReady: (t, ~readyAt: Date.t) => unit
 let rollback: (
   t,
   ~newProgressBlockNumber: option<int>,

package/src/Change.res

@@ -1,9 +1,9 @@
 @tag("type")
 type t<'entity> =
-  | @as("SET") Set({entityId: string, entity: 'entity, checkpointId: bigint})
-  | @as("DELETE") Delete({entityId: string, checkpointId: bigint})
+  | @as("SET") Set({entityId: EntityId.t, entity: 'entity, checkpointId: bigint})
+  | @as("DELETE") Delete({entityId: EntityId.t, checkpointId: bigint})
 
 @get
-external getEntityId: t<'entity> => string = "entityId"
+external getEntityId: t<'entity> => EntityId.t = "entityId"
 @get
 external getCheckpointId: t<'entity> => bigint = "checkpointId"

package/src/Config.res

@@ -390,10 +390,6 @@ let getFieldTypeAndSchema = (prop, ~enumConfigsByName: dict<Table.enumConfig<Tab
       let enumConfig = enumConfigsByName->Dict.get(enumName)->Option.getOrThrow
       (Table.Enum({config: enumConfig}), enumConfig.schema->S.toUnknown)
     }
-  | "entity" => {
-      let entityName = prop["entity"]->Option.getOrThrow
-      (Table.Entity({name: entityName}), S.string->S.toUnknown)
-    }
   | other => JsError.throwWithMessage("Unknown field type in entity config: " ++ other)
   }
 
@@ -459,7 +455,7 @@ let parseEntitiesFromJson = (
         )
       )
 
-    let compositeIndices =
+    let compositeIndexes =
       entityJson["compositeIndices"]
       ->Option.getOr([])
       ->Array.map(ci =>
@@ -474,7 +470,7 @@ let parseEntitiesFromJson = (
     let table = Table.mkTable(
       entityName,
       ~fields=Array.concat(fields, derivedFields),
-      ~compositeIndices,
+      ~compositeIndexes,
       ~description=?entityJson["description"],
     )
 

package/src/Config.res.mjs

@@ -263,16 +263,6 @@ function getFieldTypeAndSchema(prop, enumConfigsByName) {
         Utils.Schema.dbDate
       ];
       break;
-    case "entity" :
-      let entityName = Stdlib_Option.getOrThrow(prop.entity, undefined);
-      match = [
-        {
-          type: "Entity",
-          name: entityName
-        },
-        S$RescriptSchema.string
-      ];
-      break;
     case "enum" :
       let enumName = Stdlib_Option.getOrThrow(prop.enum, undefined);
       let enumConfig = Stdlib_Option.getOrThrow(enumConfigsByName[enumName], undefined);
@@ -341,11 +331,11 @@ function parseEntitiesFromJson(entitiesJson, enumConfigsByName, globalStorage) {
       return Table.mkField(prop.name, match[0], match[1], undefined, match[3], match[2], prop.name === "id", match[4], prop.linkedEntity, prop.description, prop.postgresDbName, prop.clickhouseDbName);
     });
     let derivedFields = Stdlib_Option.getOr(entityJson.derivedFields, []).map(df => Table.mkDerivedFromField(df.fieldName, df.derivedFromEntity, df.derivedFromField, df.description));
-    let compositeIndices = Stdlib_Option.getOr(entityJson.compositeIndices, []).map(ci => ci.map(f => ({
+    let compositeIndexes = Stdlib_Option.getOr(entityJson.compositeIndices, []).map(ci => ci.map(f => ({
       fieldName: f.fieldName,
       direction: f.direction === "Asc" ? "Asc" : "Desc"
     })));
-    let table = Table.mkTable(entityName, compositeIndices, fields.concat(derivedFields), entityJson.description);
+    let table = Table.mkTable(entityName, compositeIndexes, fields.concat(derivedFields), entityJson.description);
     let getApiFieldName = prop => {
       let match = prop.linkedEntity;
       if (match !== undefined) {

package/src/Core.res

@@ -10,6 +10,7 @@ type svmHyperSyncClientCtor
 type fuelHyperSyncClientCtor
 type transactionStoreCtor
 type blockStoreCtor
+type addressStoreCtor
 type fromUserApiOptions = {
   schema?: string,
   env?: dict<string>,
@@ -39,6 +40,8 @@ type addon = {
   transactionStore: transactionStoreCtor,
   @as("BlockStore")
   blockStore: blockStoreCtor,
+  @as("AddressStore")
+  addressStore: addressStoreCtor,
   // Ordered transaction-field names exposed for the field-code contract test
   // (the ReScript `transactionFields` arrays must match the Rust ordinals).
   evmTransactionFieldNames: unit => array<string>,

package/src/CrossChainState.res

@@ -10,6 +10,10 @@ type t = {
   chainIds: array<int>,
   // True once every chain has caught up to head/endBlock. Monotonic during a run.
   mutable isRealtime: bool,
+  // True once every chain has caught up and there's nothing left to process,
+  // but before the deferred schema indexes and `ready_at` are committed. The
+  // gap between this and `isRealtime` is the FinalizingIndexes phase.
+  mutable isCaughtUp: bool,
   mutable isInReorgThreshold: bool,
   // Indexer-wide fetch buffer pool (item count), shared across all chains.
   targetBufferSize: int,
@@ -32,6 +36,7 @@ let make = (
     chainStates,
     chainIds: chainStates->Dict.valuesToArray->Array.map(cs => (cs->ChainState.chainConfig).id),
     isRealtime,
+    isCaughtUp: isRealtime,
     isInReorgThreshold,
     targetBufferSize,
   }
@@ -46,6 +51,7 @@ let getChainState = (crossChainState: t, chainId) =>
 
 let chainStates = (crossChainState: t) => crossChainState.chainStates
 let isRealtime = (crossChainState: t) => crossChainState.isRealtime
+let isCaughtUp = (crossChainState: t) => crossChainState.isCaughtUp
 let isInReorgThreshold = (crossChainState: t) => crossChainState.isInReorgThreshold
 let targetBufferSize = (crossChainState: t) => crossChainState.targetBufferSize
 
@@ -129,10 +135,11 @@ let enterReorgThreshold = (crossChainState: t) => {
   crossChainState.isInReorgThreshold = true
 }
 
-// Commit each progressed chain's batch progress, then decide readiness for the
-// whole indexer. A chain is marked caught up only once EVERY chain is caught up
-// (reached endblock or fetched/processed to head) with no processable events
-// left — so no chain flips to ready while another is still backfilling.
+// Commit each progressed chain's batch progress, then record whether the whole
+// indexer has caught up (every chain reached endblock or processed to head,
+// with no processable events left). Catching up doesn't make the indexer ready:
+// the deferred schema indexes still have to be created, which `markReady`
+// below concludes once they're committed.
 let applyBatchProgress = (crossChainState: t, ~batch: Batch.t, ~blockTimestampName: string) => {
   let chainIds = crossChainState.chainIds
 
@@ -145,15 +152,14 @@ let applyBatchProgress = (crossChainState: t, ~batch: Batch.t, ~blockTimestampNa
     }
   }
 
-  let indexerCaughtUp = crossChainState->nextItemIsNone && everyChainCaughtUp.contents
+  crossChainState.isCaughtUp =
+    crossChainState.isCaughtUp || (crossChainState->nextItemIsNone && everyChainCaughtUp.contents)
 
+  // A run resumed with every chain already stamped ready needs no finalize —
+  // the indexes were committed together with those stamps.
   let allChainsReady = ref(true)
   for i in 0 to chainIds->Array.length - 1 {
-    let cs = crossChainState->getChainState(chainIds->Array.getUnsafe(i))
-    if indexerCaughtUp {
-      cs->ChainState.markReady
-    }
-    if !(cs->ChainState.isReady) {
+    if !(crossChainState->getChainState(chainIds->Array.getUnsafe(i))->ChainState.isReady) {
       allChainsReady := false
     }
   }
@@ -161,6 +167,23 @@ let applyBatchProgress = (crossChainState: t, ~batch: Batch.t, ~blockTimestampNa
   crossChainState.isRealtime = crossChainState.isRealtime || allChainsReady.contents
 }
 
+// Enter the FinalizingIndexes phase without a batch. Used when the loop finds
+// every chain already processed to its endblock — a resume that has nothing
+// left to do still owes the schema its deferred indexes before it reports ready.
+let markCaughtUp = (crossChainState: t) => crossChainState.isCaughtUp = true
+
+// Concludes the FinalizingIndexes phase: stamps every chain with the `ready_at`
+// already committed alongside the deferred schema indexes and switches the
+// indexer to realtime.
+let markReady = (crossChainState: t, ~readyAt) => {
+  for i in 0 to crossChainState.chainIds->Array.length - 1 {
+    crossChainState
+    ->getChainState(crossChainState.chainIds->Array.getUnsafe(i))
+    ->ChainState.markReady(~readyAt)
+  }
+  crossChainState.isRealtime = true
+}
+
 // --- Fetch control. ---
 
 // Chains ordered furthest-behind first by fetch-frontier progress, so the

package/src/CrossChainState.res.mjs

@@ -27,6 +27,7 @@ function make(chainStates, isInReorgThreshold, isRealtime, targetBufferSizeOpt)
     chainStates: chainStates,
     chainIds: Object.values(chainStates).map(cs => ChainState.chainConfig(cs).id),
     isRealtime: isRealtime,
+    isCaughtUp: isRealtime,
     isInReorgThreshold: isInReorgThreshold,
     targetBufferSize: targetBufferSize
   };
@@ -40,6 +41,10 @@ function isRealtime(crossChainState) {
   return crossChainState.isRealtime;
 }
 
+function isCaughtUp(crossChainState) {
+  return crossChainState.isCaughtUp;
+}
+
 function isInReorgThreshold(crossChainState) {
   return crossChainState.isInReorgThreshold;
 }
@@ -104,20 +109,27 @@ function applyBatchProgress(crossChainState, batch, blockTimestampName) {
       everyChainCaughtUp = false;
     }
   }
-  let indexerCaughtUp = nextItemIsNone(crossChainState) && everyChainCaughtUp;
+  crossChainState.isCaughtUp = crossChainState.isCaughtUp || nextItemIsNone(crossChainState) && everyChainCaughtUp;
   let allChainsReady = true;
   for (let i$1 = 0, i_finish$1 = chainIds.length; i$1 < i_finish$1; ++i$1) {
-    let cs$1 = crossChainState.chainStates[chainIds[i$1]];
-    if (indexerCaughtUp) {
-      ChainState.markReady(cs$1);
-    }
-    if (!ChainState.isReady(cs$1)) {
+    if (!ChainState.isReady(crossChainState.chainStates[chainIds[i$1]])) {
       allChainsReady = false;
     }
   }
   crossChainState.isRealtime = crossChainState.isRealtime || allChainsReady;
 }
 
+function markCaughtUp(crossChainState) {
+  crossChainState.isCaughtUp = true;
+}
+
+function markReady(crossChainState, readyAt) {
+  for (let i = 0, i_finish = crossChainState.chainIds.length; i < i_finish; ++i) {
+    ChainState.markReady(crossChainState.chainStates[crossChainState.chainIds[i]], readyAt);
+  }
+  crossChainState.isRealtime = true;
+}
+
 function priorityOrder(crossChainState) {
   return Object.values(crossChainState.chainStates).toSorted((a, b) => Primitive_float.compare(ChainState.frontierProgress(a), ChainState.frontierProgress(b)));
 }
@@ -222,6 +234,7 @@ export {
   make,
   chainStates,
   isRealtime,
+  isCaughtUp,
   isInReorgThreshold,
   targetBufferSize,
   nextItemIsNone,
@@ -229,6 +242,8 @@ export {
   createBatch,
   enterReorgThreshold,
   applyBatchProgress,
+  markCaughtUp,
+  markReady,
   priorityOrder,
   checkAndFetch,
 }

package/src/CrossChainState.resi

@@ -15,6 +15,7 @@ let make: (
 // Accessors.
 let chainStates: t => dict<ChainState.t>
 let isRealtime: t => bool
+let isCaughtUp: t => bool
 let isInReorgThreshold: t => bool
 let targetBufferSize: t => int
 
@@ -31,6 +32,8 @@ let createBatch: (
 ) => Batch.t
 let enterReorgThreshold: t => unit
 let applyBatchProgress: (t, ~batch: Batch.t, ~blockTimestampName: string) => unit
+let markCaughtUp: t => unit
+let markReady: (t, ~readyAt: Date.t) => unit
 
 // Fetch control.
 let priorityOrder: t => array<ChainState.t>

package/src/EntityId.res

@@ -0,0 +1,15 @@
+// Opaque representation of an entity id. The generated per-entity types expose
+// the real scalar (string / int / bigint); the generic storage and in-memory
+// layers work with any entity's id through this type without knowing which
+// scalar backs it. The runtime value is always the real id, never a stringified
+// form — `toKey` derives the string only where a JS object/dict key is needed.
+type t
+
+external unsafeOfAny: 'a => t = "%identity"
+external unsafeToAny: t => 'a = "%identity"
+external unsafeOfString: string => t = "%identity"
+
+// Stringified id used as a JS object/dict key. `String` matches how JS coerces
+// a value used as an object key, so an id indexes the same whether the raw
+// value or its key form is used for lookup, across string/int/bigint.
+let toKey: t => string = %raw(`String`)

package/src/EntityId.res.mjs

@@ -0,0 +1,9 @@
+// Generated by ReScript, PLEASE EDIT WITH CARE
+
+
+let toKey = String;
+
+export {
+  toKey,
+}
+/* toKey Not a pure module */

package/src/EventConfigBuilder.res

@@ -292,61 +292,6 @@ let resolveFieldSelection = (
   )
 }
 
-// ============== Client-side address filter ==============
-
-let compileAddressFilter: string => (
-  Internal.eventPayload,
-  int,
-  dict<Internal.indexingContract>,
-) => bool = %raw(`function (body) {
-  return new Function("event", "blockNumber", "indexingAddresses", body);
-}`)
-
-// Body of the client-side address filter. Two analogous registered-at-or-before
-// checks, ANDed: (1) for non-wildcard events, the log's srcAddress must itself be
-// registered (ownership is resolved structurally by partition, but the temporal
-// `effectiveStartBlock` gate lives here now); (2) a DNF of address-filtered param
-// names (OR of AND-groups) for events that filter an indexed address param. The
-// DNF is fixed here, so it's unrolled into one boolean expression — no per-event
-// closure, loop, or array. `None` only for wildcard events without a param
-// filter. Exposed for snapshotting.
-// `srcAddressExpr` is the JS expression for the event's owning address: EVM and
-// Fuel events expose `event.srcAddress`; SVM instructions expose `event.programId`.
-let buildAddressFilterBody = (
-  groups: array<array<string>>,
-  ~isWildcard: bool,
-  ~srcAddressExpr: string="event.srcAddress",
-): option<string> => {
-  let paramLeaf = name =>
-    `(ic = indexingAddresses[p[${JSON.stringify(
-        JSON.String(name),
-      )}]]) !== undefined && ic.effectiveStartBlock <= blockNumber`
-  let paramDnf = switch groups {
-  | [] => None
-  | _ =>
-    Some(
-      groups
-      ->Array.map(group => "(" ++ group->Array.map(paramLeaf)->Array.join(" && ") ++ ")")
-      ->Array.join(" || "),
-    )
-  }
-  let srcLeaf = `(ic = indexingAddresses[${srcAddressExpr}]) !== undefined && ic.effectiveStartBlock <= blockNumber`
-  switch (isWildcard, paramDnf) {
-  | (true, None) => None
-  | (true, Some(dnf)) => Some("var p = event.params, ic; return " ++ dnf ++ ";")
-  | (false, None) => Some("var ic; return " ++ srcLeaf ++ ";")
-  | (false, Some(dnf)) =>
-    Some("var p = event.params, ic; return " ++ srcLeaf ++ " && (" ++ dnf ++ ");")
-  }
-}
-
-let buildAddressFilter = (
-  groups: array<array<string>>,
-  ~isWildcard: bool,
-  ~srcAddressExpr: string="event.srcAddress",
-): option<(Internal.eventPayload, int, dict<Internal.indexingContract>) => bool> =>
-  buildAddressFilterBody(groups, ~isWildcard, ~srcAddressExpr)->Option.map(compileAddressFilter)
-
 // ============== Build complete EVM event config ==============
 
 let buildEvmEventConfig = (
@@ -431,7 +376,7 @@ let buildEvmOnEventRegistration = (
     contractRegister,
     resolvedWhere,
     filterByAddresses,
-    clientAddressFilter: ?buildAddressFilter(addressFilterParamGroups, ~isWildcard),
+    addressFilterParamGroups,
     dependsOnAddresses: Internal.dependsOnAddresses(~isWildcard, ~filterByAddresses),
     startBlock: resolvedStartBlock,
   }
@@ -516,7 +461,6 @@ let buildSvmOnEventRegistration = (
   isWildcard,
   filterByAddresses: false,
   dependsOnAddresses: Internal.dependsOnAddresses(~isWildcard, ~filterByAddresses=false),
-  clientAddressFilter: ?buildAddressFilter([], ~isWildcard, ~srcAddressExpr="event.programId"),
   startBlock,
 }
 
@@ -594,6 +538,5 @@ let buildFuelOnEventRegistration = (
   isWildcard,
   filterByAddresses: false,
   dependsOnAddresses: Internal.dependsOnAddresses(~isWildcard, ~filterByAddresses=false),
-  clientAddressFilter: ?buildAddressFilter([], ~isWildcard),
   startBlock,
 }

package/src/EventConfigBuilder.res.mjs

@@ -243,33 +243,6 @@ function resolveFieldSelection(blockFields, transactionFields, globalBlockFields
   ];
 }
 
-let compileAddressFilter = (function (body) {
-  return new Function("event", "blockNumber", "indexingAddresses", body);
-});
-
-function buildAddressFilterBody(groups, isWildcard, srcAddressExprOpt) {
-  let srcAddressExpr = srcAddressExprOpt !== undefined ? srcAddressExprOpt : "event.srcAddress";
-  let paramLeaf = name => `(ic = indexingAddresses[p[` + JSON.stringify(name) + `]]) !== undefined && ic.effectiveStartBlock <= blockNumber`;
-  let paramDnf = groups.length !== 0 ? groups.map(group => "(" + group.map(paramLeaf).join(" && ") + ")").join(" || ") : undefined;
-  let srcLeaf = `(ic = indexingAddresses[` + srcAddressExpr + `]) !== undefined && ic.effectiveStartBlock <= blockNumber`;
-  if (isWildcard) {
-    if (paramDnf !== undefined) {
-      return "var p = event.params, ic; return " + paramDnf + ";";
-    } else {
-      return;
-    }
-  } else if (paramDnf !== undefined) {
-    return "var p = event.params, ic; return " + srcLeaf + " && (" + paramDnf + ");";
-  } else {
-    return "var ic; return " + srcLeaf + ";";
-  }
-}
-
-function buildAddressFilter(groups, isWildcard, srcAddressExprOpt) {
-  let srcAddressExpr = srcAddressExprOpt !== undefined ? srcAddressExprOpt : "event.srcAddress";
-  return Stdlib_Option.map(buildAddressFilterBody(groups, isWildcard, srcAddressExpr), compileAddressFilter);
-}
-
 function buildEvmEventConfig(contractName, eventName, sighash, params, blockFields, transactionFields, globalBlockFieldsSetOpt, globalTransactionFieldsSetOpt) {
   let globalBlockFieldsSet = globalBlockFieldsSetOpt !== undefined ? Primitive_option.valFromOption(globalBlockFieldsSetOpt) : new Set();
   let globalTransactionFieldsSet = globalTransactionFieldsSetOpt !== undefined ? Primitive_option.valFromOption(globalTransactionFieldsSetOpt) : new Set();
@@ -314,7 +287,7 @@ function buildEvmOnEventRegistration(eventConfig, isWildcard, handler, contractR
     isWildcard: isWildcard,
     filterByAddresses: filterByAddresses,
     dependsOnAddresses: Internal.dependsOnAddresses(isWildcard, filterByAddresses),
-    clientAddressFilter: buildAddressFilter(match.addressFilterParamGroups, isWildcard, undefined),
+    addressFilterParamGroups: match.addressFilterParamGroups,
     startBlock: resolvedStartBlock,
     resolvedWhere: resolvedWhere
   };
@@ -367,7 +340,6 @@ function buildSvmOnEventRegistration(eventConfig, isWildcard, handler, contractR
     isWildcard: isWildcard,
     filterByAddresses: false,
     dependsOnAddresses: Internal.dependsOnAddresses(isWildcard, false),
-    clientAddressFilter: buildAddressFilter([], isWildcard, "event.programId"),
     startBlock: startBlock
   };
 }
@@ -437,7 +409,6 @@ function buildFuelOnEventRegistration(eventConfig, isWildcard, handler, contract
     isWildcard: isWildcard,
     filterByAddresses: false,
     dependsOnAddresses: Internal.dependsOnAddresses(isWildcard, false),
-    clientAddressFilter: buildAddressFilter([], isWildcard, undefined),
     startBlock: startBlock
   };
 }
@@ -457,9 +428,6 @@ export {
   buildTopicGetter,
   alwaysIncludedBlockFields,
   resolveFieldSelection,
-  compileAddressFilter,
-  buildAddressFilterBody,
-  buildAddressFilter,
   buildEvmEventConfig,
   buildEvmOnEventRegistration,
   alwaysIncludedSvmBlockFields,