envio 3.15.0-subgraph → 3.16.0-subgraph

package/src/TestIndexer.res.mjs

@@ -7,7 +7,6 @@ import * as Config from "./Config.res.mjs";
 import * as Logger from "./Logger.res.mjs";
 import * as ChainId from "./ChainId.res.mjs";
 import * as ChainMap from "./ChainMap.res.mjs";
-import * as EntityId from "./EntityId.res.mjs";
 import * as Frontier from "./db/Frontier.res.mjs";
 import * as Internal from "./Internal.res.mjs";
 import * as ChainState from "./ChainState.res.mjs";
@@ -39,11 +38,12 @@ function renderRows(rows, config) {
   return AddressRows.render(rows, config.ecosystem.name, !config.lowercaseAddresses);
 }
 
-function rowKey(scope, entityId) {
+function rowKey(table, scope, entityId) {
+  let entityKey = table.entityIdToKey(entityId);
   if (scope === "crossChain") {
-    return EntityId.toKey(entityId);
+    return entityKey;
   } else {
-    return ChainId.toString(scope) + `|` + EntityId.toKey(entityId);
+    return ChainId.toString(scope) + `|` + entityKey;
   }
 }
 
@@ -126,12 +126,12 @@ function handleWriteBatch(state, config, updatedEntities, registeredAddresses, c
         let entity = change.entity;
         let match = Internal.chainScopeChainId(scope);
         let storedEntity = chainIdField !== undefined && match !== undefined ? Internal.stampChainId(entity, chainIdField.fieldName, Primitive_option.valFromOption(match)) : entity;
-        entityDict[rowKey(scope, change.entityId)] = storedEntity;
+        entityDict[rowKey(entityConfig.table, scope, change.entityId)] = storedEntity;
         entityChangeFor(change.checkpointId).sets.push(entity);
         return;
       }
       let entityId = change.entityId;
-      Stdlib_Dict.$$delete(entityDict, rowKey(scope, entityId));
+      Stdlib_Dict.$$delete(entityDict, rowKey(entityConfig.table, scope, entityId));
       entityChangeFor(change.checkpointId).deleted.push(entityId);
     };
     param.changes.forEach(processChange);
@@ -304,11 +304,14 @@ function getEntityFromState(state, entityConfig, entityId, methodName) {
     Stdlib_JsError.throwWithMessage(`Cannot call ` + entityConfig.name + `.` + methodName + `() while indexer.process() is running. ` + "Wait for process() to complete before accessing entities directly.");
   }
   let entityDict = Stdlib_Option.getOr(state.entities[entityConfig.name], {});
+  let match = entityConfig.table;
+  let entityIdToKey = match.entityIdToKey;
+  let key = entityIdToKey(entityId);
   let field = Table.getChainIdField(entityConfig.table);
   if (field === undefined) {
-    return Stdlib_Option.map(entityDict[entityId], copyEntity);
+    return Stdlib_Option.map(entityDict[key], copyEntity);
   }
-  let matches = Object.values(entityDict).filter(entity => entity.id === entityId);
+  let matches = Object.values(entityDict).filter(entity => entityIdToKey(entity.id) === key);
   let len = matches.length;
   if (len !== 1) {
     if (len === 0) {
@@ -358,7 +361,7 @@ function makeEntitySet(state, entityConfig) {
     } else {
       scope = "crossChain";
     }
-    entityDict[rowKey(scope, entity.id)] = Utils.Dict.shallowCopy(entity);
+    entityDict[rowKey(entityConfig.table, scope, entity.id)] = Utils.Dict.shallowCopy(entity);
   };
 }
 
@@ -388,23 +391,23 @@ function makeInMemoryStorage(state) {
     name: "test-inmemory",
     entityStoreName: param => "test-inmemory",
     holding: async () => "Initialized",
-    initialize: async (param, param$1, param$2, param$3, param$4) => Stdlib_JsError.throwWithMessage("TestIndexer: initialize should not be called; the initial state is derived from config."),
+    initialize: async (param, param$1, param$2) => Stdlib_JsError.throwWithMessage("TestIndexer: initialize should not be called; the initial state is derived from config."),
     readStoredConfig: async () => Stdlib_JsError.throwWithMessage("TestIndexer: readStoredConfig should not be called; the initial state is derived from config."),
-    addChain: async (param, param$1, param$2) => Stdlib_JsError.throwWithMessage("TestIndexer: addChain should not be called; the initial state is derived from config."),
-    resumeInitialState: async (param, param$1, param$2) => Stdlib_JsError.throwWithMessage("TestIndexer: resumeInitialState should not be called; the initial state is derived from config."),
+    addChain: async (param, param$1) => Stdlib_JsError.throwWithMessage("TestIndexer: addChain should not be called; the initial state is derived from config."),
+    resumeInitialState: async (param, param$1) => Stdlib_JsError.throwWithMessage("TestIndexer: resumeInitialState should not be called; the initial state is derived from config."),
     loadOrThrow: async (filter, table) => handleLoad(state, table.tableName, filter),
     ensureQueryIndexes: async (param, param$1, param$2) => {},
-    ensureSchemaIndexes: async (param, param$1) => {},
-    finalizeBackfill: async (param, param$1, param$2) => {},
+    ensureSchemaIndexes: async param => {},
+    finalizeBackfill: async (param, param$1) => {},
     dumpEffectCache: async () => {},
     reset: async () => {},
     setChainMeta: async param => {},
     pruneStaleCheckpoints: async param => {},
     pruneStaleEntityHistory: async (param, param$1) => {},
-    getRollbackTargetCheckpoint: async (param, param$1, param$2) => Stdlib_JsError.throwWithMessage("TestIndexer: Rollback is not supported. The runner forces rollbackOnReorg off, so this should be unreachable."),
+    getRollbackTargetCheckpoint: async (param, param$1) => Stdlib_JsError.throwWithMessage("TestIndexer: Rollback is not supported. The runner forces rollbackOnReorg off, so this should be unreachable."),
     getRollbackProgressDiff: async param => Stdlib_JsError.throwWithMessage("TestIndexer: Rollback is not supported. The runner forces rollbackOnReorg off, so this should be unreachable."),
     getRollbackData: async (param, param$1) => Stdlib_JsError.throwWithMessage("TestIndexer: Rollback is not supported. The runner forces rollbackOnReorg off, so this should be unreachable."),
-    writeBatch: async (batch, param, config, param$1, param$2, updatedEntities, registeredAddresses, param$3, param$4) => handleWriteBatch(state, config, updatedEntities, registeredAddresses, batch.checkpointIds, batch.checkpointChainIds, batch.checkpointBlockNumbers, batch.checkpointEventsProcessed),
+    writeBatch: async (batch, param, config, param$1, updatedEntities, registeredAddresses, param$2, param$3) => handleWriteBatch(state, config, updatedEntities, registeredAddresses, batch.checkpointIds, batch.checkpointChainIds, batch.checkpointBlockNumbers, batch.checkpointEventsProcessed),
     close: async () => {}
   };
 }
@@ -648,7 +651,7 @@ function createTestIndexer() {
           let indexerState = IndexerState.makeFromDbState(runConfig, logger$1, persistence, initialState, registrationsByChainId, undefined, undefined, exitAfterFirstEventBlock, undefined, undefined, undefined, errHandler => {
             ErrorHandling.log(errHandler);
             reject(ErrorHandling.toExn(errHandler));
-          }, () => resolve());
+          }, () => resolve(), undefined);
           indexerStateRef.contents = Primitive_option.some(indexerState);
           IndexerLoop.start(indexerState);
         });

package/src/Throttler.res

@@ -5,27 +5,30 @@ type t = {
   mutable scheduled: option<unit => promise<unit>>,
   intervalMillis: float,
   logger: Logger.t,
+  clock: Clock.t,
 }
 
-let make = (~intervalMillis: int, ~logger) => {
-  lastRunTimeMillis: 0.,
+let make = (~intervalMillis: int, ~logger, ~clock=Clock.real) => {
+  // A clock reads from its own origin, which the first run must not wait on.
+  lastRunTimeMillis: -.(intervalMillis->Int.toFloat),
   isRunning: false,
   isAwaitingInterval: false,
   scheduled: None,
   intervalMillis: intervalMillis->Int.toFloat,
   logger,
+  clock,
 }
 
 let rec startInternal = (throttler: t) => {
   switch throttler {
   | {scheduled: Some(fn), isRunning: false, isAwaitingInterval: false} =>
-    let timeSinceLastRun = Date.now() -. throttler.lastRunTimeMillis
+    let timeSinceLastRun = throttler.clock.now() -. throttler.lastRunTimeMillis
 
     //Only execute if we are passed the interval
     if timeSinceLastRun >= throttler.intervalMillis {
       throttler.isRunning = true
       throttler.scheduled = None
-      throttler.lastRunTimeMillis = Date.now()
+      throttler.lastRunTimeMillis = throttler.clock.now()
 
       // Defer off the schedule call so work queued before it (e.g. a batch) runs first.
       NodeJs.setImmediate(() => {
@@ -45,10 +48,14 @@ let rec startInternal = (throttler: t) => {
     } else {
       //Store isAwaitingInterval in state so that timers don't continuously get created
       throttler.isAwaitingInterval = true
-      let _ = setTimeout(() => {
-        throttler.isAwaitingInterval = false
-        throttler->startInternal
-      }, Int.fromFloat(throttler.intervalMillis -. timeSinceLastRun))
+      let _ = throttler.clock.setTimeout(
+        ~label="throttled write",
+        () => {
+          throttler.isAwaitingInterval = false
+          throttler->startInternal
+        },
+        Int.fromFloat(throttler.intervalMillis -. timeSinceLastRun),
+      )
     }
   | _ => ()
   }

package/src/Throttler.res.mjs

@@ -1,16 +1,19 @@
 // Generated by ReScript, PLEASE EDIT WITH CARE
 
+import * as Clock from "./Clock.res.mjs";
 import * as Logger from "./Logger.res.mjs";
 import * as Primitive_exceptions from "@rescript/runtime/lib/es6/Primitive_exceptions.js";
 
-function make(intervalMillis, logger) {
+function make(intervalMillis, logger, clockOpt) {
+  let clock = clockOpt !== undefined ? clockOpt : Clock.real;
   return {
-    lastRunTimeMillis: 0,
+    lastRunTimeMillis: - intervalMillis,
     isRunning: false,
     isAwaitingInterval: false,
     scheduled: undefined,
     intervalMillis: intervalMillis,
-    logger: logger
+    logger: logger,
+    clock: clock
   };
 }
 
@@ -27,11 +30,11 @@ function startInternal(throttler) {
   if (fn === undefined) {
     return;
   }
-  let timeSinceLastRun = Date.now() - throttler.lastRunTimeMillis;
+  let timeSinceLastRun = throttler.clock.now() - throttler.lastRunTimeMillis;
   if (timeSinceLastRun >= throttler.intervalMillis) {
     throttler.isRunning = true;
     throttler.scheduled = undefined;
-    throttler.lastRunTimeMillis = Date.now();
+    throttler.lastRunTimeMillis = throttler.clock.now();
     setImmediate(() => {
       (async () => {
         try {
@@ -46,7 +49,7 @@ function startInternal(throttler) {
     });
   } else {
     throttler.isAwaitingInterval = true;
-    setTimeout(() => {
+    throttler.clock.setTimeout("throttled write", () => {
       throttler.isAwaitingInterval = false;
       startInternal(throttler);
     }, throttler.intervalMillis - timeSinceLastRun | 0);

package/src/Throttler.resi

@@ -15,7 +15,7 @@ Does NOT queue scheduled functions but rather overwrites them
 The logger will be used to log any errors that occur in the scheduled
 functions.
 */
-let make: (~intervalMillis: int, ~logger: Logger.t) => t
+let make: (~intervalMillis: int, ~logger: Logger.t, ~clock: Clock.t=?) => t
 
 /**
 Schedules a function to be run on a throttler, overwriting any

package/src/UserContext.res

@@ -390,11 +390,13 @@ let makeEntityContext = (params: params, ~entityConfig: Internal.entityConfig) =
         let run = currentRunOrThrow(~prop)
         let sync = run->syncOrThrow(~prop=`${prop}.getSync`)
         let scope = run->scopeOf
+        // The lookahead keys an entity as its writes do, a Bytes id by its hex.
+        let entityKey = entityConfig.table.entityIdToKey(entityId->EntityId.unsafeOfString)
         switch sync.writes->SyncWrites.getById(~entityConfig, ~entityId) {
         | Some(entity) => entity
         | None =>
           switch run.isPreload
-            ? PreloadLookahead.overlayGet(~item=run.item, ~entityName, ~scope, ~entityId)
+            ? PreloadLookahead.overlayGet(~item=run.item, ~entityName, ~scope, ~entityId=entityKey)
             : None {
           | Some(entity) => entity
           | None =>
@@ -406,7 +408,12 @@ let makeEntityContext = (params: params, ~entityConfig: Internal.entityConfig) =
             ) {
             | Some(entity) =>
               if run.isPreload && entity->Option.isNone {
-                PreloadLookahead.recordAbsent(~item=run.item, ~entityName, ~scope, ~entityId)
+                PreloadLookahead.recordAbsent(
+                  ~item=run.item,
+                  ~entityName,
+                  ~scope,
+                  ~entityId=entityKey,
+                )
               }
               entity
             | None =>
@@ -464,7 +471,9 @@ let makeEntityContext = (params: params, ~entityConfig: Internal.entityConfig) =
         | None =>
           let inMemTable =
             params.indexerState->InMemoryStore.getInMemTable(~entityConfig, ~scope=run->scopeOf)
-          switch inMemTable.latestEntityChangeById->Utils.Dict.dangerouslyGetNonOption(entityId) {
+          switch inMemTable.latestEntityChangeById->Utils.Dict.dangerouslyGetNonOption(
+            inMemTable.entityIdToKey(entityId->EntityId.unsafeOfString),
+          ) {
           | Some(change) if change->Change.getCheckpointId == run.checkpointId =>
             change->InMemoryTable.Entity.mapChangeToEntity
           | _ => None
@@ -527,7 +536,7 @@ let rec runSyncRound = async (params: params, sync: syncRun, fn: unit => unit, ~
             ~item=run.item,
             ~entityName=entityConfig.name,
             ~scope,
-            ~entityId=change->Change.getEntityId->EntityId.toKey,
+            ~entityId=entityConfig.table.entityIdToKey(change->Change.getEntityId),
             ~entity=change->InMemoryTable.Entity.mapChangeToEntity,
           )
         )

package/src/UserContext.res.mjs

@@ -4,7 +4,6 @@ import * as Env from "./Env.res.mjs";
 import * as Utils from "./Utils.res.mjs";
 import * as Logger from "./Logger.res.mjs";
 import * as ChainId from "./ChainId.res.mjs";
-import * as EntityId from "./EntityId.res.mjs";
 import * as Internal from "./Internal.res.mjs";
 import * as RunScope from "./RunScope.res.mjs";
 import * as LoadLayer from "./LoadLayer.res.mjs";
@@ -275,11 +274,12 @@ function makeEntityContext(params, entityConfig) {
     let run = currentRunOrThrow(prop);
     let sync = syncOrThrow(run, prop + `.getSync`);
     let scope = scopeOf(run);
+    let entityKey = entityConfig.table.entityIdToKey(entityId);
     let entity = SyncWrites.getById(sync.writes, entityConfig, entityId);
     if (entity !== undefined) {
       return Primitive_option.valFromOption(entity);
     }
-    let entity$1 = run.isPreload ? PreloadLookahead.overlayGet(run.item, entityName, scope, entityId) : undefined;
+    let entity$1 = run.isPreload ? PreloadLookahead.overlayGet(run.item, entityName, scope, entityKey) : undefined;
     if (entity$1 !== undefined) {
       return Primitive_option.valFromOption(entity$1);
     }
@@ -289,7 +289,7 @@ function makeEntityContext(params, entityConfig) {
     }
     let entity$3 = Primitive_option.valFromOption(entity$2);
     if (run.isPreload && Stdlib_Option.isNone(entity$3)) {
-      PreloadLookahead.recordAbsent(run.item, entityName, scope, entityId);
+      PreloadLookahead.recordAbsent(run.item, entityName, scope, entityKey);
     }
     return entity$3;
   };
@@ -313,7 +313,7 @@ function makeEntityContext(params, entityConfig) {
       return Primitive_option.valFromOption(entity);
     }
     let inMemTable = InMemoryStore.getInMemTable(params.indexerState, entityConfig, scopeOf(run));
-    let change = inMemTable.latestEntityChangeById[entityId];
+    let change = inMemTable.latestEntityChangeById[inMemTable.entityIdToKey(entityId)];
     if (change !== undefined && change.checkpointId === run.checkpointId) {
       return InMemoryTable.Entity.mapChangeToEntity(change);
     }
@@ -361,7 +361,7 @@ async function runSyncRound(params, sync, fn, round) {
     let match = sync.status;
     if (typeof match !== "object" && sync.speculated <= 0) {
       if (run.isPreload) {
-        SyncWrites.forEachChange(writes, (entityConfig, scope, change) => PreloadLookahead.overlaySet(run.item, entityConfig.name, scope, EntityId.toKey(change.entityId), InMemoryTable.Entity.mapChangeToEntity(change)));
+        SyncWrites.forEachChange(writes, (entityConfig, scope, change) => PreloadLookahead.overlaySet(run.item, entityConfig.name, scope, entityConfig.table.entityIdToKey(change.entityId), InMemoryTable.Entity.mapChangeToEntity(change)));
       } else {
         SyncWrites.commit(writes, params.indexerState);
       }

package/src/Utils.res

@@ -10,12 +10,6 @@ external toMethod: 'a => 'a = "%unsafe_to_method"
 
 @val external importPath: string => promise<unknown> = "import"
 
-// Half the delay fixed, half spread across it. Every indexer pointed at one
-// provider loses its stream, or gives up on a quiet chain, in the same instant
-// that provider blinks, and putting them all back on the same schedule is how a
-// blip becomes a stampede.
-let jitter = delay => delay / 2 + (Math.random() *. (delay / 2)->Int.toFloat)->Float.toInt
-
 // Clearing a pending timer and forgetting it are one action. A ref left holding
 // a spent id reads as a timer still pending, and the next clear looks like it
 // did something.

package/src/Utils.res.mjs

@@ -14,10 +14,6 @@ import * as Primitive_string from "@rescript/runtime/lib/es6/Primitive_string.js
 import * as S$RescriptSchema from "rescript-schema/src/S.res.mjs";
 import * as Primitive_exceptions from "@rescript/runtime/lib/es6/Primitive_exceptions.js";
 
-function jitter(delay) {
-  return (delay / 2 | 0) + (Math.random() * (delay / 2 | 0) | 0) | 0;
-}
-
 function clearTimeoutRef(timeoutId) {
   let id = timeoutId.contents;
   if (id !== undefined) {
@@ -824,7 +820,6 @@ function replaceLoneSurrogateEscapes(json) {
 let maxBackoffExponent = 20;
 
 export {
-  jitter,
   clearTimeoutRef,
   maxBackoffExponent,
   expBackoff,

package/src/Writing.res

@@ -153,7 +153,6 @@ let runOneWrite = async (state: IndexerState.t) => {
         ~batch,
         ~rollback,
         ~config,
-        ~allEntities=persistence.allEntities,
         ~updatedEntities,
         ~registeredAddresses,
         ~updatedEffectsCache,

package/src/Writing.res.mjs

@@ -121,7 +121,7 @@ async function runOneWrite(state) {
   });
   let pruneTargets = PruneStaleHistory.select(state, writtenEntityNames, Stdlib_Option.isSome(rollback));
   await Promise.all([
-    persistence.storage.writeBatch(batch, rollback, config, persistence.allEntities, updatedEffectsCache, updatedEntities, registeredAddresses, chainMetaData, (storage, timeSeconds) => IndexerState.recordStorageWrite(state, storage, timeSeconds)),
+    persistence.storage.writeBatch(batch, rollback, config, updatedEffectsCache, updatedEntities, registeredAddresses, chainMetaData, (storage, timeSeconds) => IndexerState.recordStorageWrite(state, storage, timeSeconds)),
     PruneStaleHistory.runConcurrent(state, pruneTargets)
   ]);
   IndexerState.markCommitted(state, writtenFrontier);