envio 3.15.0 → 3.16.0-subgraph

package/src/UserContext.res.mjs

@@ -1,18 +1,84 @@
 // Generated by ReScript, PLEASE EDIT WITH CARE
 
+import * as Env from "./Env.res.mjs";
 import * as Utils from "./Utils.res.mjs";
 import * as Logger from "./Logger.res.mjs";
+import * as ChainId from "./ChainId.res.mjs";
 import * as Internal from "./Internal.res.mjs";
 import * as RunScope from "./RunScope.res.mjs";
 import * as LoadLayer from "./LoadLayer.res.mjs";
+import * as SyncWrites from "./SyncWrites.res.mjs";
 import * as EntityFilter from "./db/EntityFilter.res.mjs";
-import * as IndexerState from "./IndexerState.res.mjs";
 import * as InMemoryStore from "./InMemoryStore.res.mjs";
 import * as InMemoryTable from "./InMemoryTable.res.mjs";
 import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js";
 import * as Stdlib_JsError from "@rescript/runtime/lib/es6/Stdlib_JsError.js";
+import * as Stdlib_Promise from "@rescript/runtime/lib/es6/Stdlib_Promise.js";
+import * as PreloadLookahead from "./PreloadLookahead.res.mjs";
 import * as Primitive_option from "@rescript/runtime/lib/es6/Primitive_option.js";
 import * as S$RescriptSchema from "rescript-schema/src/S.res.mjs";
+import * as Primitive_exceptions from "@rescript/runtime/lib/es6/Primitive_exceptions.js";
+
+let Suspend = /* @__PURE__ */Primitive_exceptions.create("UserContext.Suspend");
+
+function isSuspend(exn) {
+  return exn.RE_EXN_ID === Suspend;
+}
+
+let suspend = {
+  RE_EXN_ID: Suspend
+};
+
+let currentSync = {
+  contents: undefined
+};
+
+function syncOf(run) {
+  let sync = currentSync.contents;
+  if (sync !== undefined && sync.run === run) {
+    return sync;
+  }
+}
+
+function currentRunOrThrow(prop) {
+  let run = RunScope.currentHandlerRunOrThrow(prop);
+  let match = syncOf(run);
+  if (match !== undefined) {
+    let exn = match.status;
+    if (typeof exn === "object") {
+      throw exn._0;
+    }
+  }
+  return run;
+}
+
+function syncOrThrow(run, prop) {
+  let sync = syncOf(run);
+  if (sync !== undefined) {
+    return sync;
+  } else {
+    return Stdlib_JsError.throwWithMessage(`context.` + prop + ` is only available inside context.runSync.`);
+  }
+}
+
+function scheduleAndSuspend(sync, promise) {
+  sync.pending.push(promise);
+  sync.status = {
+    TAG: "Aborted",
+    _0: suspend
+  };
+  throw suspend;
+}
+
+function missOrSuspend(sync, promise, absent) {
+  if (sync.run.isPreload && sync.speculated < 32) {
+    sync.pending.push(promise);
+    sync.speculated = sync.speculated + 1 | 0;
+    return absent;
+  } else {
+    return scheduleAndSuspend(sync, promise);
+  }
+}
 
 let effectContextPrototype = (Object.create(null));
 
@@ -46,7 +112,7 @@ function isCrossChain(params, effect) {
   return Stdlib_Option.getOr(effect.crossChain, params.config.defaultCrossChain);
 }
 
-function callEffect(params, run, caller, effect, input) {
+function prepareEffectCall(params, run, caller, effect, input) {
   let handlerChainId = Internal.getItemChainId(run.item);
   let scope = isCrossChain(params, effect) ? "crossChain" : handlerChainId;
   if (caller !== undefined && scope !== "crossChain") {
@@ -61,13 +127,59 @@ function callEffect(params, run, caller, effect, input) {
     caller: run,
     effectName: effect.name
   });
-  return LoadLayer.loadEffect(params.loadManager, params.persistence, effect, {
+  let effectArgs_cacheKey = Utils.Hash.boundCacheKey(Utils.Hash.makeOrThrow(S$RescriptSchema.reverseConvertOrThrow(input, effect.input)));
+  let effectArgs_checkpointId = run.checkpointId;
+  let effectArgs = {
     input: input,
     context: effectContext,
-    cacheKey: Utils.Hash.makeOrThrow(S$RescriptSchema.reverseConvertOrThrow(input, effect.input)),
+    cacheKey: effectArgs_cacheKey,
     chainId: handlerChainId,
-    checkpointId: run.checkpointId
-  }, scope, params.indexerState, run.isPreload);
+    checkpointId: effectArgs_checkpointId
+  };
+  return [
+    scope,
+    effectArgs
+  ];
+}
+
+function callEffect(params, run, caller, effect, input) {
+  let match = prepareEffectCall(params, run, caller, effect, input);
+  return LoadLayer.loadEffect(params.loadManager, params.persistence, effect, match[1], match[0], params.indexerState, run.isPreload);
+}
+
+function effectMemoKey(effect, scope, cacheKey) {
+  if (scope === "crossChain") {
+    return effect.name + `.` + cacheKey;
+  } else {
+    return effect.name + `.` + ChainId.toString(scope) + `.` + cacheKey;
+  }
+}
+
+function effectSync(params, effect, input) {
+  let run = currentRunOrThrow("effectSync");
+  let sync = syncOrThrow(run, "effectSync");
+  let match = prepareEffectCall(params, run, undefined, effect, input);
+  let effectArgs = match[1];
+  let scope = match[0];
+  let memoKey = effectMemoKey(effect, scope, effectArgs.cacheKey);
+  let output = sync.memo[memoKey];
+  if (output !== undefined) {
+    return Primitive_option.valFromOption(output);
+  }
+  let inMemTable = InMemoryStore.getEffectInMemTable(params.indexerState, effect, scope);
+  if (InMemoryStore.hasEffectOutput(inMemTable, effectArgs.cacheKey)) {
+    let output$1 = InMemoryStore.getEffectOutputUnsafe(inMemTable, effectArgs.cacheKey);
+    sync.memo[memoKey] = output$1;
+    return output$1;
+  }
+  if (sync.speculated <= 0) {
+    return scheduleAndSuspend(sync, LoadLayer.loadEffect(params.loadManager, params.persistence, effect, effectArgs, scope, params.indexerState, run.isPreload));
+  }
+  sync.status = {
+    TAG: "Aborted",
+    _0: suspend
+  };
+  throw suspend;
 }
 
 function expiresUnread(entityConfig) {
@@ -85,35 +197,43 @@ function throwExpiresUnread(entityConfig, op) {
 
 function makeEntityContext(params, entityConfig) {
   let prop = entityConfig.name;
+  let entityName = entityConfig.name;
   let scopeOf = run => InMemoryStore.entityScope(entityConfig, Internal.getItemChainId(run.item));
   let loadById = entityId => {
-    let run = RunScope.currentHandlerRunOrThrow(prop);
+    let run = currentRunOrThrow(prop);
     return LoadLayer.loadById(params.loadManager, params.persistence, entityConfig, scopeOf(run), params.indexerState, run.isPreload, entityId);
   };
   let write = (run, change) => {
-    let scope = scopeOf(run);
-    InMemoryTable.Entity.set(InMemoryStore.getInMemTable(params.indexerState, entityConfig, scope), IndexerState.committedCheckpointIdFor(params.indexerState, scope), change);
+    let match = syncOf(run);
+    let match$1 = run.isPreload;
+    if (match !== undefined) {
+      if (match$1 && !PreloadLookahead.isActive()) {
+        return;
+      } else {
+        return SyncWrites.record(match.writes, entityConfig, scopeOf(run), change);
+      }
+    } else if (match$1) {
+      return;
+    } else {
+      return SyncWrites.write(params.indexerState, entityConfig, scopeOf(run), change);
+    }
   };
   let set = entity => {
-    let run = RunScope.currentHandlerRunOrThrow(prop);
-    if (!run.isPreload) {
-      return write(run, {
-        type: "SET",
-        entityId: entity.id,
-        entity: entity,
-        checkpointId: run.checkpointId
-      });
-    }
+    let run = currentRunOrThrow(prop);
+    write(run, {
+      type: "SET",
+      entityId: entity.id,
+      entity: entity,
+      checkpointId: run.checkpointId
+    });
   };
   let deleteUnsafe = entityId => {
-    let run = RunScope.currentHandlerRunOrThrow(prop);
-    if (!run.isPreload) {
-      return write(run, {
-        type: "DELETE",
-        entityId: entityId,
-        checkpointId: run.checkpointId
-      });
-    }
+    let run = currentRunOrThrow(prop);
+    write(run, {
+      type: "DELETE",
+      entityId: entityId,
+      checkpointId: run.checkpointId
+    });
   };
   let entityContext = expiresUnread(entityConfig) ? ({
       get: param => throwExpiresUnread(entityConfig, "get"),
@@ -125,7 +245,7 @@ function makeEntityContext(params, entityConfig) {
     }) : ({
       get: loadById,
       getWhere: filter => {
-        let run = RunScope.currentHandlerRunOrThrow(prop);
+        let run = currentRunOrThrow(prop);
         return LoadLayer.loadByFilter(params.loadManager, params.persistence, entityConfig, scopeOf(run), params.indexerState, run.isPreload, filter);
       },
       getOrThrow: (entityId, message) => loadById(entityId).then(entity => {
@@ -146,7 +266,136 @@ function makeEntityContext(params, entityConfig) {
       set: set,
       deleteUnsafe: deleteUnsafe
     });
-  return Object.assign(Object.create(Utils.StrictObject.makePrototype(op => Stdlib_JsError.throwWithMessage(`Invalid context.` + entityConfig.name + `.` + op + ` operation.`))), entityContext);
+  let object = Object.assign(Object.create(Utils.StrictObject.makePrototype(op => Stdlib_JsError.throwWithMessage(`Invalid context.` + entityConfig.name + `.` + op + ` operation.`))), entityContext);
+  if (!Env.subgraphMode.contents) {
+    return object;
+  }
+  let syncEntityContext_getSync = entityId => {
+    let run = currentRunOrThrow(prop);
+    let sync = syncOrThrow(run, prop + `.getSync`);
+    let scope = scopeOf(run);
+    let entityKey = entityConfig.table.entityIdToKey(entityId);
+    let entity = SyncWrites.getById(sync.writes, entityConfig, entityId);
+    if (entity !== undefined) {
+      return Primitive_option.valFromOption(entity);
+    }
+    let entity$1 = run.isPreload ? PreloadLookahead.overlayGet(run.item, entityName, scope, entityKey) : undefined;
+    if (entity$1 !== undefined) {
+      return Primitive_option.valFromOption(entity$1);
+    }
+    let entity$2 = LoadLayer.getByIdInMemory(entityConfig, scope, params.indexerState, entityId);
+    if (entity$2 === undefined) {
+      return missOrSuspend(sync, LoadLayer.loadById(params.loadManager, params.persistence, entityConfig, scope, params.indexerState, run.isPreload, entityId), undefined);
+    }
+    let entity$3 = Primitive_option.valFromOption(entity$2);
+    if (run.isPreload && Stdlib_Option.isNone(entity$3)) {
+      PreloadLookahead.recordAbsent(run.item, entityName, scope, entityKey);
+    }
+    return entity$3;
+  };
+  let syncEntityContext_getWhereSync = filter => {
+    let run = currentRunOrThrow(prop);
+    let sync = syncOrThrow(run, prop + `.getWhereSync`);
+    let scope = scopeOf(run);
+    let filter$1 = EntityFilter.parseOrThrow(filter, entityName, entityConfig.table);
+    let entities = LoadLayer.getByFilterInMemory(entityConfig, scope, params.indexerState, filter$1);
+    if (entities !== undefined) {
+      return SyncWrites.overlayFilter(sync.writes, entities, entityConfig, filter$1);
+    } else {
+      return missOrSuspend(sync, LoadLayer.loadByParsedFilter(params.loadManager, params.persistence, entityConfig, scope, params.indexerState, run.isPreload, filter$1), []);
+    }
+  };
+  let syncEntityContext_getInBlockSync = entityId => {
+    let run = currentRunOrThrow(prop);
+    let sync = syncOrThrow(run, prop + `.getInBlockSync`);
+    let entity = SyncWrites.getById(sync.writes, entityConfig, entityId);
+    if (entity !== undefined) {
+      return Primitive_option.valFromOption(entity);
+    }
+    let inMemTable = InMemoryStore.getInMemTable(params.indexerState, entityConfig, scopeOf(run));
+    let change = inMemTable.latestEntityChangeById[inMemTable.entityIdToKey(entityId)];
+    if (change !== undefined && change.checkpointId === run.checkpointId) {
+      return InMemoryTable.Entity.mapChangeToEntity(change);
+    }
+  };
+  let syncEntityContext = {
+    getSync: syncEntityContext_getSync,
+    getWhereSync: syncEntityContext_getWhereSync,
+    getInBlockSync: syncEntityContext_getInBlockSync
+  };
+  return Object.assign(object, syncEntityContext);
+}
+
+async function runSyncRound(params, sync, fn, round) {
+  if (round > 10000) {
+    Stdlib_JsError.throwWithMessage(`The handler suspended on a synchronous read too many times: gave up after ` + (10000).toString() + ` rounds. This usually means the code isn't deterministic across reruns.`);
+  }
+  let run = sync.run;
+  sync.status = "Active";
+  sync.pending = [];
+  sync.speculated = 0;
+  sync.writes = SyncWrites.make();
+  let startedAt = PreloadLookahead.roundStarted(run.isPreload);
+  let previous = currentSync.contents;
+  currentSync.contents = sync;
+  let outcome;
+  try {
+    fn();
+    outcome = {
+      TAG: "Ok",
+      _0: undefined
+    };
+  } catch (raw_exn) {
+    let exn = Primitive_exceptions.internalToException(raw_exn);
+    outcome = {
+      TAG: "Error",
+      _0: exn
+    };
+  }
+  currentSync.contents = previous;
+  PreloadLookahead.roundEnded(run.item, startedAt);
+  let writes = sync.writes;
+  sync.writes = SyncWrites.make();
+  let suspended;
+  if (outcome.TAG === "Ok") {
+    let match = sync.status;
+    if (typeof match !== "object" && sync.speculated <= 0) {
+      if (run.isPreload) {
+        SyncWrites.forEachChange(writes, (entityConfig, scope, change) => PreloadLookahead.overlaySet(run.item, entityConfig.name, scope, entityConfig.table.entityIdToKey(change.entityId), InMemoryTable.Entity.mapChangeToEntity(change)));
+      } else {
+        SyncWrites.commit(writes, params.indexerState);
+      }
+      suspended = false;
+    } else {
+      suspended = true;
+    }
+  } else {
+    let exn$1 = outcome._0;
+    if (exn$1.RE_EXN_ID === Suspend || sync.speculated > 0) {
+      suspended = true;
+    } else {
+      sync.pending.forEach(promise => {
+        promise.catch(param => {});
+      });
+      sync.pending = [];
+      throw exn$1;
+    }
+  }
+  if (!suspended) {
+    return;
+  }
+  let pending = sync.pending;
+  sync.pending = [];
+  let errors = [];
+  await Promise.all(pending.map(promise => Stdlib_Promise.$$catch(promise, exn => {
+    errors.push(exn);
+    return Promise.resolve();
+  })));
+  let exn$2 = errors[0];
+  if (exn$2 !== undefined) {
+    throw exn$2;
+  }
+  return await runSyncRound(params, sync, fn, round + 1 | 0);
 }
 
 function makeHandlerContext(params) {
@@ -154,7 +403,7 @@ function makeHandlerContext(params) {
   let define = (name, get) => {
     Object.defineProperty(context, name, {
       enumerable: true,
-      get: () => get(RunScope.currentHandlerRunOrThrow(name))
+      get: () => get(currentRunOrThrow(name))
     });
   };
   define("log", run => {
@@ -164,11 +413,31 @@ function makeHandlerContext(params) {
       return Logger.asContextLog(RunScope.handlerLogger(run));
     }
   });
-  let effect = (effect$1, input) => callEffect(params, RunScope.currentHandlerRunOrThrow("effect"), undefined, effect$1, input);
+  let effect = (effect$1, input) => callEffect(params, currentRunOrThrow("effect"), undefined, effect$1, input);
   Object.defineProperty(context, "effect", {
     enumerable: true,
     value: effect
   });
+  if (Env.subgraphMode.contents) {
+    Object.defineProperty(context, "effectSync", {
+      enumerable: true,
+      value: (effect, input) => effectSync(params, effect, input)
+    });
+    Object.defineProperty(context, "runSync", {
+      enumerable: true,
+      value: fn => {
+        let run = currentRunOrThrow("runSync");
+        return runSyncRound(params, {
+          run: run,
+          status: "Active",
+          pending: [],
+          memo: {},
+          writes: SyncWrites.make(),
+          speculated: 0
+        }, fn, 1);
+      }
+    });
+  }
   define("isPreload", run => run.isPreload);
   define("chain", run => params.chains[Internal.getItemChainId(run.item)]);
   Utils.Dict.forEachWithKey(params.config.userEntitiesByName, (entityConfig, name) => {
@@ -178,13 +447,32 @@ function makeHandlerContext(params) {
   return context;
 }
 
+let maxSpeculatedReads = 32;
+
+let maxSyncRounds = 10000;
+
 export {
+  Suspend,
+  isSuspend,
+  suspend,
+  currentSync,
+  syncOf,
+  currentRunOrThrow,
+  syncOrThrow,
+  scheduleAndSuspend,
+  maxSpeculatedReads,
+  missOrSuspend,
   effectContextPrototype,
   isCrossChain,
+  prepareEffectCall,
   callEffect,
+  effectMemoKey,
+  effectSync,
   expiresUnread,
   throwExpiresUnread,
   makeEntityContext,
+  maxSyncRounds,
+  runSyncRound,
   makeHandlerContext,
 }
 /* effectContextPrototype Not a pure module */

package/src/Utils.res

@@ -10,12 +10,6 @@ external toMethod: 'a => 'a = "%unsafe_to_method"
 
 @val external importPath: string => promise<unknown> = "import"
 
-// Half the delay fixed, half spread across it. Every indexer pointed at one
-// provider loses its stream, or gives up on a quiet chain, in the same instant
-// that provider blinks, and putting them all back on the same schedule is how a
-// blip becomes a stampede.
-let jitter = delay => delay / 2 + (Math.random() *. (delay / 2)->Int.toFloat)->Float.toInt
-
 // Clearing a pending timer and forgetting it are one action. A ref left holding
 // a spent id reads as a timer still pending, and the next clear looks like it
 // did something.
@@ -887,6 +881,23 @@ module Hash = {
       (any->magic)["toString"]()
     }
   }
+
+  type sha256
+  @module("node:crypto") external createSha256: (@as("sha256") _, unit) => sha256 = "createHash"
+  @send external update: (sha256, string) => sha256 = "update"
+  @send external digestHex: (sha256, @as("hex") _) => string = "digest"
+  @val @scope("Buffer") external byteLength: string => int = "byteLength"
+
+  // A cache row is keyed by its effect's input, and a Postgres btree key takes
+  // at most 2704 bytes: a contract call with a large argument (calldata of a
+  // few kB) failed the batch write. Such inputs are keyed by their sha256;
+  // every shorter input keeps the key it always had.
+  let maxRawCacheKeyBytes = 2000
+
+  let boundCacheKey = (key: string) =>
+    key->byteLength > maxRawCacheKeyBytes
+      ? "sha256:" ++ createSha256()->update(key)->digestHex
+      : key
 }
 
 let prettifyExn = exn => {

package/src/Utils.res.mjs

@@ -2,6 +2,7 @@ import packageJson from '../package.json' with { type: 'json' }
 // Generated by ReScript, PLEASE EDIT WITH CARE
 
 import * as Stdlib_Dict from "@rescript/runtime/lib/es6/Stdlib_Dict.js";
+import * as Nodecrypto from "node:crypto";
 import * as Stdlib_Array from "@rescript/runtime/lib/es6/Stdlib_Array.js";
 import * as Stdlib_JsExn from "@rescript/runtime/lib/es6/Stdlib_JsExn.js";
 import * as Primitive_int from "@rescript/runtime/lib/es6/Primitive_int.js";
@@ -13,10 +14,6 @@ import * as Primitive_string from "@rescript/runtime/lib/es6/Primitive_string.js
 import * as S$RescriptSchema from "rescript-schema/src/S.res.mjs";
 import * as Primitive_exceptions from "@rescript/runtime/lib/es6/Primitive_exceptions.js";
 
-function jitter(delay) {
-  return (delay / 2 | 0) + (Math.random() * (delay / 2 | 0) | 0) | 0;
-}
-
 function clearTimeoutRef(timeoutId) {
   let id = timeoutId.contents;
   if (id !== undefined) {
@@ -748,9 +745,19 @@ function makeOrThrow(any) {
   return hash$1 + "}";
 }
 
+function boundCacheKey(key) {
+  if (Buffer.byteLength(key) > 2000) {
+    return "sha256:" + Nodecrypto.createHash("sha256").update(key).digest("hex");
+  } else {
+    return key;
+  }
+}
+
 let Hash = {
   fail: fail,
-  makeOrThrow: makeOrThrow
+  makeOrThrow: makeOrThrow,
+  maxRawCacheKeyBytes: 2000,
+  boundCacheKey: boundCacheKey
 };
 
 function prettifyExn(exn) {
@@ -813,7 +820,6 @@ function replaceLoneSurrogateEscapes(json) {
 let maxBackoffExponent = 20;
 
 export {
-  jitter,
   clearTimeoutRef,
   maxBackoffExponent,
   expBackoff,

package/src/Writing.res

@@ -153,7 +153,6 @@ let runOneWrite = async (state: IndexerState.t) => {
         ~batch,
         ~rollback,
         ~config,
-        ~allEntities=persistence.allEntities,
         ~updatedEntities,
         ~registeredAddresses,
         ~updatedEffectsCache,

package/src/Writing.res.mjs

@@ -121,7 +121,7 @@ async function runOneWrite(state) {
   });
   let pruneTargets = PruneStaleHistory.select(state, writtenEntityNames, Stdlib_Option.isSome(rollback));
   await Promise.all([
-    persistence.storage.writeBatch(batch, rollback, config, persistence.allEntities, updatedEffectsCache, updatedEntities, registeredAddresses, chainMetaData, (storage, timeSeconds) => IndexerState.recordStorageWrite(state, storage, timeSeconds)),
+    persistence.storage.writeBatch(batch, rollback, config, updatedEffectsCache, updatedEntities, registeredAddresses, chainMetaData, (storage, timeSeconds) => IndexerState.recordStorageWrite(state, storage, timeSeconds)),
     PruneStaleHistory.runConcurrent(state, pruneTargets)
   ]);
   IndexerState.markCommitted(state, writtenFrontier);