envio 3.6.1 → 3.7.0-subgraph

package/src/UserContext.res.mjs

@@ -1,6 +1,7 @@
 // Generated by ReScript, PLEASE EDIT WITH CARE
 
 import * as Utils from "./Utils.res.mjs";
+import * as ChainId from "./ChainId.res.mjs";
 import * as Logging from "./Logging.res.mjs";
 import * as Internal from "./Internal.res.mjs";
 import * as Ecosystem from "./Ecosystem.res.mjs";
@@ -12,8 +13,70 @@ import * as InMemoryStore from "./InMemoryStore.res.mjs";
 import * as InMemoryTable from "./InMemoryTable.res.mjs";
 import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js";
 import * as Stdlib_JsError from "@rescript/runtime/lib/es6/Stdlib_JsError.js";
+import * as Stdlib_Promise from "@rescript/runtime/lib/es6/Stdlib_Promise.js";
 import * as Primitive_option from "@rescript/runtime/lib/es6/Primitive_option.js";
 import * as S$RescriptSchema from "rescript-schema/src/S.res.mjs";
+import * as Primitive_exceptions from "@rescript/runtime/lib/es6/Primitive_exceptions.js";
+
+let Suspend = /* @__PURE__ */Primitive_exceptions.create("UserContext.Suspend");
+
+function isSuspend(exn) {
+  return exn.RE_EXN_ID === Suspend;
+}
+
+function makeSyncState() {
+  return {
+    status: "Active",
+    pending: undefined,
+    memo: undefined
+  };
+}
+
+function getPending(sync) {
+  let pending = sync.pending;
+  if (pending !== undefined) {
+    return pending;
+  }
+  let pending$1 = [];
+  sync.pending = pending$1;
+  return pending$1;
+}
+
+function getMemo(sync) {
+  let memo = sync.memo;
+  if (memo !== undefined) {
+    return memo;
+  }
+  let memo$1 = {};
+  sync.memo = memo$1;
+  return memo$1;
+}
+
+function checkStatusOrThrow(params, access) {
+  let exn = params.sync.status;
+  if (typeof exn !== "object") {
+    if (exn === "Active") {
+      return;
+    } else {
+      return ErrorHandling.mkLogAndRaise(Ecosystem.getItemLogger(params.item, params.config.ecosystem), undefined, new Error(`Impossible to access ` + access + ` after the handler is resolved. Make sure you didn't miss an await in the handler.`));
+    }
+  }
+  throw exn._0;
+}
+
+function scheduleAndSuspend(params, promise) {
+  getPending(params.sync).push(promise);
+  params.sync.status = {
+    TAG: "Aborted",
+    _0: {
+      RE_EXN_ID: Suspend
+    }
+  };
+  throw {
+    RE_EXN_ID: Suspend,
+    Error: new Error()
+  };
+}
 
 let paramsByThis = new WeakMap();
 
@@ -47,28 +110,69 @@ var EffectContext = function(params, chainId, effectName, defaultShouldCache, ca
 EffectContext.prototype = effectContextPrototype;
 ;
 
-function initEffect(params) {
-  let handlerChainId = Internal.getItemChainId(params.item);
+function prepareEffectCall(params, effect, input, caller) {
   let isCrossChain = effect => Stdlib_Option.getOr(effect.crossChain, params.config.defaultCrossChain);
-  let makeCaller = caller => ((effect, input) => {
-    let scope = isCrossChain(effect) ? "crossChain" : handlerChainId;
-    if (caller !== undefined && isCrossChain(caller) && !isCrossChain(effect)) {
-      Stdlib_JsError.throwWithMessage(`The cross-chain effect "` + caller.name + `" cannot call the chain-scoped effect "` + effect.name + `", because a cross-chain effect isn't tied to a single chain. Make "` + effect.name + `" cross-chain (\`crossChain: true\`), or make "` + caller.name + `" chain-scoped (\`crossChain: false\`).`);
+  let scope = isCrossChain(effect) ? "crossChain" : Internal.getItemChainId(params.item);
+  if (caller !== undefined && isCrossChain(caller) && !isCrossChain(effect)) {
+    Stdlib_JsError.throwWithMessage(`The cross-chain effect "` + caller.name + `" cannot call the chain-scoped effect "` + effect.name + `", because a cross-chain effect isn't tied to a single chain. Make "` + effect.name + `" cross-chain (\`crossChain: true\`), or make "` + caller.name + `" chain-scoped (\`crossChain: false\`).`);
+  }
+  let tmp;
+  tmp = scope === "crossChain" ? undefined : Primitive_option.some(scope);
+  let effectContext = new EffectContext(params, tmp, effect.name, effect.defaultShouldCache, (nested, nestedInput) => callEffectAsync(params, nested, nestedInput, effect));
+  let effectArgs_cacheKey = Utils.Hash.makeOrThrow(S$RescriptSchema.reverseConvertOrThrow(input, effect.input));
+  let effectArgs_checkpointId = params.checkpointId;
+  let effectArgs = {
+    input: input,
+    context: effectContext,
+    cacheKey: effectArgs_cacheKey,
+    checkpointId: effectArgs_checkpointId
+  };
+  return [
+    scope,
+    effectArgs
+  ];
+}
+
+function callEffectAsync(params, effect, input, caller) {
+  let match = prepareEffectCall(params, effect, input, caller);
+  return LoadLayer.loadEffect(params.loadManager, params.persistence, effect, match[1], match[0], params.indexerState, params.isPreload, params.item, params.config.ecosystem);
+}
+
+function initEffect(params) {
+  return (effect, input) => {
+    checkStatusOrThrow(params, "context.effect");
+    return callEffectAsync(params, effect, input, undefined);
+  };
+}
+
+function effectMemoKey(effect, scope, cacheKey) {
+  if (scope === "crossChain") {
+    return effect.name + `.` + cacheKey;
+  } else {
+    return effect.name + `.` + ChainId.toString(scope) + `.` + cacheKey;
+  }
+}
+
+function initEffectSync(params) {
+  return (effect, input) => {
+    checkStatusOrThrow(params, "context.effectSync");
+    let match = prepareEffectCall(params, effect, input, undefined);
+    let effectArgs = match[1];
+    let scope = match[0];
+    let memo = getMemo(params.sync);
+    let memoKey = effectMemoKey(effect, scope, effectArgs.cacheKey);
+    let output = memo[memoKey];
+    if (output !== undefined) {
+      return Primitive_option.valFromOption(output);
     }
-    let tmp;
-    tmp = scope === "crossChain" ? undefined : Primitive_option.some(scope);
-    let effectContext = new EffectContext(params, tmp, effect.name, effect.defaultShouldCache, makeCaller(effect));
-    let effectArgs_cacheKey = Utils.Hash.makeOrThrow(S$RescriptSchema.reverseConvertOrThrow(input, effect.input));
-    let effectArgs_checkpointId = params.checkpointId;
-    let effectArgs = {
-      input: input,
-      context: effectContext,
-      cacheKey: effectArgs_cacheKey,
-      checkpointId: effectArgs_checkpointId
-    };
-    return LoadLayer.loadEffect(params.loadManager, params.persistence, effect, effectArgs, scope, params.indexerState, params.isPreload, params.item, params.config.ecosystem);
-  });
-  return makeCaller(undefined);
+    let inMemTable = InMemoryStore.getEffectInMemTable(params.indexerState, effect, scope);
+    if (!InMemoryStore.hasEffectOutput(inMemTable, effectArgs.cacheKey)) {
+      return scheduleAndSuspend(params, LoadLayer.loadEffect(params.loadManager, params.persistence, effect, effectArgs, scope, params.indexerState, params.isPreload, params.item, params.config.ecosystem));
+    }
+    let output$1 = InMemoryStore.getEffectOutputUnsafe(inMemTable, effectArgs.cacheKey);
+    memo[memoKey] = output$1;
+    return output$1;
+  };
 }
 
 function entityScope(params) {
@@ -97,24 +201,83 @@ function throwClickHouseReadOnly(entityConfig, op) {
   return Stdlib_JsError.throwWithMessage(`context.` + entityConfig.name + `.` + op + `() is unavailable: ClickHouse storage is currently write-only. Follow Envio releases to be notified when ClickHouse supports both reads and writes from handlers.`);
 }
 
+function getSyncHandler(params, entityId) {
+  let inMemTable = InMemoryStore.getInMemTable(params.indexerState, params.entityConfig, entityScope(params));
+  if (entityId in inMemTable.latestEntityChangeById) {
+    return InMemoryTable.Entity.getUnsafe(inMemTable)(entityId);
+  } else {
+    return scheduleAndSuspend(params, LoadLayer.loadById(params.loadManager, params.persistence, params.entityConfig, entityScope(params), params.indexerState, params.isPreload, params.item, params.config.ecosystem, entityId));
+  }
+}
+
+function getWhereSyncHandler(params, filter) {
+  let entityConfig = params.entityConfig;
+  let inMemTable = InMemoryStore.getInMemTable(params.indexerState, entityConfig, entityScope(params));
+  let hasIndex = InMemoryTable.Entity.hasIndex(inMemTable);
+  let getOnIndex = InMemoryTable.Entity.getUnsafeOnIndex(inMemTable);
+  let filters = EntityFilter.parseGetWhereOrThrow(filter, entityConfig.name, entityConfig.table);
+  let missing = [];
+  let entities = [];
+  filters.forEach(filter => {
+    let filterKey = EntityFilter.toString(filter);
+    if (hasIndex(filterKey)) {
+      entities.push(...getOnIndex(filterKey));
+    } else {
+      missing.push(LoadLayer.loadByFilter(params.loadManager, params.persistence, entityConfig, entityScope(params), params.indexerState, params.isPreload, params.item, params.config.ecosystem, filter));
+    }
+  });
+  if (Utils.$$Array.notEmpty(missing)) {
+    let pending = getPending(params.sync);
+    missing.forEach(promise => {
+      pending.push(promise);
+    });
+    params.sync.status = {
+      TAG: "Aborted",
+      _0: {
+        RE_EXN_ID: Suspend
+      }
+    };
+    throw {
+      RE_EXN_ID: Suspend,
+      Error: new Error()
+    };
+  }
+  return entities;
+}
+
+function getInBlockSyncHandler(params, entityId) {
+  let inMemTable = InMemoryStore.getInMemTable(params.indexerState, params.entityConfig, entityScope(params));
+  let change = inMemTable.latestEntityChangeById[entityId];
+  if (change !== undefined && change.checkpointId === params.checkpointId) {
+    return InMemoryTable.Entity.mapChangeToEntity(change);
+  }
+}
+
 let entityTraps_get = (params, prop) => {
+  checkStatusOrThrow(params, `context.` + params.entityConfig.name + `.` + prop);
   let isClickHouseOnly = !params.entityConfig.storage.postgres;
-  let set = params.isPreload ? noopSet : entity => InMemoryTable.Entity.set(InMemoryStore.getInMemTable(params.indexerState, params.entityConfig, entityScope(params)), IndexerState.committedCheckpointId(params.indexerState), {
-      type: "SET",
-      entityId: entity.id,
-      entity: entity,
-      checkpointId: params.checkpointId
-    });
+  let set = params.isPreload ? noopSet : entity => {
+      checkStatusOrThrow(params, `context.` + params.entityConfig.name + `.set`);
+      InMemoryTable.Entity.set(InMemoryStore.getInMemTable(params.indexerState, params.entityConfig, entityScope(params)), IndexerState.committedCheckpointId(params.indexerState), {
+        type: "SET",
+        entityId: entity.id,
+        entity: entity,
+        checkpointId: params.checkpointId
+      });
+    };
   switch (prop) {
     case "deleteUnsafe" :
       if (params.isPreload) {
         return noopDeleteUnsafe;
       } else {
-        return entityId => InMemoryTable.Entity.set(InMemoryStore.getInMemTable(params.indexerState, params.entityConfig, entityScope(params)), IndexerState.committedCheckpointId(params.indexerState), {
-          type: "DELETE",
-          entityId: entityId,
-          checkpointId: params.checkpointId
-        });
+        return entityId => {
+          checkStatusOrThrow(params, `context.` + params.entityConfig.name + `.deleteUnsafe`);
+          InMemoryTable.Entity.set(InMemoryStore.getInMemTable(params.indexerState, params.entityConfig, entityScope(params)), IndexerState.committedCheckpointId(params.indexerState), {
+            type: "DELETE",
+            entityId: entityId,
+            checkpointId: params.checkpointId
+          });
+        };
       }
     case "get" :
       if (isClickHouseOnly) {
@@ -122,6 +285,8 @@ let entityTraps_get = (params, prop) => {
       } else {
         return entityId => LoadLayer.loadById(params.loadManager, params.persistence, params.entityConfig, entityScope(params), params.indexerState, params.isPreload, params.item, params.config.ecosystem, entityId);
       }
+    case "getInBlockSync" :
+      return entityId => getInBlockSyncHandler(params, entityId);
     case "getOrCreate" :
       if (isClickHouseOnly) {
         return _entity => throwClickHouseReadOnly(params.entityConfig, "getOrCreate");
@@ -147,12 +312,16 @@ let entityTraps_get = (params, prop) => {
           }
         });
       }
+    case "getSync" :
+      return entityId => getSyncHandler(params, entityId);
     case "getWhere" :
       if (isClickHouseOnly) {
         return _filter => throwClickHouseReadOnly(params.entityConfig, "getWhere");
       } else {
         return filter => getWhereHandler(params, filter);
       }
+    case "getWhereSync" :
+      return filter => getWhereSyncHandler(params, filter);
     case "set" :
       return set;
     default:
@@ -164,16 +333,63 @@ let entityTraps = {
   get: entityTraps_get
 };
 
-let handlerTraps_get = (params, prop) => {
-  if (params.isResolved) {
-    ErrorHandling.mkLogAndRaise(Ecosystem.getItemLogger(params.item, params.config.ecosystem), undefined, new Error(`Impossible to access context.` + prop + ` after the handler is resolved. Make sure you didn't miss an await in the handler.`));
+async function runSyncRound(params, fn, round) {
+  if (round > 10000) {
+    Stdlib_JsError.throwWithMessage(`The handler suspended on a synchronous read too many times: gave up after ` + (10000).toString() + ` rounds. This usually means the code isn't deterministic across reruns.`);
+  }
+  params.sync.status = "Active";
+  params.sync.pending = undefined;
+  let suspended;
+  try {
+    fn();
+    suspended = false;
+  } catch (raw_exn) {
+    let exn = Primitive_exceptions.internalToException(raw_exn);
+    if (exn.RE_EXN_ID === Suspend) {
+      suspended = true;
+    } else {
+      let pending = params.sync.pending;
+      if (pending !== undefined) {
+        params.sync.pending = undefined;
+        pending.forEach(promise => {
+          Utils.$$Promise.silentCatch(promise);
+        });
+      }
+      throw exn;
+    }
   }
+  let pending$1 = params.sync.pending;
+  if (pending$1 === undefined) {
+    return;
+  }
+  params.sync.pending = undefined;
+  if (suspended) {
+    let errors = [];
+    await Promise.all(pending$1.map(promise => Stdlib_Promise.$$catch(promise, exn => {
+      errors.push(exn);
+      return Promise.resolve();
+    })));
+    let exn$1 = errors[0];
+    if (exn$1 !== undefined) {
+      throw exn$1;
+    }
+    return await runSyncRound(params, fn, round + 1 | 0);
+  }
+  pending$1.forEach(promise => {
+    Utils.$$Promise.silentCatch(promise);
+  });
+}
+
+let handlerTraps_get = (params, prop) => {
+  checkStatusOrThrow(params, `context.` + prop);
   switch (prop) {
     case "chain" :
       let chainId = Internal.getItemChainId(params.item);
       return params.chains[chainId];
     case "effect" :
       return initEffect(params);
+    case "effectSync" :
+      return initEffectSync(params);
     case "isPreload" :
       return params.isPreload;
     case "log" :
@@ -182,6 +398,8 @@ let handlerTraps_get = (params, prop) => {
       } else {
         return Ecosystem.getItemUserLogger(params.item, params.config.ecosystem);
       }
+    case "runSync" :
+      return fn => runSyncRound(params, fn, 1);
     default:
       let entityConfig = params.config.userEntitiesByName[prop];
       if (entityConfig !== undefined) {
@@ -194,7 +412,7 @@ let handlerTraps_get = (params, prop) => {
           isPreload: params.isPreload,
           chains: params.chains,
           config: params.config,
-          isResolved: params.isResolved,
+          sync: params.sync,
           entityConfig: entityConfig
         }, entityTraps);
       } else {
@@ -211,16 +429,34 @@ function getHandlerContext(params) {
   return new Proxy(params, handlerTraps);
 }
 
+let maxSyncRounds = 10000;
+
 export {
+  Suspend,
+  isSuspend,
+  makeSyncState,
+  getPending,
+  getMemo,
+  checkStatusOrThrow,
+  scheduleAndSuspend,
   paramsByThis,
   effectContextPrototype,
+  prepareEffectCall,
+  callEffectAsync,
   initEffect,
+  effectMemoKey,
+  initEffectSync,
   entityScope,
   getWhereHandler,
   noopSet,
   noopDeleteUnsafe,
   throwClickHouseReadOnly,
+  getSyncHandler,
+  getWhereSyncHandler,
+  getInBlockSyncHandler,
   entityTraps,
+  maxSyncRounds,
+  runSyncRound,
   handlerTraps,
   getHandlerContext,
 }

package/src/Utils.res

@@ -576,6 +576,9 @@ module Set = {
   @send
   external intersection: (t<'value>, t<'value>) => t<'value> = "intersection"
 
+  @send
+  external union: (t<'value>, t<'value>) => t<'value> = "union"
+
   let immutableAdd: (t<'a>, 'a) => t<'a> = %raw(`(set, value) => {
     return new Set([...set, value])
   }`)

package/src/bindings/ClickHouse.res

@@ -14,8 +14,13 @@ type execParams = {query: string}
 @module("@clickhouse/client")
 external createClient: clientConfig => client = "createClient"
 
+// `command`, not `exec`: exec hands its response stream to the caller and holds
+// the socket until it is consumed, which nothing here does. The pool is 10
+// sockets wide, so a schema whose DDL needs more than that — initialize issues
+// 2N+3 statements for N entities — stalled every statement past the tenth until
+// the 30s request timeout freed one. command destroys the stream for us.
 @send
-external exec: (client, execParams) => promise<unit> = "exec"
+external command: (client, execParams) => promise<unit> = "command"
 
 @send
 external close: client => promise<unit> = "close"
@@ -628,21 +633,21 @@ let initialize = async (
       // CLUSTER removes the database from every node — the engine's own log
       // can't replicate the drop of the database it lives in — and SYNC waits
       // for the drop to finish before the CREATE below.
-      await client->exec({
+      await client->command({
         query: `DROP DATABASE IF EXISTS ${database} ON CLUSTER '{cluster}' SYNC`,
       })
     } else {
-      await client->exec({
+      await client->command({
         query: `TRUNCATE DATABASE IF EXISTS ${database}${onClusterClause(~onCluster=ddlOnCluster)}`,
       })
     }
-    await client->exec({
+    await client->command({
       query: `CREATE DATABASE IF NOT EXISTS ${database}${databaseOnClusterClause}${databaseEngineClause}`,
     })
 
     await Promise.all(
       entities->Array.map(entityConfig =>
-        client->exec({
+        client->command({
           query: makeCreateHistoryTableQuery(
             ~entityConfig,
             ~database,
@@ -653,7 +658,7 @@ let initialize = async (
         })
       ),
     )->Utils.Promise.ignoreValue
-    await client->exec({
+    await client->command({
       query: makeCreateCheckpointsTableQuery(
         ~database,
         ~replicated,
@@ -670,14 +675,14 @@ let initialize = async (
     // caught up before creating the views. ON CLUSTER must precede the
     // database name in this command's grammar.
     if hasReplicatedDatabaseEngine {
-      await client->exec({
+      await client->command({
         query: `SYSTEM SYNC DATABASE REPLICA ON CLUSTER '{cluster}' ${database}`,
       })
     }
 
     await Promise.all(
       entities->Array.map(entityConfig =>
-        client->exec({
+        client->command({
           query: makeCreateViewQuery(~entityConfig, ~database, ~onCluster=ddlOnCluster),
         })
       ),
@@ -697,7 +702,7 @@ let resume = async (client, ~database: string, ~checkpointId: Internal.checkpoin
   try {
     // Try to use the database - will throw if it doesn't exist
     try {
-      await client->exec({query: `USE ${database}`})
+      await client->command({query: `USE ${database}`})
     } catch {
     | exn =>
       Logging.errorWithExn(
@@ -717,14 +722,14 @@ let resume = async (client, ~database: string, ~checkpointId: Internal.checkpoin
     await Promise.all(
       tables->Array.map(table => {
         let tableName = table["name"]
-        client->exec({
+        client->command({
           query: `ALTER TABLE ${database}.\`${tableName}\` DELETE WHERE \`${EntityHistory.checkpointIdFieldName}\` > ${checkpointId->BigInt.toString}`,
         })
       }),
     )->Utils.Promise.ignoreValue
 
     // Delete stale checkpoints
-    await client->exec({
+    await client->command({
       query: `DELETE FROM ${database}.\`${InternalTable.Checkpoints.table.tableName}\` WHERE \`${Table.idFieldName}\` > ${checkpointId->BigInt.toString}`,
     })
   } catch {

package/src/bindings/ClickHouse.res.mjs

@@ -428,31 +428,31 @@ async function initialize(client, database, entities, param, chainIdModeOpt) {
       }
     }
     if (hasReplicatedDatabaseEngine) {
-      await client.exec({
+      await client.command({
         query: `DROP DATABASE IF EXISTS ` + database + ` ON CLUSTER '{cluster}' SYNC`
       });
     } else {
-      await client.exec({
+      await client.command({
         query: `TRUNCATE DATABASE IF EXISTS ` + database + (
           ddlOnCluster ? ` ON CLUSTER '{cluster}'` : ""
         )
       });
     }
-    await client.exec({
+    await client.command({
       query: `CREATE DATABASE IF NOT EXISTS ` + database + databaseOnClusterClause + databaseEngineClause
     });
-    await Promise.all(entities.map(entityConfig => client.exec({
+    await Promise.all(entities.map(entityConfig => client.command({
       query: makeCreateHistoryTableQuery(entityConfig, database, replicated, ddlOnCluster, chainIdMode)
     })));
-    await client.exec({
+    await client.command({
       query: makeCreateCheckpointsTableQuery(database, replicated, ddlOnCluster, chainIdMode)
     });
     if (hasReplicatedDatabaseEngine) {
-      await client.exec({
+      await client.command({
         query: `SYSTEM SYNC DATABASE REPLICA ON CLUSTER '{cluster}' ` + database
       });
     }
-    await Promise.all(entities.map(entityConfig => client.exec({
+    await Promise.all(entities.map(entityConfig => client.command({
       query: makeCreateViewQuery(entityConfig, database, ddlOnCluster)
     })));
     return Logging.trace("ClickHouse storage initialization completed successfully");
@@ -466,7 +466,7 @@ async function initialize(client, database, entities, param, chainIdModeOpt) {
 async function resume(client, database, checkpointId) {
   try {
     try {
-      await client.exec({
+      await client.command({
         query: `USE ` + database
       });
     } catch (raw_exn) {
@@ -480,11 +480,11 @@ async function resume(client, database, checkpointId) {
     let tables = (await tablesResult.json()).data;
     await Promise.all(tables.map(table => {
       let tableName = table.name;
-      return client.exec({
+      return client.command({
         query: `ALTER TABLE ` + database + `.\`` + tableName + `\` DELETE WHERE \`` + EntityHistory.checkpointIdFieldName + `\` > ` + checkpointId.toString()
       });
     }));
-    return await client.exec({
+    return await client.command({
       query: `DELETE FROM ` + database + `.\`` + InternalTable.Checkpoints.table.tableName + `\` WHERE \`` + Table.idFieldName + `\` > ` + checkpointId.toString()
     });
   } catch (raw_exn$1) {

package/src/bindings/Vitest.res

@@ -98,7 +98,7 @@ external afterEach: (unit => unit) => unit = "afterEach"
 // Async Module
 // ============================================================================
 
-type options = {retry?: int}
+type options = {retry?: int, timeout?: int}
 
 module Async = {
   @module("vitest")