envio 3.6.0 → 3.6.1-subgraph

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "envio",
-  "version": "3.6.0",
+  "version": "3.6.1-subgraph",
   "type": "module",
   "description": "A latency and sync speed optimized, developer friendly blockchain data indexer.",
   "bin": "./bin.mjs",
@@ -69,10 +69,10 @@
     "tsx": "4.21.0"
   },
   "optionalDependencies": {
-    "envio-linux-x64": "3.6.0",
-    "envio-linux-x64-musl": "3.6.0",
-    "envio-linux-arm64": "3.6.0",
-    "envio-darwin-x64": "3.6.0",
-    "envio-darwin-arm64": "3.6.0"
+    "envio-linux-x64": "3.6.1-subgraph",
+    "envio-linux-x64-musl": "3.6.1-subgraph",
+    "envio-linux-arm64": "3.6.1-subgraph",
+    "envio-darwin-x64": "3.6.1-subgraph",
+    "envio-darwin-arm64": "3.6.1-subgraph"
   }
 }

package/src/ChainState.res

@@ -549,10 +549,16 @@ let targetBlock = (cs: t, ~chainTargetItems: float) => {
   let bufferBlockNumber = fetchState->FetchState.bufferBlockNumber
   switch cs->effectiveDensity {
   | Some(density) if density > 0. =>
-    Pervasives.min(
-      fetchCeiling,
-      bufferBlockNumber + Math.ceil(chainTargetItems /. density)->Float.toInt,
-    )
+    // Decided by comparison so no unbounded value is ever converted to int:
+    // at low densities chainTargetItems /. density exceeds the int range, and
+    // truncating it wraps negative — the target collapses below the frontier
+    // and the chain stops querying. The division only runs when its result is
+    // provably below the ceiling-bounded range.
+    if density *. (fetchCeiling - bufferBlockNumber)->Int.toFloat <= chainTargetItems {
+      fetchCeiling
+    } else {
+      bufferBlockNumber + Math.ceil(chainTargetItems /. density)->Float.toInt
+    }
   | _ => Pervasives.min(bufferBlockNumber + coldTargetRange, fetchCeiling)
   }
 }

package/src/ChainState.res.mjs

@@ -368,7 +368,11 @@ function targetBlock(cs, chainTargetItems) {
   let bufferBlockNumber = FetchState.bufferBlockNumber(fetchState);
   let density = effectiveDensity(cs);
   if (density !== undefined && density > 0) {
-    return Primitive_int.min(fetchCeiling$1, bufferBlockNumber + (Math.ceil(chainTargetItems / density) | 0) | 0);
+    if (density * (fetchCeiling$1 - bufferBlockNumber | 0) <= chainTargetItems) {
+      return fetchCeiling$1;
+    } else {
+      return bufferBlockNumber + (Math.ceil(chainTargetItems / density) | 0) | 0;
+    }
   } else {
     return Primitive_int.min(bufferBlockNumber + 20000 | 0, fetchCeiling$1);
   }

package/src/Config.res

@@ -119,6 +119,9 @@ type t = {
   userEntities: array<Internal.entityConfig>,
   allEntities: array<Internal.entityConfig>,
   allEnums: array<Table.enumConfig<Table.enum>>,
+  // Set only in subgraph mode: the translated manifest, which is what makes
+  // the subgraph runtime take over handler registration.
+  subgraph: option<JSON.t>,
 }
 
 module EnvioAddresses = {
@@ -613,6 +616,7 @@ let publicConfigSchema = S.schema(s =>
     "svm": s.matches(S.option(publicConfigEcosystemSchema)),
     "enums": s.matches(S.option(S.dict(S.array(S.string)))),
     "entities": s.matches(S.option(S.array(entityJsonSchema))),
+    "subgraph": s.matches(S.option(S.json(~validate=false))),
   }
 )
 
@@ -1110,6 +1114,7 @@ let fromPublic = (publicConfigJson: JSON.t) => {
     userEntities,
     allEntities,
     allEnums,
+    subgraph: publicConfig["subgraph"],
   }
 }
 
@@ -1246,6 +1251,13 @@ let stripSensitiveData = (json: JSON.t): JSON.t => {
       stripChains(obj->Dict.get("evm"))
       stripChains(obj->Dict.get("fuel"))
       stripChains(obj->Dict.get("svm"))
+      // A subgraph names its endpoint through ENVIO_SUBGRAPH_RPC, and those
+      // URLs usually carry a provider key. Rotating one must not read as an
+      // incompatible config either.
+      switch obj->Dict.get("subgraph") {
+      | Some(Object(subgraph)) => subgraph->Utils.Dict.deleteInPlace("rpcUrls")
+      | _ => ()
+      }
     }
   | _ => ()
   }

package/src/Config.res.mjs

@@ -433,7 +433,8 @@ let publicConfigSchema = S$RescriptSchema.schema(s => ({
   fuel: s.m(S$RescriptSchema.option(publicConfigEcosystemSchema)),
   svm: s.m(S$RescriptSchema.option(publicConfigEcosystemSchema)),
   enums: s.m(S$RescriptSchema.option(S$RescriptSchema.dict(S$RescriptSchema.array(S$RescriptSchema.string)))),
-  entities: s.m(S$RescriptSchema.option(S$RescriptSchema.array(entityJsonSchema)))
+  entities: s.m(S$RescriptSchema.option(S$RescriptSchema.array(entityJsonSchema))),
+  subgraph: s.m(S$RescriptSchema.option(S$RescriptSchema.json(false)))
 }));
 
 function fromPublic(publicConfigJson) {
@@ -757,7 +758,8 @@ function fromPublic(publicConfigJson) {
     userEntitiesByName: userEntitiesByName,
     userEntities: userEntities,
     allEntities: allEntities,
-    allEnums: allEnums
+    allEnums: allEnums,
+    subgraph: publicConfig.subgraph
   };
 }
 
@@ -902,6 +904,10 @@ function stripSensitiveData(json) {
     stripChains(cloned["evm"]);
     stripChains(cloned["fuel"]);
     stripChains(cloned["svm"]);
+    let match = cloned["subgraph"];
+    if (typeof match === "object" && match !== null && !Array.isArray(match)) {
+      Utils.Dict.deleteInPlace(match, "rpcUrls");
+    }
   }
   return cloned;
 }

package/src/Core.res

@@ -24,8 +24,16 @@ type fromUserApiResult = {
   indexerCode: Null.t<string>,
 }
 
+type fromSubgraphOptions = {
+  name?: string,
+  env?: dict<string>,
+  files?: dict<string>,
+  root?: string,
+}
+
 type addon = {
   getConfigJson: (~configPath: Null.t<string>, ~directory: Null.t<string>) => string,
+  fromSubgraph: (string, string, fromSubgraphOptions) => fromUserApiResult,
   encodeIndexedTopic: (~abiType: string, ~value: unknown) => EvmTypes.Hex.t,
   fromUserApi: (string, fromUserApiOptions) => fromUserApiResult,
   runCli: (~args: array<string>, ~envioPackageDir: Null.t<string>) => promise<Null.t<string>>,
@@ -232,6 +240,13 @@ let fromUserApi = (~schema=?, ~env=?, ~files=?, ~withIndexerTypes=false, yaml) =
   )
 }
 
+/// Parses a subgraph project without touching the filesystem: subgraph.yaml and
+/// its schema inline, ABI bodies through `files`.
+let fromSubgraph = (~name=?, ~env=?, ~files=?, ~root=?, ~manifest, ~schema) => {
+  let addon = getAddon()
+  addon.fromSubgraph(manifest, schema, {?name, ?env, ?files, ?root})
+}
+
 let runCli = args => {
   let addon = getAddon()
   addon.runCli(~args, ~envioPackageDir=Null.make(envioPackageDir))

package/src/Core.res.mjs

@@ -191,6 +191,16 @@ function fromUserApi(schema, env, files, withIndexerTypesOpt, yaml) {
   });
 }
 
+function fromSubgraph(name, env, files, root, manifest, schema) {
+  let addon = getAddon();
+  return addon.fromSubgraph(manifest, schema, {
+    name: name,
+    env: env,
+    files: files,
+    root: root
+  });
+}
+
 function runCli(args) {
   let addon = getAddon();
   return addon.runCli(args, envioPackageDir);
@@ -208,6 +218,7 @@ export {
   getAddon,
   getConfigJson,
   fromUserApi,
+  fromSubgraph,
   runCli,
 }
 /* envioPackageDir Not a pure module */

package/src/EventProcessing.res

@@ -51,7 +51,7 @@ let runEventHandlerOrThrow = async (
       isPreload: false,
       chains,
       config,
-      isResolved: false,
+      sync: UserContext.makeSyncState(),
     }
     await handler(
       (
@@ -61,7 +61,7 @@ let runEventHandlerOrThrow = async (
         }: Internal.handlerArgs
       ),
     )
-    contextParams.isResolved = true
+    contextParams.sync.status = Resolved
   } catch {
   | exn =>
     throw(
@@ -102,7 +102,7 @@ let runHandlerOrThrow = async (
         isPreload: false,
         chains,
         config,
-        isResolved: false,
+        sync: UserContext.makeSyncState(),
       }
       await handler(
         Ecosystem.makeOnBlockArgs(
@@ -111,7 +111,7 @@ let runHandlerOrThrow = async (
           ~context=UserContext.getHandlerContext(contextParams),
         ),
       )
-      contextParams.isResolved = true
+      contextParams.sync.status = Resolved
     } catch {
     | exn =>
       throw(
@@ -186,7 +186,7 @@ let preloadBatchOrThrow = async (
                   checkpointId,
                   isPreload: true,
                   chains,
-                  isResolved: false,
+                  sync: UserContext.makeSyncState(),
                   config,
                 }),
               })
@@ -221,7 +221,7 @@ let preloadBatchOrThrow = async (
                   checkpointId,
                   isPreload: true,
                   chains,
-                  isResolved: false,
+                  sync: UserContext.makeSyncState(),
                   config,
                 }),
               )

package/src/EventProcessing.res.mjs

@@ -38,6 +38,7 @@ let ProcessingError = /* @__PURE__ */Primitive_exceptions.create("EventProcessin
 async function runEventHandlerOrThrow(item, checkpointId, handler, indexerState, loadManager, persistence, chains, config) {
   let timeBeforeHandler = Performance.now();
   try {
+    let contextParams_sync = UserContext.makeSyncState();
     let contextParams = {
       item: item,
       checkpointId: checkpointId,
@@ -47,13 +48,13 @@ async function runEventHandlerOrThrow(item, checkpointId, handler, indexerState,
       isPreload: false,
       chains: chains,
       config: config,
-      isResolved: false
+      sync: contextParams_sync
     };
     await handler({
       event: Ecosystem.getItemEvent(item, config.ecosystem),
       context: UserContext.getHandlerContext(contextParams)
     });
-    contextParams.isResolved = true;
+    contextParams_sync.status = "Resolved";
   } catch (raw_exn) {
     let exn = Primitive_exceptions.internalToException(raw_exn);
     throw {
@@ -79,6 +80,7 @@ async function runHandlerOrThrow(item, checkpointId, indexerState, loadManager,
     }
   }
   try {
+    let contextParams_sync = UserContext.makeSyncState();
     let contextParams = {
       item: item,
       checkpointId: checkpointId,
@@ -88,10 +90,10 @@ async function runHandlerOrThrow(item, checkpointId, indexerState, loadManager,
       isPreload: false,
       chains: chains,
       config: config,
-      isResolved: false
+      sync: contextParams_sync
     };
     await item.onBlockRegistration.handler(Ecosystem.makeOnBlockArgs(item.blockNumber, config.ecosystem, UserContext.getHandlerContext(contextParams)));
-    contextParams.isResolved = true;
+    contextParams_sync.status = "Resolved";
     return;
   } catch (raw_exn) {
     let exn = Primitive_exceptions.internalToException(raw_exn);
@@ -133,7 +135,7 @@ async function preloadBatchOrThrow(batch, loadManager, persistence, config, inde
                 isPreload: true,
                 chains: chains,
                 config: config,
-                isResolved: false
+                sync: UserContext.makeSyncState()
               })
             }).then(() => IndexerState.endPreloadHandler(indexerState, timerRef, contractName, eventName))));
           } catch (exn) {
@@ -151,7 +153,7 @@ async function preloadBatchOrThrow(batch, loadManager, persistence, config, inde
             isPreload: true,
             chains: chains,
             config: config,
-            isResolved: false
+            sync: UserContext.makeSyncState()
           })))));
         } catch (exn$1) {
           

package/src/HandlerLoader.res

@@ -77,18 +77,30 @@ let autoLoadFromSrcHandlers = async (~handlers: string) => {
 // `HandlerRegister.finishRegistration`. This loads the user handler files
 // (populating the global `HandlerRegister` registry as a side effect) and
 // returns the resulting per-chain registrations.
+// The subgraph runtime lives in this package but is never exported: it's
+// reached only from here, when the resolved config carries a translated
+// manifest.
+let registerSubgraph: (JSON.t, ~isDev: bool) => promise<unit> = %raw(`(subgraph, isDev) =>
+  import("./subgraph/runtime.ts").then((m) => m.registerSubgraph({ ...subgraph, isDev }))`)
+
 let registerAllHandlers = async (~config: Config.t): HandlerRegister.registrationsByChainId => {
   HandlerRegister.startRegistration(~config)
 
-  // Auto-load all .js files from src/handlers directory
-  await autoLoadFromSrcHandlers(~handlers=config.handlers)
+  switch config.subgraph {
+  // A subgraph project's `src/` holds AssemblyScript mappings, not envio
+  // handlers, so the usual auto-load would import them without a scope.
+  | Some(subgraph) => await registerSubgraph(subgraph, ~isDev=config.isDev)
+  | None =>
+    // Auto-load all .js files from src/handlers directory
+    await autoLoadFromSrcHandlers(~handlers=config.handlers)
 
-  // Load contract-specific handlers
-  let _ = await config.contractHandlers
-  ->Array.map(({name, handler}) => {
-    registerContractHandlers(~contractName=name, ~handler)
-  })
-  ->Promise.all
+    // Load contract-specific handlers
+    let _ = await config.contractHandlers
+    ->Array.map(({name, handler}) => {
+      registerContractHandlers(~contractName=name, ~handler)
+    })
+    ->Promise.all
+  }
 
   HandlerRegister.finishRegistration(~config)
 }

package/src/HandlerLoader.res.mjs

@@ -56,10 +56,18 @@ async function autoLoadFromSrcHandlers(handlers) {
   })));
 }
 
+let registerSubgraph = ((subgraph, isDev) =>
+  import("./subgraph/runtime.ts").then((m) => m.registerSubgraph({ ...subgraph, isDev })));
+
 async function registerAllHandlers(config) {
   HandlerRegister.startRegistration(config);
-  await autoLoadFromSrcHandlers(config.handlers);
-  await Promise.all(config.contractHandlers.map(param => registerContractHandlers(param.name, param.handler)));
+  let subgraph = config.subgraph;
+  if (subgraph !== undefined) {
+    await registerSubgraph(subgraph, config.isDev);
+  } else {
+    await autoLoadFromSrcHandlers(config.handlers);
+    await Promise.all(config.contractHandlers.map(param => registerContractHandlers(param.name, param.handler)));
+  }
   return HandlerRegister.finishRegistration(config);
 }
 
@@ -67,6 +75,7 @@ export {
   toImportUrl,
   registerContractHandlers,
   autoLoadFromSrcHandlers,
+  registerSubgraph,
   registerAllHandlers,
 }
 /*  Not a pure module */

package/src/Metrics.res

@@ -445,15 +445,15 @@ let renderMetrics = (b: builder, metrics: t) => {
     ~entries=chains,
     ~value=m => m.endBlock->Option.map(Int.toFloat),
   )
-  b->series(
+  b->seriesOpt(
     ~name="envio_source_request_total",
-    ~help="The number of requests made to data sources.",
+    ~help="The number of requests made to data sources. Heights pushed by a subscription stream are counted here too, under the heightPush and heightPushIgnored methods.",
     ~kind="counter",
     ~entries=sourceRequests,
-    ~value=s => s.count->Int.toFloat,
+    ~value=s => s.count > 0 ? Some(s.count->Int.toFloat) : None,
   )
-  // Skips a method's seconds line when it has no timing (e.g. heightSubscription,
-  // which only ever records a count).
+  // Skips a method's seconds line when it has no timing (e.g. heightPush, which
+  // only ever records a count).
   b->seriesOpt(
     ~name="envio_source_request_seconds_total",
     ~help="Cumulative time spent on data source requests.",

package/src/Metrics.res.mjs

@@ -144,7 +144,11 @@ function renderMetrics(b, metrics) {
   single(b, "envio_indexing_target_buffer_size", "The indexer-wide target buffer size shared across all chains. The actual number of items in the queue may exceed this value, but the indexer always tries to keep the buffer filled up to this target.", "gauge", metrics.targetBufferSize);
   series(b, "envio_indexing_buffer_block", "The highest block number that has been fully fetched by the indexer.", "gauge", chains, m => m.bufferBlockNumber);
   seriesOpt(b, "envio_indexing_end_block", "The block number to stop indexing at. (inclusive)", "gauge", chains, m => Stdlib_Option.map(m.endBlock, prim => prim));
-  series(b, "envio_source_request_total", "The number of requests made to data sources.", "counter", sourceRequests, s => s.count);
+  seriesOpt(b, "envio_source_request_total", "The number of requests made to data sources. Heights pushed by a subscription stream are counted here too, under the heightPush and heightPushIgnored methods.", "counter", sourceRequests, s => {
+    if (s.count > 0) {
+      return s.count;
+    }
+  });
   seriesOpt(b, "envio_source_request_seconds_total", "Cumulative time spent on data source requests.", "counter", sourceRequests, s => {
     if (s.seconds !== 0) {
       return s.seconds;