envio 3.3.2 → 3.5.0-alpha.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "envio",
-  "version": "3.3.2",
+  "version": "3.5.0-alpha.0",
   "type": "module",
   "description": "A latency and sync speed optimized, developer friendly blockchain data indexer.",
   "bin": "./bin.mjs",
@@ -69,10 +69,10 @@
     "tsx": "4.21.0"
   },
   "optionalDependencies": {
-    "envio-linux-x64": "3.3.2",
-    "envio-linux-x64-musl": "3.3.2",
-    "envio-linux-arm64": "3.3.2",
-    "envio-darwin-x64": "3.3.2",
-    "envio-darwin-arm64": "3.3.2"
+    "envio-linux-x64": "3.5.0-alpha.0",
+    "envio-linux-x64-musl": "3.5.0-alpha.0",
+    "envio-linux-arm64": "3.5.0-alpha.0",
+    "envio-darwin-x64": "3.5.0-alpha.0",
+    "envio-darwin-arm64": "3.5.0-alpha.0"
   }
 }

package/src/Api.res

@@ -4,12 +4,5 @@
 let indexer: unknown = Main.getGlobalIndexer()
 
 let createTestIndexer: unit => unknown = () => {
-  let workerPath =
-    NodeJs.Path.join(
-      NodeJs.Path.getDirname(NodeJs.ImportMeta.importMeta),
-      "TestIndexerWorker.res.mjs",
-    )->NodeJs.Path.toString
-  TestIndexer.makeCreateTestIndexer(~config=Config.load(), ~workerPath)()->(
-    Utils.magic: TestIndexer.t<'a> => unknown
-  )
+  TestIndexer.createTestIndexer()->(Utils.magic: TestIndexer.t<'a> => unknown)
 }

package/src/Api.res.mjs

@@ -1,16 +1,12 @@
 // Generated by ReScript, PLEASE EDIT WITH CARE
 
 import * as Main from "./Main.res.mjs";
-import * as Path from "path";
-import * as Config from "./Config.res.mjs";
-import * as NodeJs from "./bindings/NodeJs.res.mjs";
 import * as TestIndexer from "./TestIndexer.res.mjs";
 
 let indexer = Main.getGlobalIndexer();
 
 function createTestIndexer() {
-  let workerPath = Path.join(NodeJs.Path.getDirname(import.meta), "TestIndexerWorker.res.mjs");
-  return TestIndexer.makeCreateTestIndexer(Config.load(), workerPath)();
+  return TestIndexer.createTestIndexer();
 }
 
 export {

package/src/Bin.res

@@ -52,8 +52,8 @@ let applyEnv = (env: dict<JSON.t>) =>
 let run = async args => {
   try {
     switch (await Core.runCli(args))->Null.toOption {
-    // Rust-only command (codegen / init / stop / docker / metrics / help /
-    // version / scripts) — nothing for JS to do, exit cleanly.
+    // Rust-only command (codegen / init / serve / stop / docker / metrics /
+    // help / version / scripts) — nothing for JS to do, exit cleanly.
     | None => ()
     | Some(json) =>
       switch decodeCommand(json->JSON.parseOrThrow) {

package/src/ChainState.res

@@ -224,7 +224,12 @@ let makeInternal = (
       ~lowercaseAddresses,
     )
   | Config.FuelSourceConfig({hypersync}) => [
-      HyperFuelSource.make({chain, endpointUrl: hypersync, apiToken: Env.envioApiToken}),
+      FuelHyperSyncSource.make({
+        chain,
+        endpointUrl: hypersync,
+        apiToken: Env.envioApiToken,
+        onEventRegistrations,
+      }),
     ]
   | Config.SvmSourceConfig({hypersync, rpc}) =>
     switch (hypersync, rpc) {

package/src/ChainState.res.mjs

@@ -17,12 +17,12 @@ import * as SourceManager from "./sources/SourceManager.res.mjs";
 import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js";
 import * as ReorgDetection from "./ReorgDetection.res.mjs";
 import * as Stdlib_JsError from "@rescript/runtime/lib/es6/Stdlib_JsError.js";
-import * as HyperFuelSource from "./sources/HyperFuelSource.res.mjs";
 import * as Primitive_float from "@rescript/runtime/lib/es6/Primitive_float.js";
 import * as Primitive_option from "@rescript/runtime/lib/es6/Primitive_option.js";
 import * as TransactionStore from "./sources/TransactionStore.res.mjs";
 import * as IndexingAddresses from "./IndexingAddresses.res.mjs";
 import * as SvmHyperSyncSource from "./sources/SvmHyperSyncSource.res.mjs";
+import * as FuelHyperSyncSource from "./sources/FuelHyperSyncSource.res.mjs";
 import * as SafeCheckpointTracking from "./SafeCheckpointTracking.res.mjs";
 
 function configAddresses(chainConfig) {
@@ -132,10 +132,11 @@ function makeInternal(chainConfig, indexingAddresses, startBlock, endBlock, firs
       sources$1 = EvmChain.makeSources(chain, onEventRegistrations, sources.hypersync, evmRpcs, lowercaseAddresses);
       break;
     case "FuelSourceConfig" :
-      sources$1 = [HyperFuelSource.make({
+      sources$1 = [FuelHyperSyncSource.make({
           chain: chain,
           endpointUrl: sources.hypersync,
-          apiToken: Env.envioApiToken
+          apiToken: Env.envioApiToken,
+          onEventRegistrations: onEventRegistrations
         })];
       break;
     case "SvmSourceConfig" :

package/src/Config.res

@@ -314,10 +314,27 @@ let propertySchema = S.schema(s =>
   }
 )
 
+let clickhouseTableOptionsSchema: S.t<Internal.clickhouseTableOptions> = S.object(s => {
+  Internal.partitionBy: ?s.field("partitionBy", S.option(S.string)),
+  orderBy: ?s.field("orderBy", S.option(S.array(S.string))),
+  ttl: ?s.field("ttl", S.option(S.string)),
+})
+
+// The entity's `clickhouse` storage arg mirrors the @storage directive:
+// either a boolean or a table options object (which implies enabled).
+type entityClickhouseStorageJson =
+  | Enabled(bool)
+  | TableOptions(Internal.clickhouseTableOptions)
+
+let entityClickhouseStorageJsonSchema = S.union([
+  S.bool->S.shape(enabled => Enabled(enabled)),
+  clickhouseTableOptionsSchema->S.shape(options => TableOptions(options)),
+])
+
 let entityStorageSchema = S.schema(s =>
   {
     "postgres": s.matches(S.option(S.bool)),
-    "clickhouse": s.matches(S.option(S.bool)),
+    "clickhouse": s.matches(S.option(entityClickhouseStorageJsonSchema)),
   }
 )
 
@@ -483,9 +500,21 @@ let parseEntitiesFromJson = (
     // config didn't enable, and that at least one backend stays true
     // for an annotated entity — so `getOr(false)` is safe here.
     let storage: Internal.entityStorage = switch entityJson["storage"] {
-    | Some(s) => {
-        postgres: s["postgres"]->Option.getOr(false),
-        clickhouse: s["clickhouse"]->Option.getOr(false),
+    | Some(s) =>
+      switch s["clickhouse"] {
+      | Some(TableOptions(options)) => {
+          postgres: s["postgres"]->Option.getOr(false),
+          clickhouse: true,
+          clickhouseOptions: options,
+        }
+      | Some(Enabled(clickhouse)) => {
+          postgres: s["postgres"]->Option.getOr(false),
+          clickhouse,
+        }
+      | None => {
+          postgres: s["postgres"]->Option.getOr(false),
+          clickhouse: false,
+        }
       }
     | None => {
         postgres: globalStorage.postgres,
@@ -966,10 +995,14 @@ let fromPublic = (publicConfigJson: JSON.t) => {
 
   let allEntities = userEntities->Array.concat([EnvioAddresses.entityConfig])
 
+  // Keyed by the capitalized entity name to match the handler-context
+  // accessor (`context.Pool_snapshots`) the generated types expose, while
+  // entityConfig.name stays the original schema name used for the physical
+  // Postgres/ClickHouse tables.
   let userEntitiesByName =
     userEntities
     ->Array.map(entityConfig => {
-      (entityConfig.name, entityConfig)
+      (entityConfig.name->Utils.String.capitalize, entityConfig)
     })
     ->Dict.fromArray
 

package/src/Config.res.mjs

@@ -192,9 +192,26 @@ let propertySchema = S$RescriptSchema.schema(s => ({
   description: s.m(S$RescriptSchema.option(S$RescriptSchema.string))
 }));
 
+let clickhouseTableOptionsSchema = S$RescriptSchema.object(s => ({
+  partitionBy: s.f("partitionBy", S$RescriptSchema.option(S$RescriptSchema.string)),
+  orderBy: s.f("orderBy", S$RescriptSchema.option(S$RescriptSchema.array(S$RescriptSchema.string))),
+  ttl: s.f("ttl", S$RescriptSchema.option(S$RescriptSchema.string))
+}));
+
+let entityClickhouseStorageJsonSchema = S$RescriptSchema.union([
+  S$RescriptSchema.shape(S$RescriptSchema.bool, enabled => ({
+    TAG: "Enabled",
+    _0: enabled
+  })),
+  S$RescriptSchema.shape(clickhouseTableOptionsSchema, options => ({
+    TAG: "TableOptions",
+    _0: options
+  }))
+]);
+
 let entityStorageSchema = S$RescriptSchema.schema(s => ({
   postgres: s.m(S$RescriptSchema.option(S$RescriptSchema.bool)),
-  clickhouse: s.m(S$RescriptSchema.option(S$RescriptSchema.bool))
+  clickhouse: s.m(S$RescriptSchema.option(entityClickhouseStorageJsonSchema))
 }));
 
 let entityJsonSchema = S$RescriptSchema.schema(s => ({
@@ -349,10 +366,20 @@ function parseEntitiesFromJson(entitiesJson, enumConfigsByName, globalStorage) {
     let storage;
     if (s !== undefined) {
       let s$1 = Primitive_option.valFromOption(s);
-      storage = {
-        postgres: Stdlib_Option.getOr(s$1.postgres, false),
-        clickhouse: Stdlib_Option.getOr(s$1.clickhouse, false)
-      };
+      let match = s$1.clickhouse;
+      storage = match !== undefined ? (
+          match.TAG === "Enabled" ? ({
+              postgres: Stdlib_Option.getOr(s$1.postgres, false),
+              clickhouse: match._0
+            }) : ({
+              postgres: Stdlib_Option.getOr(s$1.postgres, false),
+              clickhouse: true,
+              clickhouseOptions: match._0
+            })
+        ) : ({
+          postgres: Stdlib_Option.getOr(s$1.postgres, false),
+          clickhouse: false
+        });
     } else {
       storage = {
         postgres: globalStorage.postgres,
@@ -677,7 +704,7 @@ function fromPublic(publicConfigJson) {
   let userEntities = parseEntitiesFromJson(Stdlib_Option.getOr(publicConfig.entities, []), enumConfigsByName, globalStorage);
   let allEntities = userEntities.concat([entityConfig]);
   let userEntitiesByName = Object.fromEntries(userEntities.map(entityConfig => [
-    entityConfig.name,
+    Utils.$$String.capitalize(entityConfig.name),
     entityConfig
   ]));
   let contractHandlers = publicContractsConfig !== undefined ? Object.entries(publicContractsConfig).map(param => ({
@@ -1042,6 +1069,8 @@ export {
   compositeIndexFieldSchema,
   derivedFieldSchema,
   propertySchema,
+  clickhouseTableOptionsSchema,
+  entityClickhouseStorageJsonSchema,
   entityStorageSchema,
   entityJsonSchema,
   getFieldTypeAndSchema,

package/src/Core.res

@@ -4,32 +4,38 @@
 
 // NAPI encodes Rust `Option<T>` as `null | T` (never `undefined`), so the
 // tighter `Null.t` captures the exact boundary shape.
-type evmHypersyncClientCtor
+type evmHyperSyncClientCtor
 type evmRpcClientCtor
-type svmHypersyncClientCtor
-type hyperfuelClientCtor
+type svmHyperSyncClientCtor
+type fuelHyperSyncClientCtor
 type transactionStoreCtor
 type blockStoreCtor
-type parseConfigYamlOptions = {
+type fromUserApiOptions = {
   schema?: string,
   env?: dict<string>,
   files?: dict<string>,
-  isRescript?: bool,
+  withIndexerTypes?: bool,
+}
+
+type fromUserApiResult = {
+  config: string,
+  indexerTypes: Null.t<string>,
 }
 
 type addon = {
   getConfigJson: (~configPath: Null.t<string>, ~directory: Null.t<string>) => string,
   encodeIndexedTopic: (~abiType: string, ~value: unknown) => EvmTypes.Hex.t,
-  parseConfigYaml: (string, parseConfigYamlOptions) => string,
+  fromUserApi: (string, fromUserApiOptions) => fromUserApiResult,
   runCli: (~args: array<string>, ~envioPackageDir: Null.t<string>) => promise<Null.t<string>>,
-  @as("EvmHypersyncClient")
-  evmHypersyncClient: evmHypersyncClientCtor,
+  requestShutdown: unit => unit,
+  @as("EvmHyperSyncClient")
+  evmHyperSyncClient: evmHyperSyncClientCtor,
   @as("EvmRpcClient")
   evmRpcClient: evmRpcClientCtor,
-  @as("SvmHypersyncClient")
-  svmHypersyncClient: svmHypersyncClientCtor,
-  @as("HyperfuelClient")
-  hyperfuelClient: hyperfuelClientCtor,
+  @as("SvmHyperSyncClient")
+  svmHyperSyncClient: svmHyperSyncClientCtor,
+  @as("FuelHyperSyncClient")
+  fuelHyperSyncClient: fuelHyperSyncClientCtor,
   @as("TransactionStore")
   transactionStore: transactionStoreCtor,
   @as("BlockStore")
@@ -207,17 +213,56 @@ let getConfigJson = (~configPath=?, ~directory=?) => {
 
 // Pure config entry point: no cwd, config file, schema file, .env, or process env lookup.
 // A supplied schema is authoritative; omitted or blank schema text means an empty schema.
-let parseConfigYaml = (~schema=?, ~env=?, ~files=?, ~isRescript=false, yaml) => {
+// With `withIndexerTypes`, the single parse also returns the generated
+// `.envio/types.d.ts`, so callers can type-check handlers against the config's
+// `indexer` surface without re-parsing.
+let fromUserApi = (~schema=?, ~env=?, ~files=?, ~withIndexerTypes=false, yaml) => {
   let addon = getAddon()
-  addon.parseConfigYaml(yaml, {
-    ?schema,
-    ?env,
-    ?files,
-    isRescript,
-  })
+  addon.fromUserApi(
+    yaml,
+    {
+      ?schema,
+      ?env,
+      ?files,
+      withIndexerTypes,
+    },
+  )
 }
 
+let runWithShutdownSignals = %raw(`function(run, requestShutdown) {
+  var onShutdown = function() { requestShutdown(); };
+  process.once("SIGINT", onShutdown);
+  process.once("SIGTERM", onShutdown);
+  return run().finally(function() {
+    process.off("SIGINT", onShutdown);
+    process.off("SIGTERM", onShutdown);
+  });
+}`)
+
+// True only when the parsed subcommand is `serve`. The subcommand is the
+// first positional arg; global value-taking flags (`-d`/`--directory`/
+// `--config`) consume the token after them, and other leading flags are
+// skipped — so an unrelated arg value that happens to equal "serve" (e.g.
+// `envio init --name serve`) does not trigger the serve-only shutdown wiring.
+let isServeCommand: array<string> => bool = %raw(`function(args) {
+  var valueFlags = { "-d": true, "--directory": true, "--config": true };
+  for (var i = 0; i < args.length; i++) {
+    var a = args[i];
+    if (a[0] === "-") {
+      if (a.indexOf("=") === -1 && valueFlags[a]) { i++; }
+      continue;
+    }
+    return a === "serve";
+  }
+  return false;
+}`)
+
 let runCli = args => {
   let addon = getAddon()
-  addon.runCli(~args, ~envioPackageDir=Null.make(envioPackageDir))
+  let invoke = () => addon.runCli(~args, ~envioPackageDir=Null.make(envioPackageDir))
+  if isServeCommand(args) {
+    runWithShutdownSignals(invoke, () => addon.requestShutdown())
+  } else {
+    invoke()
+  }
 }

package/src/Core.res.mjs

@@ -180,20 +180,48 @@ function getConfigJson(configPath, directory) {
   return addon.getConfigJson(Stdlib_Null.fromOption(configPath), Stdlib_Null.fromOption(directory));
 }
 
-function parseConfigYaml(schema, env, files, isRescriptOpt, yaml) {
-  let isRescript = isRescriptOpt !== undefined ? isRescriptOpt : false;
+function fromUserApi(schema, env, files, withIndexerTypesOpt, yaml) {
+  let withIndexerTypes = withIndexerTypesOpt !== undefined ? withIndexerTypesOpt : false;
   let addon = getAddon();
-  return addon.parseConfigYaml(yaml, {
+  return addon.fromUserApi(yaml, {
     schema: schema,
     env: env,
     files: files,
-    isRescript: isRescript
+    withIndexerTypes: withIndexerTypes
   });
 }
 
+let runWithShutdownSignals = (function(run, requestShutdown) {
+  var onShutdown = function() { requestShutdown(); };
+  process.once("SIGINT", onShutdown);
+  process.once("SIGTERM", onShutdown);
+  return run().finally(function() {
+    process.off("SIGINT", onShutdown);
+    process.off("SIGTERM", onShutdown);
+  });
+});
+
+let isServeCommand = (function(args) {
+  var valueFlags = { "-d": true, "--directory": true, "--config": true };
+  for (var i = 0; i < args.length; i++) {
+    var a = args[i];
+    if (a[0] === "-") {
+      if (a.indexOf("=") === -1 && valueFlags[a]) { i++; }
+      continue;
+    }
+    return a === "serve";
+  }
+  return false;
+});
+
 function runCli(args) {
   let addon = getAddon();
-  return addon.runCli(args, envioPackageDir);
+  let invoke = () => addon.runCli(args, envioPackageDir);
+  if (isServeCommand(args)) {
+    return runWithShutdownSignals(invoke, () => addon.requestShutdown());
+  } else {
+    return addon.runCli(args, envioPackageDir);
+  }
 }
 
 export {
@@ -207,7 +235,9 @@ export {
   addonRef,
   getAddon,
   getConfigJson,
-  parseConfigYaml,
+  fromUserApi,
+  runWithShutdownSignals,
+  isServeCommand,
   runCli,
 }
 /* envioPackageDir Not a pure module */

package/src/EnvioGlobal.res

@@ -15,7 +15,6 @@
 // deduplication hint instead of silently mixing shapes across builds.
 type t = {
   version: string,
-  eventRegistrations: dict<unknown>,
   mutable activeRegistration: option<unknown>,
   preRegistered: array<unknown>,
   rollbackCommitCallbacks: array<unknown>,
@@ -40,7 +39,6 @@ let value: t = {
   | None =>
     let fresh = {
       version,
-      eventRegistrations: Dict.make(),
       activeRegistration: None,
       preRegistered: [],
       rollbackCommitCallbacks: [],

package/src/EnvioGlobal.res.mjs

@@ -12,7 +12,6 @@ let value;
 if (existing == null) {
   let fresh = {
     version: version,
-    eventRegistrations: {},
     activeRegistration: undefined,
     preRegistered: [],
     rollbackCommitCallbacks: [],

package/src/ExitOnCaughtUp.res

@@ -10,8 +10,12 @@ let run = async (state: IndexerState.t) => {
     ->IndexerState.simulateDeadInputTracker
     ->Option.flatMap(SimulateDeadInputTracker.failureMessage) {
     | None =>
-      Logging.info("Exiting with success")
-      NodeJs.process->NodeJs.exitWithCode(Success)
+      switch state->IndexerState.onExit {
+      | Some(onExit) => onExit()
+      | None =>
+        Logging.info("Exiting with success")
+        NodeJs.process->NodeJs.exitWithCode(Success)
+      }
     | Some(message) => state->IndexerState.errorExit(ErrorHandling.make(Utils.Error.make(message)))
     }
   }

package/src/ExitOnCaughtUp.res.mjs

@@ -18,6 +18,10 @@ async function run(state) {
   let message = Stdlib_Option.flatMap(IndexerState.simulateDeadInputTracker(state), SimulateDeadInputTracker.failureMessage);
   if (message !== undefined) {
     return IndexerState.errorExit(state, ErrorHandling.make(new Error(message), undefined, undefined));
+  }
+  let onExit = IndexerState.onExit(state);
+  if (onExit !== undefined) {
+    return onExit();
   } else {
     Logging.info("Exiting with success");
     Process.exit(0);

package/src/FetchState.res

@@ -1006,8 +1006,8 @@ OptimizedPartitions.t => {
 
           // Addresses with different start blocks within range share a partition;
           // events before each address's effectiveStartBlock are dropped on the
-          // client side (EventRouter for the srcAddress, the event's
-          // clientAddressFilter for address-valued params).
+          // client side (the event's clientAddressFilter for address-valued
+          // params).
           if shouldJoinCurrentStartBlock {
             addressesRef :=
               addressesRef.contents->Array.concat(byStartBlock->Dict.getUnsafe(nextStartBlockKey))
@@ -1411,7 +1411,7 @@ let registerDynamicContracts = (
 // Drop events an address-param filter rejects. A merged partition may
 // over-fetch a wildcard event whose indexed address param references an
 // address registered after the log's block; `clientAddressFilter` is the
-// param-level analogue of EventRouter's srcAddress effectiveStartBlock check.
+// param-level analogue of the srcAddress effectiveStartBlock check.
 let filterByClientAddress = (
   items: array<Internal.item>,
   ~indexingAddresses: IndexingAddresses.t,