envio 3.7.0 → 3.9.0-resolvers.0

package/index.js

@@ -9,6 +9,16 @@ import { schema as bigDecimalSchema } from "./src/bindings/BigDecimal.res.mjs";
 
 export * from "./src/Envio.res.mjs";
 export { indexer, createTestIndexer } from "./src/Api.res.mjs";
+export {
+  createResolver,
+  defineType,
+  defineInput,
+  defineEnum,
+  defineScalar,
+  getRegisteredResolvers,
+  buildRegisteredManifest,
+  ResolverError,
+} from "./src/resolvers/index.js";
 export { default as BigDecimal } from "bignumber.js";
 
 // Important! Should match the index.d.ts file

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "envio",
-  "version": "3.7.0",
+  "version": "3.9.0-resolvers.0",
   "type": "module",
   "description": "A latency and sync speed optimized, developer friendly blockchain data indexer.",
   "bin": "./bin.mjs",
@@ -41,7 +41,6 @@
     "licenses"
   ],
   "dependencies": {
-    "@clickhouse/client": "1.17.0",
     "@elastic/ecs-pino-format": "1.4.0",
     "@fuel-ts/crypto": "0.96.1",
     "@fuel-ts/errors": "0.96.1",
@@ -69,10 +68,10 @@
     "tsx": "4.21.0"
   },
   "optionalDependencies": {
-    "envio-linux-x64": "3.7.0",
-    "envio-linux-x64-musl": "3.7.0",
-    "envio-linux-arm64": "3.7.0",
-    "envio-darwin-x64": "3.7.0",
-    "envio-darwin-arm64": "3.7.0"
+    "envio-linux-x64": "3.9.0-resolvers.0",
+    "envio-linux-x64-musl": "3.9.0-resolvers.0",
+    "envio-linux-arm64": "3.9.0-resolvers.0",
+    "envio-darwin-x64": "3.9.0-resolvers.0",
+    "envio-darwin-arm64": "3.9.0-resolvers.0"
   }
 }

package/src/AddressRows.res

@@ -0,0 +1,121 @@
+type key = {
+  chainId: ChainId.t,
+  address: NodeJs.Buffer.t,
+  contractId: int,
+}
+
+type row = {
+  chainId: ChainId.t,
+  address: NodeJs.Buffer.t,
+  contractId: int,
+  registrationBlock: int,
+}
+
+type staged = {
+  row: row,
+  checkpointId: Internal.checkpointId,
+}
+
+type seedRows = {
+  addresses: array<NodeJs.Buffer.t>,
+  contractIds: array<int>,
+  registrationBlocks: array<int>,
+}
+
+let emptySeedRows = (): seedRows => {
+  addresses: [],
+  contractIds: [],
+  registrationBlocks: [],
+}
+
+// Config addresses are registered by no event. Rollback never reaches them.
+let configRegistrationBlock = -1
+
+let keyOf = (row: row): key => {
+  chainId: row.chainId,
+  address: row.address,
+  contractId: row.contractId,
+}
+
+let packBuffers = (addresses: array<NodeJs.Buffer.t>) => (
+  NodeJs.Buffer.concat(addresses),
+  addresses->Array.map(address => address->NodeJs.Buffer.length),
+)
+
+let packKeys = (rows: array<row>) => packBuffers(rows->Array.map(row => row.address))
+
+let seedRowsOf = (rows: array<row>): seedRows => {
+  let addresses = []
+  let contractIds = []
+  let registrationBlocks = []
+  rows->Array.forEach(row => {
+    addresses->Array.push(row.address)->ignore
+    contractIds->Array.push(row.contractId)->ignore
+    registrationBlocks->Array.push(row.registrationBlock)->ignore
+  })
+  {addresses, contractIds, registrationBlocks}
+}
+
+// Rows arrive clustered per chain. Normalizing a chain id is a schema parse.
+let group = (rows: array<row>): dict<seedRows> => {
+  let rowsByChain: dict<array<row>> = Dict.make()
+  let runChainId = ref(None)
+  let runKey = ref("")
+  rows->Array.forEach(row => {
+    if runChainId.contents !== Some(row.chainId) {
+      runChainId := Some(row.chainId)
+      runKey := row.chainId->ChainId.normalizeOrThrow->ChainId.toString
+    }
+    rowsByChain->Utils.Dict.push(runKey.contents, row)
+  })
+  rowsByChain->Utils.Dict.mapValues(seedRowsOf)
+}
+
+module Table = {
+  type t = dict<row>
+
+  let dictKey = (key: key) =>
+    `${key.chainId->ChainId.toString}|${key.contractId->Int.toString}|${key.address->NodeJs.Buffer.toBase64}`
+
+  let make = (): t => Dict.make()
+
+  let clear = (table: t) => table->Utils.Dict.clearInPlace
+
+  let insert = (table: t, row: row) => {
+    let key = row->keyOf->dictKey
+    switch table->Utils.Dict.dangerouslyGetNonOption(key) {
+    | Some(_) => ()
+    | None => table->Dict.set(key, row)
+    }
+  }
+
+  let insertMany = (table: t, rows: array<row>) => rows->Array.forEach(row => table->insert(row))
+
+  let delete = (table: t, key: key) => table->Utils.Dict.deleteInPlace(key->dictKey)
+
+  let rows = (table: t): array<row> => table->Dict.valuesToArray
+
+  let groupByChain = (table: t) => table->rows->group
+}
+
+let render = (rows: array<row>, ~ecosystem: string, ~shouldChecksum: bool) => {
+  let (bytes, lengths) = rows->packKeys
+  Core.getAddon().renderAddresses(~ecosystem, ~shouldChecksum, ~bytes, ~lengths)
+}
+
+let renderOfContract = (
+  rows: seedRows,
+  ~ecosystem: string,
+  ~shouldChecksum: bool,
+  ~contractId: int,
+) => {
+  let (bytes, lengths) = packBuffers(rows.addresses)
+  Core.getAddon().renderContractAddresses(
+    ~ecosystem,
+    ~shouldChecksum,
+    ~bytes,
+    ~lengths,
+    ~contractIds=rows.contractIds,
+    ~contractId,
+  )
+}

package/src/AddressRows.res.mjs

@@ -0,0 +1,143 @@
+// Generated by ReScript, PLEASE EDIT WITH CARE
+
+import * as Core from "./Core.res.mjs";
+import * as Utils from "./Utils.res.mjs";
+import * as ChainId from "./ChainId.res.mjs";
+import * as Primitive_option from "@rescript/runtime/lib/es6/Primitive_option.js";
+
+function emptySeedRows() {
+  return {
+    addresses: [],
+    contractIds: [],
+    registrationBlocks: []
+  };
+}
+
+function keyOf(row) {
+  return {
+    chainId: row.chainId,
+    address: row.address,
+    contractId: row.contractId
+  };
+}
+
+function packBuffers(addresses) {
+  return [
+    Buffer.concat(addresses),
+    addresses.map(address => address.length)
+  ];
+}
+
+function packKeys(rows) {
+  return packBuffers(rows.map(row => row.address));
+}
+
+function seedRowsOf(rows) {
+  let addresses = [];
+  let contractIds = [];
+  let registrationBlocks = [];
+  rows.forEach(row => {
+    addresses.push(row.address);
+    contractIds.push(row.contractId);
+    registrationBlocks.push(row.registrationBlock);
+  });
+  return {
+    addresses: addresses,
+    contractIds: contractIds,
+    registrationBlocks: registrationBlocks
+  };
+}
+
+function group(rows) {
+  let rowsByChain = {};
+  let runChainId = {
+    contents: undefined
+  };
+  let runKey = {
+    contents: ""
+  };
+  rows.forEach(row => {
+    if (runChainId.contents !== Primitive_option.some(row.chainId)) {
+      runChainId.contents = Primitive_option.some(row.chainId);
+      runKey.contents = ChainId.toString(ChainId.normalizeOrThrow(row.chainId));
+    }
+    Utils.Dict.push(rowsByChain, runKey.contents, row);
+  });
+  return Utils.Dict.mapValues(rowsByChain, seedRowsOf);
+}
+
+function dictKey(key) {
+  return ChainId.toString(key.chainId) + `|` + key.contractId.toString() + `|` + key.address.toString("base64");
+}
+
+function make() {
+  return {};
+}
+
+function clear(table) {
+  Utils.Dict.clearInPlace(table);
+}
+
+function insert(table, row) {
+  let key = dictKey(keyOf(row));
+  let match = table[key];
+  if (match !== undefined) {
+    return;
+  } else {
+    table[key] = row;
+    return;
+  }
+}
+
+function insertMany(table, rows) {
+  rows.forEach(row => insert(table, row));
+}
+
+function $$delete(table, key) {
+  Utils.Dict.deleteInPlace(table, dictKey(key));
+}
+
+function rows(table) {
+  return Object.values(table);
+}
+
+function groupByChain(table) {
+  return group(Object.values(table));
+}
+
+let Table = {
+  dictKey: dictKey,
+  make: make,
+  clear: clear,
+  insert: insert,
+  insertMany: insertMany,
+  $$delete: $$delete,
+  rows: rows,
+  groupByChain: groupByChain
+};
+
+function render(rows, ecosystem, shouldChecksum) {
+  let match = packKeys(rows);
+  return Core.getAddon().renderAddresses(ecosystem, shouldChecksum, match[0], match[1]);
+}
+
+function renderOfContract(rows, ecosystem, shouldChecksum, contractId) {
+  let match = packBuffers(rows.addresses);
+  return Core.getAddon().renderContractAddresses(ecosystem, shouldChecksum, match[0], match[1], rows.contractIds, contractId);
+}
+
+let configRegistrationBlock = -1;
+
+export {
+  emptySeedRows,
+  configRegistrationBlock,
+  keyOf,
+  packBuffers,
+  packKeys,
+  seedRowsOf,
+  group,
+  Table,
+  render,
+  renderOfContract,
+}
+/* Core Not a pure module */

package/src/Batch.res

@@ -43,6 +43,7 @@ type t = {
   checkpointBlockNumbers: array<int>,
   checkpointBlockHashes: array<Null.t<string>>,
   checkpointEventsProcessed: array<int>,
+  registeredAddresses: array<AddressRows.staged>,
 }
 
 let getProgressedChainsById = {
@@ -188,6 +189,15 @@ let addReorgCheckpoints = (
   }
 }
 
+// Checkpoint ids are handed out in ascending order as each chain's items are
+// walked in block order, so within a chain a higher id always means a later
+// block. Rollback preserves that: it deletes the chain's ids above its target
+// before any higher one is allocated, so the ids the re-indexed blocks get are
+// above everything the chain still holds. An isolated rollback leans on the
+// invariant — it deletes by `chain_id = c AND id > target`, which is only the
+// chain's stale suffix if ids and blocks agree on order within the chain. Ids
+// interleave freely *across* chains, which is exactly why that predicate can't
+// be the id bound alone.
 let prepareBatch = (
   ~checkpointIdBeforeBatch,
   ~chainsBeforeBatch: dict<chainBeforeBatch>,
@@ -326,6 +336,7 @@ let prepareBatch = (
     checkpointBlockNumbers,
     checkpointBlockHashes,
     checkpointEventsProcessed,
+    registeredAddresses: [],
   }
 }
 

package/src/Batch.res.mjs

@@ -185,7 +185,8 @@ function prepareBatch(checkpointIdBeforeBatch, chainsBeforeBatch, batchSizeTarge
     checkpointChainIds: checkpointChainIds,
     checkpointBlockNumbers: checkpointBlockNumbers,
     checkpointBlockHashes: checkpointBlockHashes,
-    checkpointEventsProcessed: checkpointEventsProcessed
+    checkpointEventsProcessed: checkpointEventsProcessed,
+    registeredAddresses: []
   };
 }
 

package/src/Bin.res

@@ -18,11 +18,13 @@ type startCmd = {
 }
 type migrateCmd = {reset: bool, config: JSON.t}
 type dropSchemaCmd = {config: JSON.t}
+type resolversCmd = {mode: string, cwd: string, env: dict<JSON.t>, config: JSON.t}
 
 type command =
   | Start(startCmd)
   | Migrate(migrateCmd)
   | DropSchema(dropSchemaCmd)
+  | Resolvers(resolversCmd)
 
 let decodeCommand = (json: JSON.t): command => {
   let obj = switch json->JSON.Decode.object {
@@ -37,6 +39,7 @@ let decodeCommand = (json: JSON.t): command => {
   | "start" => Start(json->(Utils.magic: JSON.t => startCmd))
   | "migrate" => Migrate(json->(Utils.magic: JSON.t => migrateCmd))
   | "drop-schema" => DropSchema(json->(Utils.magic: JSON.t => dropSchemaCmd))
+  | "resolvers" => Resolvers(json->(Utils.magic: JSON.t => resolversCmd))
   | other => JsError.throwWithMessage(`Unknown command kind: ${other}`)
   }
 }
@@ -61,6 +64,12 @@ let run = async args => {
         Config.prime(config)
         processChdir(cwd)
         applyEnv(env)
+        // `envio dev` only — see ResolverProcess.startForDev. A deployment
+        // runs the resolvers as their own Deployment, never from here.
+        let loaded = Config.load()
+        if loaded.isDev {
+          let _ = await ResolverProcess.startForDev(~config=loaded, ~projectRoot=cwd)
+        }
         await Main.start(~reset)
       | Migrate({reset, config}) =>
         Config.prime(config)
@@ -68,6 +77,44 @@ let run = async args => {
       | DropSchema({config}) =>
         Config.prime(config)
         await Main.dropSchema()
+      | Resolvers({mode, cwd, env, config}) =>
+        Config.prime(config)
+        processChdir(cwd)
+        applyEnv(env)
+        switch mode {
+        | "manifest" => await ResolverProcess.writeManifest(~config=Config.load(), ~projectRoot=cwd)
+        | "metadata" =>
+          let handlerUrl = switch Env.Resolvers.publicUrl() {
+          | Some(url) => url
+          | None =>
+            JsError.throwWithMessage(
+              "ENVIO_RESOLVERS_PUBLIC_URL is not set. It is the URL Hasura posts to and is baked into every action, so the metadata cannot be printed without it.",
+            )
+          }
+          let metadata = await ResolverProcess.metadataJson(
+            ~config=Config.load(),
+            ~projectRoot=cwd,
+            ~handlerUrl,
+          )
+          // stdout, so it pipes into `hasura metadata apply` or `jq` rather
+          // than being buried in the log stream.
+          Console.log(JSON.stringify(metadata, ~space=2))
+        | "migrate" =>
+          let {applied, reasons} = await ResolverProcess.migrate(
+            ~config=Config.load(),
+            ~projectRoot=cwd,
+          )
+          Logging.info(
+            applied
+              ? `Updated Hasura to match these resolvers: ${reasons->Array.join("; ")}`
+              : "Hasura already matches these resolvers; nothing to update",
+          )
+        // Serving keeps the event loop alive on its own, so returning here
+        // leaves the process running rather than exiting.
+        | _ =>
+          let running = await ResolverProcess.serve(~config=Config.load(), ~projectRoot=cwd)
+          running->ResolverProcess.handleSignals
+        }
       }
     }
   } catch {

package/src/Bin.res.mjs

@@ -1,5 +1,6 @@
 // Generated by ReScript, PLEASE EDIT WITH CARE
 
+import * as Env from "./Env.res.mjs";
 import * as Core from "./Core.res.mjs";
 import * as Main from "./Main.res.mjs";
 import * as Utils from "./Utils.res.mjs";
@@ -11,6 +12,7 @@ import * as Stdlib_JSON from "@rescript/runtime/lib/es6/Stdlib_JSON.js";
 import * as Stdlib_JsExn from "@rescript/runtime/lib/es6/Stdlib_JsExn.js";
 import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js";
 import * as Stdlib_JsError from "@rescript/runtime/lib/es6/Stdlib_JsError.js";
+import * as ResolverProcess from "./resolvers/ResolverProcess.res.mjs";
 import * as Primitive_exceptions from "@rescript/runtime/lib/es6/Primitive_exceptions.js";
 
 let setEnvVar = ((k, v) => { process.env[k] = v; });
@@ -36,6 +38,11 @@ function decodeCommand(json) {
         TAG: "Migrate",
         _0: json
       };
+    case "resolvers" :
+      return {
+        TAG: "Resolvers",
+        _0: json
+      };
     case "start" :
       return {
         TAG: "Start",
@@ -65,9 +72,14 @@ async function run(args) {
     switch (match.TAG) {
       case "Start" :
         let match$1 = match._0;
+        let cwd = match$1.cwd;
         Config.prime(match$1.config);
-        process.chdir(match$1.cwd);
+        process.chdir(cwd);
         applyEnv(match$1.env);
+        let loaded = Config.load();
+        if (loaded.isDev) {
+          await ResolverProcess.startForDev(loaded, cwd);
+        }
         return await Main.start(undefined, match$1.reset, undefined, undefined, undefined);
       case "Migrate" :
         let match$2 = match._0;
@@ -76,6 +88,27 @@ async function run(args) {
       case "DropSchema" :
         Config.prime(match._0.config);
         return await Main.dropSchema();
+      case "Resolvers" :
+        let match$3 = match._0;
+        let cwd$1 = match$3.cwd;
+        Config.prime(match$3.config);
+        process.chdir(cwd$1);
+        applyEnv(match$3.env);
+        switch (match$3.mode) {
+          case "manifest" :
+            return await ResolverProcess.writeManifest(Config.load(), cwd$1);
+          case "metadata" :
+            let url = Env.Resolvers.publicUrl();
+            let handlerUrl = url !== undefined ? url : Stdlib_JsError.throwWithMessage("ENVIO_RESOLVERS_PUBLIC_URL is not set. It is the URL Hasura posts to and is baked into every action, so the metadata cannot be printed without it.");
+            let metadata = await ResolverProcess.metadataJson(Config.load(), cwd$1, handlerUrl);
+            console.log(JSON.stringify(metadata, undefined, 2));
+            return;
+          case "migrate" :
+            let match$4 = await ResolverProcess.migrate(Config.load(), cwd$1);
+            return Logging.info(match$4.applied ? `Updated Hasura to match these resolvers: ` + match$4.reasons.join("; ") : "Hasura already matches these resolvers; nothing to update");
+          default:
+            return ResolverProcess.handleSignals(await ResolverProcess.serve(Config.load(), cwd$1, undefined, undefined, undefined, undefined));
+        }
     }
   } catch (raw_exn) {
     let exn = Primitive_exceptions.internalToException(raw_exn);