envio 3.0.0-rc.1 → 3.0.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "envio",
-  "version": "3.0.0-rc.1",
+  "version": "3.0.1",
   "type": "module",
   "description": "A latency and sync speed optimized, developer friendly blockchain data indexer.",
   "bin": "./bin.mjs",
@@ -71,10 +71,10 @@
     "tsx": "4.21.0"
   },
   "optionalDependencies": {
-    "envio-linux-x64": "3.0.0-rc.1",
-    "envio-linux-x64-musl": "3.0.0-rc.1",
-    "envio-linux-arm64": "3.0.0-rc.1",
-    "envio-darwin-x64": "3.0.0-rc.1",
-    "envio-darwin-arm64": "3.0.0-rc.1"
+    "envio-linux-x64": "3.0.1",
+    "envio-linux-x64-musl": "3.0.1",
+    "envio-linux-arm64": "3.0.1",
+    "envio-darwin-x64": "3.0.1",
+    "envio-darwin-arm64": "3.0.1"
   }
 }

package/src/Config.res

@@ -957,11 +957,18 @@ let rec canonicalJson = (json: JSON.t): JSON.t =>
   }
 
 // Returns dotted leaf paths (`a.b[i].c`) where `stored` differs from
-// `current`. Skips subtrees whose canonical JSON matches.
+// `current`, restricted to the highest-priority top-level tier with any
+// diff. Tiers in order: version → name → storage → ecosystem
+// (evm/fuel/svm) → entities → other top-level keys. The first tier
+// containing a diff is the only one rendered; lower tiers are silenced
+// so a single noisy section doesn't bury the actionable change.
 let diffPaths = (~stored: JSON.t, ~current: JSON.t): array<string> => {
+  let canonEq = (a: JSON.t, b: JSON.t) =>
+    JSON.stringify(canonicalJson(a)) === JSON.stringify(canonicalJson(b))
+
   let acc = []
   let rec go = (s: JSON.t, c: JSON.t, prefix: string) => {
-    if JSON.stringify(canonicalJson(s)) === JSON.stringify(canonicalJson(c)) {
+    if canonEq(s, c) {
       ()
     } else {
       switch (s, c) {
@@ -987,25 +994,91 @@ let diffPaths = (~stored: JSON.t, ~current: JSON.t): array<string> => {
           | (Some(sv), Some(cv)) => go(sv, cv, p)
           }
         }
-      | _ =>
-        // Type mismatch or scalar diff
-        acc->Array.push(prefix === "" ? "<root>" : prefix)->ignore
+      | _ => acc->Array.push(prefix === "" ? "<root>" : prefix)->ignore
+      }
+    }
+  }
+
+  let getTopKey = (j: JSON.t, k: string) =>
+    switch j {
+    | Object(d) => d->Dict.get(k)
+    | _ => None
+    }
+  let topKeyDiffers = (k: string) =>
+    switch (getTopKey(stored, k), getTopKey(current, k)) {
+    | (None, None) => false
+    | (None, _) | (_, None) => true
+    | (Some(s), Some(c)) => !canonEq(s, c)
+    }
+  let runTier = (keys: array<string>) =>
+    keys->Array.forEach(k =>
+      switch (getTopKey(stored, k), getTopKey(current, k)) {
+      | (None, None) => ()
+      | (None, _) | (_, None) => acc->Array.push(k)->ignore
+      | (Some(s), Some(c)) => go(s, c, k)
+      }
+    )
+
+  switch (stored, current) {
+  | (Object(sObj), Object(cObj)) =>
+    let tiers = [["version"], ["name"], ["storage"], ["evm", "fuel", "svm"], ["entities"]]
+    let firstHit = tiers->Array.reduce(None, (acc, tier) =>
+      switch acc {
+      | Some(_) => acc
+      | None =>
+        switch tier->Array.filter(topKeyDiffers) {
+        | [] => None
+        | hits => Some(hits)
+        }
       }
+    )
+    switch firstHit {
+    | Some(hits) => runTier(hits)
+    | None =>
+      let knownSet = Utils.Set.fromArray(tiers->Belt.Array.concatMany)
+      let extras =
+        Utils.Set.fromArray(Array.concat(sObj->Dict.keysToArray, cObj->Dict.keysToArray))
+        ->Utils.Set.toArray
+        ->Array.filter(k => !(knownSet->Utils.Set.has(k)))
+        ->Array.toSorted(String.compare)
+        ->Array.filter(topKeyDiffers)
+      runTier(extras)
     }
+  | _ => go(stored, current, "")
   }
-  go(stored, current, "")
   acc
 }
 
 // Throws an `incompatible config` error listing each path in `changedPaths`,
-// plus the revert/reset remediation. `~resetCommand` lets each call site
-// (`envio dev -r` / `envio start -r` / `envio local db-migrate setup`)
-// surface the right wipe-and-restart command.
-let throwIfIncompatible = (changedPaths: array<string>, ~resetCommand: string) => {
+// plus the remediation options. `~resetCommand` is rendered as-is for
+// option 2 (the wipe-and-redo). `~runCommand` controls option 3 (parallel
+// indexer recipe): when `None`, option 3 is omitted — the migrate flow
+// uses this because running a second indexer doesn't apply.
+// `~hasClickhouse` adds the extra env line so users running both
+// Postgres and Clickhouse get a complete override.
+let throwIfIncompatible = (
+  changedPaths: array<string>,
+  ~resetCommand: string,
+  ~runCommand: option<string>,
+  ~hasClickhouse: bool,
+) => {
   if changedPaths->Array.length > 0 {
     let bullets = changedPaths->Array.map(p => `    - ${p}`)->Array.joinUnsafe("\n")
+    let option1 = "Revert the changes above"
+    let padTo = (s, col) => s ++ " "->String.repeat(Math.Int.max(col - String.length(s), 1))
+    let col = Math.Int.max(String.length(option1), String.length(resetCommand)) + 2
+    let option3 = switch runCommand {
+    | None => ""
+    | Some(cmd) =>
+      let clickhouseLine = hasClickhouse ? "       ENVIO_CLICKHOUSE_DATABASE=<new_db> \\\n" : ""
+      `\n  3. Run a second indexer alongside this one — keep both datasets:\n       ENVIO_PG_SCHEMA=<new_schema> \\\n${clickhouseLine}       ENVIO_INDEXER_PORT=<new_port> \\\n       ${cmd}`
+    }
     JsError.throwWithMessage(
-      `The following config changes are incompatible with the existing indexer data:\n\n${bullets}\n\nPick one:\n\n  1. Revert the changes above    # resume indexing where it left off\n  2. ${resetCommand}                # wipe the database and re-index from scratch`,
+      `The following config changes are incompatible with the existing indexer data:\n\n${bullets}\n\nPick one:\n  1. ${option1->padTo(
+          col,
+        )}# resume indexing where it left off\n  2. ${resetCommand->padTo(
+          col,
+        )}# delete all indexed data and start over${option3}`,
     )
   }
 }

package/src/Config.res.mjs

@@ -780,9 +780,10 @@ function canonicalJson(json) {
 }
 
 function diffPaths(stored, current) {
+  let canonEq = (a, b) => JSON.stringify(canonicalJson(a)) === JSON.stringify(canonicalJson(b));
   let acc = [];
   let go = (s, c, prefix) => {
-    if (JSON.stringify(canonicalJson(s)) === JSON.stringify(canonicalJson(c))) {
+    if (canonEq(s, c)) {
       return;
     }
     if (Array.isArray(s)) {
@@ -830,16 +831,92 @@ function diffPaths(stored, current) {
     }
     acc.push(prefix === "" ? "<root>" : prefix);
   };
-  go(stored, current, "");
+  let getTopKey = (j, k) => {
+    if (typeof j === "object" && j !== null && !Array.isArray(j)) {
+      return j[k];
+    }
+  };
+  let topKeyDiffers = k => {
+    let match = getTopKey(stored, k);
+    let match$1 = getTopKey(current, k);
+    if (match !== undefined) {
+      if (match$1 !== undefined) {
+        return !canonEq(match, match$1);
+      } else {
+        return true;
+      }
+    } else {
+      return match$1 !== undefined;
+    }
+  };
+  let runTier = keys => {
+    keys.forEach(k => {
+      let match = getTopKey(stored, k);
+      let match$1 = getTopKey(current, k);
+      if (match !== undefined) {
+        if (match$1 !== undefined) {
+          return go(match, match$1, k);
+        } else {
+          acc.push(k);
+          return;
+        }
+      } else if (match$1 !== undefined) {
+        acc.push(k);
+        return;
+      } else {
+        return;
+      }
+    });
+  };
+  if (typeof stored === "object" && stored !== null && !Array.isArray(stored) && typeof current === "object" && current !== null && !Array.isArray(current)) {
+    let tiers = [
+      ["version"],
+      ["name"],
+      ["storage"],
+      [
+        "evm",
+        "fuel",
+        "svm"
+      ],
+      ["entities"]
+    ];
+    let firstHit = Stdlib_Array.reduce(tiers, undefined, (acc, tier) => {
+      if (acc !== undefined) {
+        return acc;
+      }
+      let hits = tier.filter(topKeyDiffers);
+      if (hits.length !== 0) {
+        return hits;
+      }
+    });
+    if (firstHit !== undefined) {
+      runTier(firstHit);
+    } else {
+      let knownSet = new Set(Belt_Array.concatMany(tiers));
+      runTier(Array.from(new Set(Object.keys(stored).concat(Object.keys(current)))).filter(k => !knownSet.has(k)).toSorted(Primitive_string.compare).filter(topKeyDiffers));
+    }
+  } else {
+    go(stored, current, "");
+  }
   return acc;
 }
 
-function throwIfIncompatible(changedPaths, resetCommand) {
+function throwIfIncompatible(changedPaths, resetCommand, runCommand, hasClickhouse) {
   if (changedPaths.length === 0) {
     return;
   }
   let bullets = changedPaths.map(p => `    - ` + p).join("\n");
-  Stdlib_JsError.throwWithMessage(`The following config changes are incompatible with the existing indexer data:\n\n` + bullets + `\n\nPick one:\n\n  1. Revert the changes above    # resume indexing where it left off\n  2. ` + resetCommand + `                # wipe the database and re-index from scratch`);
+  let option1 = "Revert the changes above";
+  let padTo = (s, col) => s + " ".repeat(Math.max(col - s.length | 0, 1));
+  let col = Math.max(option1.length, resetCommand.length) + 2 | 0;
+  let option3;
+  if (runCommand !== undefined) {
+    let clickhouseLine = hasClickhouse ? "       ENVIO_CLICKHOUSE_DATABASE=<new_db> \\\n" : "";
+    option3 = `\n  3. Run a second indexer alongside this one — keep both datasets:\n       ENVIO_PG_SCHEMA=<new_schema> \\\n` + clickhouseLine + `       ENVIO_INDEXER_PORT=<new_port> \\\n       ` + runCommand;
+  } else {
+    option3 = "";
+  }
+  Stdlib_JsError.throwWithMessage(`The following config changes are incompatible with the existing indexer data:\n\n` + bullets + `\n\nPick one:\n  1. ` + padTo(option1, col) + `# resume indexing where it left off\n  2. ` + padTo(resetCommand, col) + `# delete all indexed data and start over` + option3);
 }
 
 function loadWithoutRegistrations() {

package/src/InMemoryTable.res

@@ -71,7 +71,6 @@ module Entity = {
     fieldNameIndices: make(~hash=TableIndices.Index.getFieldName),
   }
 
-  exception UndefinedKey(string)
   let updateIndices = (
     self: t<'entity>,
     ~entity: 'entity,
@@ -83,8 +82,7 @@ module Entity = {
       let fieldValue =
         entity
         ->(Utils.magic: 'entity => dict<TableIndices.FieldValue.t>)
-        ->Dict.get(fieldName)
-        ->Option.getUnsafe
+        ->Dict.getUnsafe(fieldName)
       if !(index->TableIndices.Index.evaluate(~fieldName, ~fieldValue)) {
         entityIndices->Utils.Set.delete(index)->ignore
       }
@@ -93,27 +91,25 @@ module Entity = {
     self.fieldNameIndices.dict
     ->Dict.keysToArray
     ->Array.forEach(fieldName => {
-      switch (
-        entity->(Utils.magic: 'entity => dict<TableIndices.FieldValue.t>)->Dict.get(fieldName),
-        self.fieldNameIndices.dict->Dict.get(fieldName),
-      ) {
-      | (Some(fieldValue), Some(indices)) =>
-        indices
-        ->values
-        ->Array.forEach(((index, relatedEntityIds)) => {
-          if index->TableIndices.Index.evaluate(~fieldName, ~fieldValue) {
-            //Add entity id to indices and add index to entity indicies
-            relatedEntityIds->Utils.Set.add(getEntityIdUnsafe(entity))->ignore
-            entityIndices->Utils.Set.add(index)->ignore
-          } else {
-            relatedEntityIds->Utils.Set.delete(getEntityIdUnsafe(entity))->ignore
-          }
-        })
-      | _ =>
-        UndefinedKey(fieldName)->ErrorHandling.mkLogAndRaise(
-          ~msg="Expected field name to exist on the referenced index and the provided entity",
-        )
-      }
+      let indices = self.fieldNameIndices.dict->Dict.getUnsafe(fieldName)
+      // A missing key reads as `undefined`, which matches the `None` arm of
+      // `FieldValue.t` (`option<...>`). Mirror `addEmptyIndex` so nullable
+      // FK columns that were omitted on the set entity don't crash.
+      let fieldValue =
+        entity
+        ->(Utils.magic: 'entity => dict<TableIndices.FieldValue.t>)
+        ->Dict.getUnsafe(fieldName)
+      indices
+      ->values
+      ->Array.forEach(((index, relatedEntityIds)) => {
+        if index->TableIndices.Index.evaluate(~fieldName, ~fieldValue) {
+          //Add entity id to indices and add index to entity indicies
+          relatedEntityIds->Utils.Set.add(getEntityIdUnsafe(entity))->ignore
+          entityIndices->Utils.Set.add(index)->ignore
+        } else {
+          relatedEntityIds->Utils.Set.delete(getEntityIdUnsafe(entity))->ignore
+        }
+      })
     })
   }
 

package/src/InMemoryTable.res.mjs

@@ -90,8 +90,6 @@ function make$1() {
   };
 }
 
-let UndefinedKey = /* @__PURE__ */Primitive_exceptions.create("InMemoryTable.Entity.UndefinedKey");
-
 function updateIndices(self, entity, entityIndices) {
   entityIndices.forEach(index => {
     let fieldName = TableIndices.Index.getFieldName(index);
@@ -102,22 +100,9 @@ function updateIndices(self, entity, entityIndices) {
     }
   });
   Object.keys(self.fieldNameIndices.dict).forEach(fieldName => {
-    let match = entity[fieldName];
-    let match$1 = self.fieldNameIndices.dict[fieldName];
-    if (match === undefined) {
-      return ErrorHandling.mkLogAndRaise(undefined, "Expected field name to exist on the referenced index and the provided entity", {
-        RE_EXN_ID: UndefinedKey,
-        _1: fieldName
-      });
-    }
-    if (match$1 === undefined) {
-      return ErrorHandling.mkLogAndRaise(undefined, "Expected field name to exist on the referenced index and the provided entity", {
-        RE_EXN_ID: UndefinedKey,
-        _1: fieldName
-      });
-    }
-    let fieldValue = Primitive_option.valFromOption(match);
-    Object.values(match$1.dict).forEach(param => {
+    let indices = self.fieldNameIndices.dict[fieldName];
+    let fieldValue = entity[fieldName];
+    Object.values(indices.dict).forEach(param => {
       let relatedEntityIds = param[1];
       let index = param[0];
       if (TableIndices.Index.evaluate(index, fieldName, fieldValue)) {
@@ -333,7 +318,6 @@ let Entity = {
   getEntityIdUnsafe: getEntityIdUnsafe,
   makeIndicesSerializedToValue: makeIndicesSerializedToValue,
   make: make$1,
-  UndefinedKey: UndefinedKey,
   updateIndices: updateIndices,
   deleteEntityFromIndices: deleteEntityFromIndices,
   initValue: initValue,

package/src/Main.res

@@ -622,6 +622,7 @@ let migrate = async (~reset) => {
     ~chainConfigs=config.chainMap->ChainMap.values,
     ~envioInfo=getEnvioInfo(),
     ~resetCommand="envio local db-migrate setup",
+    ~runCommand=None,
   )
   await persistence.storage.close()
 }
@@ -668,6 +669,7 @@ let start = async (
     ~chainConfigs=configWithoutRegistrations.chainMap->ChainMap.values,
     ~envioInfo=getEnvioInfo(),
     ~resetCommand=isDevelopmentMode ? "envio dev -r" : "envio start -r",
+    ~runCommand=Some(isDevelopmentMode ? "envio dev" : "envio start"),
   )
 
   // `Config.loadWithoutRegistrations` never sees registration state; handler,

package/src/Main.res.mjs

@@ -472,7 +472,7 @@ function getEnvioInfo() {
 async function migrate(reset) {
   let config = Config.loadWithoutRegistrations();
   let persistence = PgStorage.makePersistenceFromConfig(config, undefined);
-  await Persistence.init(persistence, ChainMap.values(config.chainMap), Config.stripSensitiveData(Config.getPublicConfigJson()), "envio local db-migrate setup", reset);
+  await Persistence.init(persistence, ChainMap.values(config.chainMap), Config.stripSensitiveData(Config.getPublicConfigJson()), "envio local db-migrate setup", undefined, reset);
   return await persistence.storage.close();
 }
 
@@ -497,7 +497,7 @@ async function start(persistence, resetOpt, isTestOpt, exitAfterFirstEventBlockO
   let isDevelopmentMode = !isTest && configWithoutRegistrations.isDev;
   let persistence$1 = persistence !== undefined ? persistence : PgStorage.makePersistenceFromConfig(configWithoutRegistrations, undefined);
   globalPersistenceRef.contents = persistence$1;
-  await Persistence.init(persistence$1, ChainMap.values(configWithoutRegistrations.chainMap), Config.stripSensitiveData(Config.getPublicConfigJson()), isDevelopmentMode ? "envio dev -r" : "envio start -r", reset);
+  await Persistence.init(persistence$1, ChainMap.values(configWithoutRegistrations.chainMap), Config.stripSensitiveData(Config.getPublicConfigJson()), isDevelopmentMode ? "envio dev -r" : "envio start -r", isDevelopmentMode ? "envio dev" : "envio start", reset);
   let match = await HandlerLoader.registerAllHandlers(configWithoutRegistrations);
   let registrations = match[1];
   let config = match[0];

package/src/Persistence.res

@@ -168,7 +168,7 @@ let make = (
 }
 
 let init = {
-  async (persistence, ~chainConfigs, ~envioInfo, ~resetCommand, ~reset=false) => {
+  async (persistence, ~chainConfigs, ~envioInfo, ~resetCommand, ~runCommand, ~reset=false) => {
     try {
       let shouldRun = switch persistence.storageStatus {
       | Unknown => true
@@ -212,7 +212,22 @@ let init = {
           | None => ["envio info is missing — storage initialized by an older envio"]
           | Some(stored) => Config.diffPaths(~stored, ~current=envioInfo)
           }
-          Config.throwIfIncompatible(changedPaths, ~resetCommand)
+          // `storage.clickhouse` is serialized as a plain bool by the
+          // public config (see Rust `StorageConfig`), so probe for
+          // `Boolean(true)`, not an object.
+          let hasClickhouse = switch envioInfo {
+          | Object(d) =>
+            switch d->Dict.get("storage") {
+            | Some(Object(s)) =>
+              switch s->Dict.get("clickhouse") {
+              | Some(Boolean(true)) => true
+              | _ => false
+              }
+            | _ => false
+            }
+          | _ => false
+          }
+          Config.throwIfIncompatible(changedPaths, ~resetCommand, ~runCommand, ~hasClickhouse)
           persistence.storageStatus = Ready(initialState)
           let progress = Dict.make()
           initialState.chains->Array.forEach(c => {

package/src/Persistence.res.mjs

@@ -27,7 +27,7 @@ function make(userEntities, allEnums, storage) {
   };
 }
 
-async function init(persistence, chainConfigs, envioInfo, resetCommand, resetOpt) {
+async function init(persistence, chainConfigs, envioInfo, resetCommand, runCommand, resetOpt) {
   let reset = resetOpt !== undefined ? resetOpt : false;
   try {
     let promise = persistence.storageStatus;
@@ -70,7 +70,19 @@ async function init(persistence, chainConfigs, envioInfo, resetCommand, resetOpt
         let initialState$1 = await persistence.storage.resumeInitialState();
         let stored = initialState$1.envioInfo;
         let changedPaths = stored !== undefined ? Config.diffPaths(stored, envioInfo) : ["envio info is missing — storage initialized by an older envio"];
-        Config.throwIfIncompatible(changedPaths, resetCommand);
+        let hasClickhouse;
+        if (typeof envioInfo === "object" && envioInfo !== null && !Array.isArray(envioInfo)) {
+          let match$1 = envioInfo["storage"];
+          if (typeof match$1 === "object" && match$1 !== null && !Array.isArray(match$1)) {
+            let match$2 = match$1["clickhouse"];
+            hasClickhouse = match$2 === true;
+          } else {
+            hasClickhouse = false;
+          }
+        } else {
+          hasClickhouse = false;
+        }
+        Config.throwIfIncompatible(changedPaths, resetCommand, runCommand, hasClickhouse);
         persistence.storageStatus = {
           TAG: "Ready",
           _0: initialState$1