envio 3.13.0-rc.0 → 3.14.0-subgraph

package/bin.mjs

@@ -1,3 +1,12 @@
 #!/usr/bin/env node
-import { run } from "./src/Bin.res.mjs";
+import { unsupportedNodeMessage } from "./src/NodeVersion.mjs";
+
+// Checked before the rest loads, which an older Node may not even parse.
+const unsupported = unsupportedNodeMessage();
+if (unsupported !== undefined) {
+  console.error(unsupported);
+  process.exit(1);
+}
+
+const { run } = await import("./src/Bin.res.mjs");
 await run(process.argv.slice(2));

package/index.d.ts

@@ -12,6 +12,17 @@ declare const bigintSchema: Sury.Schema<bigint>;
 /** Ethereum address — a 20-byte hex string prefixed with `0x`. */
 export type Address = `0x${string}`;
 
+/** A value a `Json` entity field can hold. A `bigint` isn't JSON — store its
+ * decimal string with `.toString()`. Shadows the global `JSON` interface only
+ * where imported; the `JSON` object itself is unaffected. */
+export type JSON =
+  | string
+  | number
+  | boolean
+  | null
+  | readonly JSON[]
+  | { readonly [key: string]: JSON };
+
 /** Structured logger bound to an event or handler context. Messages are
  * displayed in the console and the Envio Hosted Service. */
 export type Logger = {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "envio",
-  "version": "3.13.0-rc.0",
+  "version": "3.14.0-subgraph",
   "type": "module",
   "description": "A latency and sync speed optimized, developer friendly blockchain data indexer.",
   "bin": "./bin.mjs",
@@ -27,7 +27,7 @@
     "dapp"
   ],
   "engines": {
-    "node": ">=22.0.0"
+    "node": ">=22.15.0"
   },
   "files": [
     "bin.mjs",
@@ -42,11 +42,6 @@
   ],
   "dependencies": {
     "@elastic/ecs-pino-format": "1.4.0",
-    "@fuel-ts/crypto": "0.96.1",
-    "@fuel-ts/errors": "0.96.1",
-    "@fuel-ts/hasher": "0.96.1",
-    "@fuel-ts/math": "0.96.1",
-    "@fuel-ts/utils": "0.96.1",
     "bignumber.js": "9.3.1",
     "eventsource": "4.1.0",
     "express": "4.19.2",
@@ -56,20 +51,13 @@
     "rescript-schema": "9.5.1",
     "viem": "2.54.0",
     "dotenv": "16.4.5",
-    "date-fns": "3.3.1",
-    "@rescript/react": "0.14.1",
-    "react": "19.2.5",
-    "ink": "6.8.0",
-    "ink-big-text": "2.0.0",
-    "ink-spinner": "5.0.0",
-    "postgres": "3.4.8",
-    "tsx": "4.21.0"
+    "postgres": "3.4.8"
   },
   "optionalDependencies": {
-    "envio-linux-x64": "3.13.0-rc.0",
-    "envio-linux-x64-musl": "3.13.0-rc.0",
-    "envio-linux-arm64": "3.13.0-rc.0",
-    "envio-darwin-x64": "3.13.0-rc.0",
-    "envio-darwin-arm64": "3.13.0-rc.0"
+    "envio-linux-x64": "3.14.0-subgraph",
+    "envio-linux-x64-musl": "3.14.0-subgraph",
+    "envio-linux-arm64": "3.14.0-subgraph",
+    "envio-darwin-x64": "3.14.0-subgraph",
+    "envio-darwin-arm64": "3.14.0-subgraph"
   }
 }

package/rescript.json

@@ -12,10 +12,7 @@
     "module": "esmodule",
     "in-source": true
   },
-  "jsx": {
-    "version": 4
-  },
-  "dependencies": ["rescript-schema", "@rescript/react"],
+  "dependencies": ["rescript-schema"],
   "compiler-flags": ["-open RescriptSchema"],
   "warnings": {
     "error": "+a"

package/src/BatchProcessing.res

@@ -159,8 +159,6 @@ and processNextBatch = async (state: IndexerState.t, ~scheduleFetch): unit => {
         // Can safely reset rollback state, since overwrite is not possible.
         state->IndexerState.clearRollback
         state->IndexerState.applyBatchProgress(~batch)
-        // Before the finalize below, so a chain says where it finished ahead of
-        // the process saying what it does about that.
         state->IndexerState.reportFinished
 
         // Backfilling → FinalizingIndexes → Ready. Awaiting here holds the

package/src/ChainState.res

@@ -426,7 +426,6 @@ let getLatestValidScannedBlock = (cs: t, ~blockStore: BlockStore.t, ~blockNumber
   ->BlockStore.latestValidBlockFromStore(blockStore, blockNumbers)
   ->Null.toOption
 let safeCheckpointTracking = (cs: t) => cs.safeCheckpointTracking
-let isProgressAtHead = (cs: t) => cs.isProgressAtHead
 let committedProgressBlockNumber = (cs: t) => cs.committedProgressBlockNumber
 let committedProgressBlockTime = (cs: t) => cs.committedProgressBlockTime
 let numEventsProcessed = (cs: t) => cs.numEventsProcessed
@@ -660,30 +659,6 @@ let hasProcessedToEndblock = (cs: t) => {
   }
 }
 
-// Where this chain has finished indexing, the first time it gets there.
-// `EndBlock` is terminal: the chain indexed everything it was configured to.
-// `Backfill` is the rest of the history, up to the point where blocks can
-// still be reorged, which is as far as a chain indexes before the indexer
-// crosses into them.
-type finished = EndBlock(int) | Backfill(int)
-
-let takeFinished = (cs: t) =>
-  if cs.reportedFinished {
-    None
-  } else {
-    switch (cs.fetchState.endBlock, cs->hasProcessedToEndblock, cs.isProgressAtHead) {
-    | (Some(endBlock), true, _) => {
-        cs.reportedFinished = true
-        Some(EndBlock(endBlock))
-      }
-    | (_, _, true) => {
-        cs.reportedFinished = true
-        Some(Backfill(cs.committedProgressBlockNumber))
-      }
-    | _ => None
-    }
-  }
-
 // Caught up as judged by persisted values alone: progress reached the endBlock,
 // or the head the previous run had already observed (less the lag that holds the
 // tip back). Unlike `isFetchingAtHead` this doesn't move when a fresh height
@@ -708,6 +683,45 @@ let isDurablyCaughtUp = (cs: t) => {
   atEndBlock || atHead
 }
 
+// This chain has indexed everything there was to index: it reached its end
+// block, or its progress reached the head. The single reading of that, for
+// everything that turns on it — what the chain reports, and whether the whole
+// process may finalize.
+//
+// Both spellings, because neither covers the other: `isProgressAtHead` latches
+// the moment a batch's progress met the head known when it was created, which a
+// head that has since moved on would read as behind; `isDurablyCaughtUp` judges
+// the head as it stands, which is all a run resumed at the head has — it has no
+// batch to latch anything.
+let hasCaughtUp = (cs: t) => cs.isProgressAtHead || cs->isDurablyCaughtUp
+
+// Where this chain has finished indexing, the first time it gets there.
+// `EndBlock` is terminal: the chain indexed everything it was configured to.
+// `Backfill` is the rest of the history, up to the point where blocks can
+// still be reorged, which is as far as a chain indexes before the indexer
+// crosses into them.
+type finished = EndBlock(int) | Backfill(int)
+
+let takeFinished = (cs: t) =>
+  if cs.reportedFinished {
+    None
+  } else {
+    switch (cs.fetchState.endBlock, cs->hasProcessedToEndblock, cs->hasCaughtUp) {
+    | (Some(endBlock), true, _) => {
+        cs.reportedFinished = true
+        Some(EndBlock(endBlock))
+      }
+    | (_, _, true) => {
+        cs.reportedFinished = true
+        // Finishing a backfill happens in a run, and a chain resumed with its
+        // `ready_at` already committed did none in this one. Being at the end
+        // block, above, stays true across runs and is said on every one.
+        cs->isReady ? None : Some(Backfill(cs.committedProgressBlockNumber))
+      }
+    | _ => None
+    }
+  }
+
 let getHighestBlockBelowThreshold = (cs: t): int => {
   let highestBlockBelowThreshold = cs.fetchState.knownHeight - cs.maxReorgDepth
   highestBlockBelowThreshold < 0 ? 0 : highestBlockBelowThreshold

package/src/ChainState.res.mjs

@@ -220,10 +220,6 @@ function safeCheckpointTracking(cs) {
   return cs.safeCheckpointTracking;
 }
 
-function isProgressAtHead(cs) {
-  return cs.isProgressAtHead;
-}
-
 function committedProgressBlockNumber(cs) {
   return cs.committedProgressBlockNumber;
 }
@@ -421,13 +417,29 @@ function hasProcessedToEndblock(cs) {
   }
 }
 
+function hasCaughtUp(cs) {
+  if (cs.isProgressAtHead) {
+    return true;
+  } else {
+    let fetchState = cs.fetchState;
+    let committedProgressBlockNumber = cs.committedProgressBlockNumber;
+    let atEndBlock = Stdlib_Option.mapOr(fetchState.endBlock, false, endBlock => committedProgressBlockNumber >= endBlock);
+    let atHead = fetchState.knownHeight > 0 && committedProgressBlockNumber >= Primitive_int.max(0, fetchState.knownHeight - cs.chainConfig.blockLag | 0);
+    if (atEndBlock) {
+      return true;
+    } else {
+      return atHead;
+    }
+  }
+}
+
 function takeFinished(cs) {
   if (cs.reportedFinished) {
     return;
   }
   let match = cs.fetchState.endBlock;
   let match$1 = hasProcessedToEndblock(cs);
-  let match$2 = cs.isProgressAtHead;
+  let match$2 = hasCaughtUp(cs);
   if (match !== undefined && match$1) {
     cs.reportedFinished = true;
     return {
@@ -437,22 +449,14 @@ function takeFinished(cs) {
   }
   if (match$2) {
     cs.reportedFinished = true;
-    return {
-      TAG: "Backfill",
-      _0: cs.committedProgressBlockNumber
-    };
-  }
-}
-
-function isDurablyCaughtUp(cs) {
-  let fetchState = cs.fetchState;
-  let committedProgressBlockNumber = cs.committedProgressBlockNumber;
-  let atEndBlock = Stdlib_Option.mapOr(fetchState.endBlock, false, endBlock => committedProgressBlockNumber >= endBlock);
-  let atHead = fetchState.knownHeight > 0 && committedProgressBlockNumber >= Primitive_int.max(0, fetchState.knownHeight - cs.chainConfig.blockLag | 0);
-  if (atEndBlock) {
-    return true;
-  } else {
-    return atHead;
+    if (cs.timestampCaughtUpToHeadOrEndblock !== undefined) {
+      return;
+    } else {
+      return {
+        TAG: "Backfill",
+        _0: cs.committedProgressBlockNumber
+      };
+    }
   }
 }
 
@@ -946,7 +950,6 @@ export {
   getReorgThresholdBlockNumbersBelow,
   getLatestValidScannedBlock,
   safeCheckpointTracking,
-  isProgressAtHead,
   committedProgressBlockNumber,
   committedProgressBlockTime,
   numEventsProcessed,
@@ -980,7 +983,7 @@ export {
   reorgThresholdLiftsCeiling,
   reorgThresholdEntryMessage,
   hasProcessedToEndblock,
-  isDurablyCaughtUp,
+  hasCaughtUp,
   getHighestBlockBelowThreshold,
   isActivelyIndexing,
   isReady,

package/src/ChainState.resi

@@ -70,7 +70,6 @@ let getLatestValidScannedBlock: (
   ~blockNumbers: array<int>,
 ) => option<int>
 let safeCheckpointTracking: t => option<SafeCheckpointTracking.t>
-let isProgressAtHead: t => bool
 let committedProgressBlockNumber: t => int
 let committedProgressBlockTime: t => option<int>
 let numEventsProcessed: t => float
@@ -132,7 +131,7 @@ let takeFinished: t => option<finished>
 let reorgThresholdLiftsCeiling: t => bool
 let reorgThresholdEntryMessage: string
 let hasProcessedToEndblock: t => bool
-let isDurablyCaughtUp: t => bool
+let hasCaughtUp: t => bool
 let getHighestBlockBelowThreshold: t => int
 let isActivelyIndexing: t => bool
 let isReady: t => bool

package/src/Config.res

@@ -136,6 +136,9 @@ type t = {
   reorgThresholdReadyTolerance: int,
   lowercaseAddresses: bool,
   isDev: bool,
+  // The operator's `-d` / `--config` flags, repeated in every command printed
+  // for them to run next. Empty when they gave neither.
+  projectFlags: string,
   // An `envio start --chain` process: drives a subset of the schema's chains
   // while sibling processes drive the rest, so it only touches what its own
   // chains own — their partitions' indexes, their `ready_at`, their resume.
@@ -147,6 +150,10 @@ type t = {
   // the schema alone: a cross-chain entity has rows any chain's reorg can
   // reach, so its checkpoints have to be comparable across chains.
   checkpointSequence: CheckpointSequence.t,
+  envioInfo: JSON.t,
+  // Set only in subgraph mode: the translated manifest, which is what makes
+  // the subgraph runtime take over handler registration.
+  subgraph: option<JSON.t>,
 }
 
 type rpcSourceFor = | @as("sync") Sync | @as("fallback") Fallback | @as("realtime") Realtime
@@ -595,6 +602,7 @@ let publicConfigSchema = S.schema(s =>
     "handlers": s.matches(S.option(S.string)),
     "isDev": s.matches(S.option(S.bool)),
     "isolatedChains": s.matches(S.option(S.array(ChainId.schema))),
+    "projectFlags": s.matches(S.option(S.string)),
     "fullBatchSize": s.matches(S.option(S.int)),
     "rollbackOnReorg": s.matches(S.option(S.bool)),
     "saveFullHistory": s.matches(S.option(S.bool)),
@@ -607,6 +615,7 @@ let publicConfigSchema = S.schema(s =>
     "svm": s.matches(S.option(publicConfigEcosystemSchema)),
     "enums": s.matches(S.option(S.dict(S.array(S.string)))),
     "entities": s.matches(S.option(S.array(entityJsonSchema))),
+    "subgraph": s.matches(S.option(S.json(~validate=false))),
   }
 )
 
@@ -666,6 +675,45 @@ let isolate = (config: t, ~chainIds: array<ChainId.t>) => {
   }
 }
 
+// What the command decided rather than the project's files: `envio dev` vs
+// `envio start`, which chains this process drives, and how it found the
+// project.
+let commandFields = ["isDev", "isolatedChains", "projectFlags"]
+
+let withProjectFlags = (command, ~config: t) =>
+  switch config.projectFlags {
+  | "" => command
+  | flags => `${command} ${flags}`
+  }
+
+let ecosystemFields = ["evm", "fuel", "svm"]
+
+// What `envio_info` records: the public config without the command's fields
+// or its chains. A chain is recorded by its own `envio_chains` row and its
+// config's `envio_addresses` rows instead, so adding one never rewrites this
+// record. The rest of a chain's config — its sources, its block lag, a
+// contract's start block — is read from config.yaml on every start.
+let toEnvioInfo = (publicConfigJson: JSON.t): JSON.t => {
+  let envioInfo = publicConfigJson->JSON.stringify->JSON.parseOrThrow
+  switch envioInfo {
+  | Object(obj) => {
+      commandFields->Array.forEach(field => obj->Utils.Dict.deleteInPlace(field))
+      // The subgraph blob is how the runtime finds mappings. None of it
+      // describes stored data — that's already in `evm` / `entities` — so a
+      // specVersion bump or a mapping-path edit must not force a reset.
+      obj->Utils.Dict.deleteInPlace("subgraph")
+      ecosystemFields->Array.forEach(ecosystem =>
+        switch obj->Dict.get(ecosystem) {
+        | Some(Object(ecosystemDict)) => ecosystemDict->Utils.Dict.deleteInPlace("chains")
+        | _ => ()
+        }
+      )
+    }
+  | _ => ()
+  }
+  envioInfo
+}
+
 let fromPublic = (publicConfigJson: JSON.t) => {
   let maxAddrInPartition = Env.maxAddrInPartition
   // Parse public config
@@ -1136,11 +1184,14 @@ let fromPublic = (publicConfigJson: JSON.t) => {
     reorgThresholdReadyTolerance: 100,
     lowercaseAddresses,
     isDev: publicConfig["isDev"]->Option.getOr(false),
+    projectFlags: publicConfig["projectFlags"]->Option.getOr(""),
     isolated: false,
     userEntitiesByName,
     userEntities,
     allEnums,
     checkpointSequence: CheckpointSequence.fromEntities(userEntities),
+    envioInfo: publicConfigJson->toEnvioInfo,
+    subgraph: publicConfig["subgraph"],
   }
 
   switch publicConfig["isolatedChains"] {
@@ -1255,49 +1306,6 @@ let getPublicConfigJson = () =>
   | None => Core.getConfigJson()->JSON.parseOrThrow
   }
 
-// Drops source URLs from each chain so RPC/hypersync edits don't trigger
-// the resume-time compat check (and don't end up in `envio_info`). Also
-// drops `isDev`, which toggles between `envio dev` and `envio start` and
-// has no bearing on schema/indexing compatibility.
-let stripSensitiveData = (json: JSON.t): JSON.t => {
-  let cloned = json->JSON.stringify->JSON.parseOrThrow
-  let stripChains = (ecosystem: option<JSON.t>) =>
-    switch ecosystem {
-    | Some(Object(ecosystemDict)) =>
-      switch ecosystemDict->Dict.get("chains") {
-      | Some(Object(chains)) =>
-        chains
-        ->Dict.valuesToArray
-        ->Array.forEach(chainJson =>
-          switch chainJson {
-          | Object(chain) => {
-              chain->Utils.Dict.deleteInPlace("rpcs")
-              chain->Utils.Dict.deleteInPlace("hypersync")
-            }
-          | _ => ()
-          }
-        )
-      | _ => ()
-      }
-    | _ => ()
-    }
-  switch cloned {
-  | Object(obj) => {
-      obj->Utils.Dict.deleteInPlace("isDev")
-      obj->Utils.Dict.deleteInPlace("isolatedChains")
-      stripChains(obj->Dict.get("evm"))
-      stripChains(obj->Dict.get("fuel"))
-      stripChains(obj->Dict.get("svm"))
-    }
-  | _ => ()
-  }
-  cloned
-}
-
-// What the storage layer records as the config this schema was built from,
-// and checks a resuming run against.
-let envioInfo = () => getPublicConfigJson()->stripSensitiveData
-
 // Postgres jsonb doesn't preserve key order, so canonicalize with sorted
 // keys before string-comparing.
 let rec canonicalJson = (json: JSON.t): JSON.t =>
@@ -1317,9 +1325,10 @@ let rec canonicalJson = (json: JSON.t): JSON.t =>
 // Returns dotted leaf paths (`a.b[i].c`) where `stored` differs from
 // `current`, restricted to the highest-priority top-level tier with any
 // diff. Tiers in order: version → name → storage → ecosystem
-// (evm/fuel/svm) → entities → other top-level keys. The first tier
-// containing a diff is the only one rendered; lower tiers are silenced
-// so a single noisy section doesn't bury the actionable change.
+// (evm/fuel/svm, and the chains a resume puts beside them) → entities →
+// other top-level keys. The first tier containing a diff is the only one
+// rendered; lower tiers are silenced so a single noisy section doesn't bury
+// the actionable change.
 let diffPaths = (~stored: JSON.t, ~current: JSON.t): array<string> => {
   let canonEq = (a: JSON.t, b: JSON.t) =>
     JSON.stringify(canonicalJson(a)) === JSON.stringify(canonicalJson(b))
@@ -1387,7 +1396,7 @@ let diffPaths = (~stored: JSON.t, ~current: JSON.t): array<string> => {
       ["chainIdMode"],
       ["name"],
       ["storage"],
-      ["evm", "fuel", "svm"],
+      ["evm", "fuel", "svm", "chains"],
       ["entities"],
     ]
     let firstHit = tiers->Array.reduce(None, (acc, tier) =>
@@ -1417,65 +1426,34 @@ let diffPaths = (~stored: JSON.t, ~current: JSON.t): array<string> => {
   acc
 }
 
-// Throws an `incompatible config` error listing each path in `changedPaths`,
-// plus the remediation options. `~resetCommand` is rendered as-is for
-// option 2 (the wipe-and-redo). `~runCommand` controls option 3 (parallel
-// indexer recipe): when `None`, option 3 is omitted — the migrate flow
-// uses this because running a second indexer doesn't apply.
-// `~hasClickhouse` adds the extra env line so users running both
-// Postgres and Clickhouse get a complete override.
-let throwIfIncompatible = (
+// The `incompatible config` error listing each path in `changedPaths`, plus
+// the remediation options. `~resetCommand` is rendered as-is for option 2
+// (the wipe-and-redo). `~runCommand` controls option 3 (parallel indexer
+// recipe): when `None`, option 3 is omitted — the migrate flow uses this
+// because running a second indexer doesn't apply. `~hasClickhouse` adds the
+// extra env line so users running both Postgres and Clickhouse get a complete
+// override.
+let incompatibleMessage = (
   changedPaths: array<string>,
   ~resetCommand: string,
   ~runCommand: option<string>,
   ~hasClickhouse: bool,
 ) => {
-  if changedPaths->Array.length > 0 {
-    let bullets = changedPaths->Array.map(p => `    - ${p}`)->Array.joinUnsafe("\n")
-    let option1 = "Revert the changes above"
-    let padTo = (s, col) => s ++ " "->String.repeat(Math.Int.max(col - String.length(s), 1))
-    let col = Math.Int.max(String.length(option1), String.length(resetCommand)) + 2
-    let option3 = switch runCommand {
-    | None => ""
-    | Some(cmd) =>
-      let clickhouseLine = hasClickhouse ? "       ENVIO_CLICKHOUSE_DATABASE=<new_db> \\\n" : ""
-      `\n  3. Run a second indexer alongside this one — keep both datasets:\n       ENVIO_PG_SCHEMA=<new_schema> \\\n${clickhouseLine}       ENVIO_INDEXER_PORT=<new_port> \\\n       ${cmd}`
-    }
-    JsError.throwWithMessage(
-      `The following config changes are incompatible with the existing indexer data:\n\n${bullets}\n\nPick one:\n  1. ${option1->padTo(
-          col,
-        )}# resume indexing where it left off\n  2. ${resetCommand->padTo(
-          col,
-        )}# delete all indexed data and start over${option3}`,
-    )
-  }
-}
-
-let throwIfResumeIncompatible = (
-  ~storedEnvioInfo: option<JSON.t>,
-  ~storedContractMapping: ContractMapping.t,
-  ~envioInfo: JSON.t,
-  ~contractMapping: ContractMapping.t,
-  ~resetCommand: string,
-  ~runCommand: option<string>,
-) => {
-  let changedPaths = switch storedEnvioInfo {
-  | None => ["storage was initialized by an older envio version"]
-  | Some(stored) => diffPaths(~stored, ~current=envioInfo)
-  }
-  let changedPaths =
-    storedContractMapping->ContractMapping.isEqual(contractMapping)
-      ? changedPaths
-      : changedPaths->Array.concat(["contracts"])
-  let hasClickhouse = switch envioInfo {
-  | Object(d) =>
-    switch d->Dict.get("storage") {
-    | Some(Object(s)) => s->Dict.get("clickhouse") == Some(Boolean(true))
-    | _ => false
-    }
-  | _ => false
+  let bullets = changedPaths->Array.map(p => `    - ${p}`)->Array.joinUnsafe("\n")
+  let option1 = "Revert the changes above"
+  let padTo = (s, col) => s ++ " "->String.repeat(Math.Int.max(col - String.length(s), 1))
+  let col = Math.Int.max(String.length(option1), String.length(resetCommand)) + 2
+  let option3 = switch runCommand {
+  | None => ""
+  | Some(cmd) =>
+    let clickhouseLine = hasClickhouse ? "       ENVIO_CLICKHOUSE_DATABASE=<new_db> \\\n" : ""
+    `\n  3. Run a second indexer alongside this one — keep both datasets:\n       ENVIO_PG_SCHEMA=<new_schema> \\\n${clickhouseLine}       ENVIO_INDEXER_PORT=<new_port> \\\n       ${cmd}`
   }
-  throwIfIncompatible(changedPaths, ~resetCommand, ~runCommand, ~hasClickhouse)
+  `The following config changes are incompatible with the existing indexer data:\n\n${bullets}\n\nPick one:\n  1. ${option1->padTo(
+      col,
+    )}# resume indexing where it left off\n  2. ${resetCommand->padTo(
+      col,
+    )}# delete all indexed data and start over${option3}`
 }
 
 // The returned value is a pure function of the JSON: it holds only event