envio 3.9.0 → 3.10.0

package/src/db/InternalTable.res.mjs

@@ -9,6 +9,7 @@ import * as AddressRows from "../AddressRows.res.mjs";
 import * as Stdlib_Null from "@rescript/runtime/lib/es6/Stdlib_Null.js";
 import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js";
 import * as Stdlib_JsError from "@rescript/runtime/lib/es6/Stdlib_JsError.js";
+import * as CheckpointBounds from "./CheckpointBounds.res.mjs";
 import * as S$RescriptSchema from "rescript-schema/src/S.res.mjs";
 import * as Primitive_exceptions from "@rescript/runtime/lib/es6/Primitive_exceptions.js";
 
@@ -389,13 +390,35 @@ let dbSchema = S$RescriptSchema.object(s => ({
   events_processed: s.f("events_processed", S$RescriptSchema.int)
 }));
 
-let table$4 = Table.mkTable("envio_checkpoints", undefined, [
-  Table.mkField("id", "UInt64", S$RescriptSchema.bigint, undefined, undefined, undefined, true, undefined, undefined, undefined, undefined, undefined, undefined),
-  Table.mkField("chain_id", "ChainId", ChainId.schema, undefined, undefined, undefined, undefined, undefined, undefined, undefined, undefined, undefined, undefined),
-  Table.mkField("block_number", "Int32", S$RescriptSchema.int, undefined, undefined, undefined, undefined, undefined, undefined, undefined, undefined, undefined, undefined),
-  Table.mkField("block_hash", "String", S$RescriptSchema.$$null(S$RescriptSchema.string), undefined, undefined, true, undefined, undefined, undefined, undefined, undefined, undefined, undefined),
-  Table.mkField("events_processed", "Int32", S$RescriptSchema.int, undefined, undefined, undefined, undefined, undefined, undefined, undefined, undefined, undefined, undefined)
-], undefined);
+let columns = [
+  {
+    field: Table.mkField("id", "UInt64", S$RescriptSchema.bigint, undefined, undefined, undefined, true, undefined, undefined, undefined, undefined, undefined, undefined),
+    clickHouseFieldType: "UInt64",
+    valuesOf: batch => batch.checkpointIds
+  },
+  {
+    field: Table.mkField("chain_id", "ChainId", ChainId.schema, undefined, undefined, undefined, undefined, undefined, undefined, undefined, undefined, undefined, undefined),
+    clickHouseFieldType: "ChainId",
+    valuesOf: batch => batch.checkpointChainIds
+  },
+  {
+    field: Table.mkField("block_number", "Int32", S$RescriptSchema.int, undefined, undefined, undefined, undefined, undefined, undefined, undefined, undefined, undefined, undefined),
+    clickHouseFieldType: "Int32",
+    valuesOf: batch => batch.checkpointBlockNumbers
+  },
+  {
+    field: Table.mkField("block_hash", "String", S$RescriptSchema.$$null(S$RescriptSchema.string), undefined, undefined, true, undefined, undefined, undefined, undefined, undefined, undefined, undefined),
+    clickHouseFieldType: "String",
+    valuesOf: batch => batch.checkpointBlockHashes
+  },
+  {
+    field: Table.mkField("events_processed", "Int32", S$RescriptSchema.int, undefined, undefined, undefined, undefined, undefined, undefined, undefined, undefined, undefined, undefined),
+    clickHouseFieldType: "UInt64",
+    valuesOf: batch => batch.checkpointEventsProcessed
+  }
+];
+
+let table$4 = Table.mkTable("envio_checkpoints", undefined, columns.map(param => param.field), undefined);
 
 function makeGetReorgCheckpointsQuery(pgSchema) {
   return `WITH reorg_chains AS (
@@ -415,7 +438,8 @@ FROM "` + pgSchema + `"."` + table$4.tableName + `" cp
 INNER JOIN reorg_chains rc 
   ON cp."` + "chain_id" + `" = rc.id
 WHERE cp."` + "block_hash" + `" IS NOT NULL
-  AND cp."` + "block_number" + `" >= rc.safe_block;`;
+  AND cp."` + "block_number" + `" >= rc.safe_block
+ORDER BY cp."` + "id" + `";`;
 }
 
 function makeCommitedCheckpointIdQuery(pgSchema) {
@@ -442,16 +466,22 @@ function insert$2(sql, pgSchema, checkpointIds, checkpointChainIds, checkpointBl
   ], {prepare: true});
 }
 
-function rollback(sql, pgSchema, rollbackTargetCheckpointId) {
-  return sql.unsafe(`DELETE FROM "` + pgSchema + `"."` + table$4.tableName + `" WHERE "` + "id" + `" > $1;`, [rollbackTargetCheckpointId.toString()], {prepare: true});
+let chainIdColumn = "chain_id";
+
+function rollback(sql, pgSchema, floors) {
+  let tableRef = `"` + pgSchema + `"."` + table$4.tableName + `"`;
+  let bounds = CheckpointBounds.sql(floors.floors, chainIdColumn, tableRef);
+  return sql.unsafe(`DELETE FROM ` + tableRef + bounds.using + ` WHERE "` + "id" + `" > ` + bounds.checkpointId + bounds.usingMatch + `;`, CheckpointBounds.params(floors.floors), {prepare: true});
 }
 
-function makePruneStaleCheckpointsQuery(pgSchema) {
-  return `DELETE FROM "` + pgSchema + `"."` + table$4.tableName + `" WHERE "` + "id" + `" < $1;`;
+function makePruneStaleCheckpointsQuery(pgSchema, safeCheckpoints) {
+  let tableRef = `"` + pgSchema + `"."` + table$4.tableName + `"`;
+  let bounds = CheckpointBounds.sql(safeCheckpoints, chainIdColumn, tableRef);
+  return `DELETE FROM ` + tableRef + bounds.using + ` WHERE "` + "id" + `" < ` + bounds.checkpointId + bounds.usingMatch + `;`;
 }
 
-function pruneStaleCheckpoints(sql, pgSchema, safeCheckpointId) {
-  return sql.unsafe(makePruneStaleCheckpointsQuery(pgSchema), [safeCheckpointId.toString()], {prepare: true});
+function pruneStaleCheckpoints(sql, pgSchema, safeCheckpoints) {
+  return sql.unsafe(makePruneStaleCheckpointsQuery(pgSchema, safeCheckpoints), CheckpointBounds.params(safeCheckpoints), {prepare: true});
 }
 
 function makeGetRollbackTargetCheckpointQuery(pgSchema) {
@@ -471,28 +501,31 @@ function getRollbackTargetCheckpoint(sql, pgSchema, reorgChainId, lastKnownValid
   return rawResult.then(rows => Stdlib_Option.map(rows[0], row => BigInt(row.id)));
 }
 
-function makeGetRollbackProgressDiffQuery(pgSchema) {
+function makeGetRollbackProgressDiffQuery(pgSchema, floors) {
+  let bounds = CheckpointBounds.sql(floors.floors, chainIdColumn, "t");
   return `SELECT 
-  "` + "chain_id" + `"::float8 as "` + "chain_id" + `",
-  SUM("` + "events_processed" + `") as events_processed_diff,
-  MIN("` + "block_number" + `") - 1 as new_progress_block_number
-FROM "` + pgSchema + `"."` + table$4.tableName + `"
-WHERE "` + "id" + `" > $1
-GROUP BY "` + "chain_id" + `";`;
+  t."` + "chain_id" + `"::float8 as "` + "chain_id" + `",
+  SUM(t."` + "events_processed" + `") as events_processed_diff,
+  MIN(t."` + "block_number" + `") - 1 as new_progress_block_number
+FROM "` + pgSchema + `"."` + table$4.tableName + `" t` + bounds.join + `
+WHERE t."` + "id" + `" > ` + bounds.checkpointId + `
+GROUP BY t."` + "chain_id" + `";`;
 }
 
-function getRollbackProgressDiff(sql, pgSchema, rollbackTargetCheckpointId) {
-  return sql.unsafe(makeGetRollbackProgressDiffQuery(pgSchema), [rollbackTargetCheckpointId.toString()], {prepare: true});
+function getRollbackProgressDiff(sql, pgSchema, floors) {
+  return sql.unsafe(makeGetRollbackProgressDiffQuery(pgSchema, floors), CheckpointBounds.params(floors.floors), {prepare: true});
 }
 
 let Checkpoints = {
   dbSchema: dbSchema,
   initialCheckpointId: 0n,
+  columns: columns,
   table: table$4,
   makeGetReorgCheckpointsQuery: makeGetReorgCheckpointsQuery,
   makeCommitedCheckpointIdQuery: makeCommitedCheckpointIdQuery,
   makeInsertCheckpointQuery: makeInsertCheckpointQuery,
   insert: insert$2,
+  chainIdColumn: chainIdColumn,
   rollback: rollback,
   makePruneStaleCheckpointsQuery: makePruneStaleCheckpointsQuery,
   pruneStaleCheckpoints: pruneStaleCheckpoints,

package/src/db/RollbackFloors.res

@@ -0,0 +1,55 @@
+// How far back a reorg rollback takes each chain: the last checkpoint each keeps,
+// and for a chain whose own reorg set its floor, the last block that reorg left
+// valid. Superseding an unwritten rollback with a new one is a pointwise
+// minimum, so a merge can neither lose a chain's floor nor raise it.
+type t = {
+  floors: CheckpointBounds.t,
+  // Progress recomputed from the surviving checkpoints alone can land above the
+  // fork: the blocks between it and the chain's next checkpoint carried no
+  // events on the orphaned chain, but the chain replacing it can have its own.
+  // Absent for a chain dragged along by another chain's reorg.
+  forkBlockNumberByChain: array<(ChainId.t, int)>,
+}
+
+// The rollback of a schema whose chains can't have written each other's rows.
+let isolated = (~chainId, ~floorCheckpointId, ~forkBlockNumber) => {
+  floors: PerChain([(chainId, floorCheckpointId)]),
+  forkBlockNumberByChain: [(chainId, forkBlockNumber)],
+}
+
+// The rollback of a schema with a cross-chain entity: every chain goes back to
+// the same checkpoint.
+let global = (~floorCheckpointId, ~reorgChainId, ~forkBlockNumber) => {
+  floors: EveryChain(floorCheckpointId),
+  forkBlockNumberByChain: [(reorgChainId, forkBlockNumber)],
+}
+
+let mergeByChain = (pending: array<(ChainId.t, 'a)>, next: array<(ChainId.t, 'a)>) => {
+  let merged = pending->Array.copy
+  next->Array.forEach(((chainId, value)) =>
+    switch merged->Array.findIndexOpt(((mergedChainId, _)) => mergedChainId === chainId) {
+    | Some(idx) =>
+      let (_, pendingValue) = merged->Array.getUnsafe(idx)
+      merged->Array.setUnsafe(idx, (chainId, Pervasives.min(pendingValue, value)))
+    | None => merged->Array.push((chainId, value))
+    }
+  )
+  merged
+}
+
+let merge = (pending: t, next: t) => {
+  floors: switch (pending.floors, next.floors) {
+  | (EveryChain(pending), EveryChain(next)) => EveryChain(Pervasives.min(pending, next))
+  | (PerChain(pending), PerChain(next)) => PerChain(mergeByChain(pending, next))
+  | (EveryChain(_), PerChain(_)) | (PerChain(_), EveryChain(_)) =>
+    JsError.throwWithMessage(
+      "Internal error: a rollback reaching every chain can't be merged with one isolated to a chain. The schema decides which a run builds, and it doesn't change.",
+    )
+  },
+  forkBlockNumberByChain: mergeByChain(pending.forkBlockNumberByChain, next.forkBlockNumberByChain),
+}
+
+let forkBlockNumber = (floors: t, chainId) =>
+  floors.forkBlockNumberByChain
+  ->Array.find(((forkChainId, _)) => forkChainId === chainId)
+  ->Option.map(((_, forkBlockNumber)) => forkBlockNumber)

package/src/db/RollbackFloors.res.mjs

@@ -0,0 +1,92 @@
+// Generated by ReScript, PLEASE EDIT WITH CARE
+
+import * as Stdlib_Array from "@rescript/runtime/lib/es6/Stdlib_Array.js";
+import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js";
+import * as Stdlib_JsError from "@rescript/runtime/lib/es6/Stdlib_JsError.js";
+import * as Primitive_bigint from "@rescript/runtime/lib/es6/Primitive_bigint.js";
+import * as Primitive_object from "@rescript/runtime/lib/es6/Primitive_object.js";
+
+function isolated(chainId, floorCheckpointId, forkBlockNumber) {
+  return {
+    floors: {
+      TAG: "PerChain",
+      _0: [[
+          chainId,
+          floorCheckpointId
+        ]]
+    },
+    forkBlockNumberByChain: [[
+        chainId,
+        forkBlockNumber
+      ]]
+  };
+}
+
+function global(floorCheckpointId, reorgChainId, forkBlockNumber) {
+  return {
+    floors: {
+      TAG: "EveryChain",
+      _0: floorCheckpointId
+    },
+    forkBlockNumberByChain: [[
+        reorgChainId,
+        forkBlockNumber
+      ]]
+  };
+}
+
+function mergeByChain(pending, next) {
+  let merged = pending.slice();
+  next.forEach(param => {
+    let value = param[1];
+    let chainId = param[0];
+    let idx = Stdlib_Array.findIndexOpt(merged, param => param[0] === chainId);
+    if (idx !== undefined) {
+      let match = merged[idx];
+      merged[idx] = [
+        chainId,
+        Primitive_object.min(match[1], value)
+      ];
+      return;
+    }
+    merged.push([
+      chainId,
+      value
+    ]);
+  });
+  return merged;
+}
+
+function merge(pending, next) {
+  let match = pending.floors;
+  let match$1 = next.floors;
+  let tmp;
+  tmp = match.TAG === "EveryChain" ? (
+      match$1.TAG === "EveryChain" ? ({
+          TAG: "EveryChain",
+          _0: Primitive_bigint.min(match._0, match$1._0)
+        }) : Stdlib_JsError.throwWithMessage("Internal error: a rollback reaching every chain can't be merged with one isolated to a chain. The schema decides which a run builds, and it doesn't change.")
+    ) : (
+      match$1.TAG === "EveryChain" ? Stdlib_JsError.throwWithMessage("Internal error: a rollback reaching every chain can't be merged with one isolated to a chain. The schema decides which a run builds, and it doesn't change.") : ({
+          TAG: "PerChain",
+          _0: mergeByChain(match._0, match$1._0)
+        })
+    );
+  return {
+    floors: tmp,
+    forkBlockNumberByChain: mergeByChain(pending.forkBlockNumberByChain, next.forkBlockNumberByChain)
+  };
+}
+
+function forkBlockNumber(floors, chainId) {
+  return Stdlib_Option.map(floors.forkBlockNumberByChain.find(param => param[0] === chainId), param => param[1]);
+}
+
+export {
+  isolated,
+  global,
+  mergeByChain,
+  merge,
+  forkBlockNumber,
+}
+/* No side effect */

package/src/db/Table.res

@@ -22,8 +22,9 @@ type fieldType =
   | Uint32
   | UInt52
   | SmallInt
-  // Raw bytes. Only the internal tables use it — the one column that stores an
-  // address in the binary form the Rust address store keys on.
+  // Raw bytes: the `Bytes` scalar under `bytes_type: uint8array`, and the
+  // internal column that stores an address in the binary form the Rust address
+  // store keys on.
   | Bytea
   | UInt64
   | Int32
@@ -288,6 +289,9 @@ let getChainIdField = (table): option<field> =>
     }
   )
 
+// None for a cross-chain entity, whose rows no single chain owns.
+let getPgChainIdColumn = table => table->getChainIdField->Option.map(getPgDbFieldName)
+
 let getFieldByName = (table, fieldName) =>
   table.fields->Array.find(field => field->getUserDefinedFieldName === fieldName)
 
@@ -360,7 +364,10 @@ let queryFields: table => dict<queryField> = Utils.WeakMap.memoize(table => {
         field->getApiFieldName,
         {
           fieldSchema: field.fieldSchema,
-          arrayFieldSchema: S.array(field.fieldSchema)->S.toUnknown,
+          arrayFieldSchema: switch field.fieldType {
+          | Bytea => Utils.Schema.bytesArray->S.toUnknown
+          | _ => S.array(field.fieldSchema)->S.toUnknown
+          },
           pgDbFieldName: field->getPgDbFieldName,
           isChainId: field.isChainId,
         },
@@ -431,6 +438,7 @@ type sqlParams<'entity> = {
   quotedFieldNames: array<string>,
   quotedNonPrimaryFieldNames: array<string>,
   arrayFieldTypes: array<string>,
+  byteaColumnIndexes: array<int>,
   hasArrayField: bool,
 }
 
@@ -438,6 +446,9 @@ let toSqlParams = (table: table, ~schema, ~pgSchema, ~chainIdMode: ChainId.mode=
   let quotedFieldNames = []
   let quotedNonPrimaryFieldNames = []
   let arrayFieldTypes = []
+  // Positions of the bytea columns among the unnest parameters, which the
+  // caller binds as array literals (see `Utils.Bytes.toPgArrayLiteral`).
+  let byteaColumnIndexes = []
   let hasArrayField = ref(false)
 
   let dbSchema: S.t<dict<unknown>> = S.schema(s =>
@@ -450,7 +461,7 @@ let toSqlParams = (table: table, ~schema, ~pgSchema, ~chainIdMode: ChainId.mode=
           | BigInt => Utils.BigInt.schema->S.toUnknown
           | Option(child)
           | Null(child) =>
-            S.null(child->coerceSchema)->S.toUnknown
+            Utils.Schema.nullTolerant(child->coerceSchema)->S.toUnknown
           | Array(child) => {
               hasArrayField := true
               S.array(child->coerceSchema)->S.toUnknown
@@ -472,6 +483,12 @@ let toSqlParams = (table: table, ~schema, ~pgSchema, ~chainIdMode: ChainId.mode=
         | Some(field) => field
         | None => throw(NonExistingTableField(location))
         }
+        switch field {
+        | Field({isArray: true}) => hasArrayField := true
+        | Field({fieldType: Bytea}) =>
+          byteaColumnIndexes->Array.push(arrayFieldTypes->Array.length)->ignore
+        | _ => ()
+        }
 
         // Schema locations use API field names, while the SQL references
         // columns by their possibly renamed db names.
@@ -521,6 +538,7 @@ let toSqlParams = (table: table, ~schema, ~pgSchema, ~chainIdMode: ChainId.mode=
     quotedFieldNames,
     quotedNonPrimaryFieldNames,
     arrayFieldTypes,
+    byteaColumnIndexes,
     hasArrayField: hasArrayField.contents,
   }
 }

package/src/db/Table.res.mjs

@@ -253,6 +253,10 @@ function getChainIdField(table) {
   });
 }
 
+function getPgChainIdColumn(table) {
+  return Stdlib_Option.map(getChainIdField(table), getPgDbFieldName);
+}
+
 function getFieldByName(table, fieldName) {
   return table.fields.find(field => getUserDefinedFieldName(field) === fieldName);
 }
@@ -307,9 +311,12 @@ let queryFields = Utils.$$WeakMap.memoize(table => {
       return;
     }
     let field$1 = field._0;
+    let match = field$1.fieldType;
+    let tmp;
+    tmp = typeof match !== "object" && match === "Bytea" ? Utils.Schema.bytesArray : S$RescriptSchema.array(field$1.fieldSchema);
     dict[getApiFieldName(field$1)] = {
       fieldSchema: field$1.fieldSchema,
-      arrayFieldSchema: S$RescriptSchema.array(field$1.fieldSchema),
+      arrayFieldSchema: tmp,
       pgDbFieldName: getPgDbFieldName(field$1),
       isChainId: field$1.isChainId
     };
@@ -368,6 +375,7 @@ function toSqlParams(table, schema, pgSchema, chainIdModeOpt) {
   let quotedFieldNames = [];
   let quotedNonPrimaryFieldNames = [];
   let arrayFieldTypes = [];
+  let byteaColumnIndexes = [];
   let hasArrayField = {
     contents: false
   };
@@ -400,7 +408,7 @@ function toSqlParams(table, schema, pgSchema, chainIdModeOpt) {
           switch (child.TAG) {
             case "option" :
             case "null" :
-              return S$RescriptSchema.$$null(coerceSchema(child._0));
+              return Utils.Schema.nullTolerant(coerceSchema(child._0));
             case "array" :
               hasArrayField.contents = true;
               return S$RescriptSchema.array(coerceSchema(child._0));
@@ -423,6 +431,17 @@ function toSqlParams(table, schema, pgSchema, chainIdModeOpt) {
           Error: new Error()
         };
       }
+      if (field$1.TAG === "Field") {
+        let match = field$1._0;
+        if (match.isArray) {
+          hasArrayField.contents = true;
+        } else {
+          let tmp = match.fieldType;
+          if (typeof tmp !== "object" && tmp === "Bytea") {
+            byteaColumnIndexes.push(arrayFieldTypes.length);
+          }
+        }
+      }
       let quotedDbName = `"` + getPgFieldName(field$1) + `"`;
       quotedFieldNames.push(quotedDbName);
       if (field$1.TAG === "Field") {
@@ -432,20 +451,20 @@ function toSqlParams(table, schema, pgSchema, chainIdModeOpt) {
           quotedNonPrimaryFieldNames.push(quotedDbName);
         }
       }
-      let tmp;
+      let tmp$1;
       if (field$1.TAG === "Field") {
         let f = field$1._0;
         let pgFieldType = getPgFieldType(f.fieldType, pgSchema, true, false, f.isNullable, chainIdMode);
-        let match = f.fieldType;
-        tmp = typeof match !== "object" ? (
-            match === "Boolean" ? "INTEGER" + `[]::` + pgFieldType : pgFieldType
+        let match$1 = f.fieldType;
+        tmp$1 = typeof match$1 !== "object" ? (
+            match$1 === "Boolean" ? "INTEGER" + `[]::` + pgFieldType : pgFieldType
           ) : (
-            match.type === "Enum" ? "TEXT" + `[]::` + pgFieldType : pgFieldType
+            match$1.type === "Enum" ? "TEXT" + `[]::` + pgFieldType : pgFieldType
           );
       } else {
-        tmp = "TEXT[]";
+        tmp$1 = "TEXT[]";
       }
-      arrayFieldTypes.push(tmp);
+      arrayFieldTypes.push(tmp$1);
       dict[location] = s.m(coerceSchema(param.schema));
     });
     return dict;
@@ -455,6 +474,7 @@ function toSqlParams(table, schema, pgSchema, chainIdModeOpt) {
     quotedFieldNames: quotedFieldNames,
     quotedNonPrimaryFieldNames: quotedNonPrimaryFieldNames,
     arrayFieldTypes: arrayFieldTypes,
+    byteaColumnIndexes: byteaColumnIndexes,
     hasArrayField: hasArrayField.contents
   };
 }
@@ -505,6 +525,7 @@ export {
   getLinkedEntityFields,
   getDerivedFromFields,
   getChainIdField,
+  getPgChainIdColumn,
   getFieldByName,
   NoIdField,
   getIdFieldOrThrow,

package/src/sources/Evm.res

@@ -77,6 +77,8 @@ let make = (~logger: Pino.t): Ecosystem.t => {
   blockTimestampName: "timestamp",
   blockHashName: "hash",
   onBlockMethodName: "onBlock",
+  contractNoun: "contract",
+  eventNoun: "event",
   // EVM filter shape: `{block: {number: {_gte?, _lte?, _every?}}}`.
   // The inner range chunk is returned as raw `S.unknown` and parsed a
   // second time in `Main.res` by the shared `blockRangeSchema`.

package/src/sources/Evm.res.mjs

@@ -56,6 +56,8 @@ function make(logger) {
     blockTimestampName: "timestamp",
     blockHashName: "hash",
     onBlockMethodName: "onBlock",
+    contractNoun: "contract",
+    eventNoun: "event",
     onBlockFilterSchema: S$RescriptSchema.object(s => s.f("block", S$RescriptSchema.option(S$RescriptSchema.object(s2 => s2.f("number", S$RescriptSchema.unknown))))),
     onEventBlockFilterSchema: S$RescriptSchema.object(s => s.f("block", S$RescriptSchema.option(S$RescriptSchema.strict(S$RescriptSchema.object(s2 => s2.f("number", S$RescriptSchema.unknown)))))),
     logger: logger,

package/src/sources/EvmHyperSyncSource.res

@@ -205,7 +205,7 @@ Learn more or get a free Envio API token at: https://envio.dev/app/api-tokens`)
     name,
     sourceFor: Sync,
     chainId,
-    pollingInterval: 100,
+    pollingInterval: HyperSync.pollingInterval,
     poweredByHyperSync: true,
     getBlockHashes,
     getHeightOrThrow: async () => {
@@ -227,7 +227,7 @@ Learn more or get a free Envio API token at: https://envio.dev/app/api-tokens`)
       {height, requestStats: [{method: "getHeight", seconds}]}
     },
     getItemsOrThrow,
-    createHeightSubscription: (~onHeight) =>
-      HyperSyncHeightStream.subscribe(~hyperSyncUrl=endpointUrl, ~apiToken, ~chainId, ~onHeight),
+    createHeightSubscription: (~onHeight, ~onStatus) =>
+      HyperSyncSSE.subscribe(~hyperSyncUrl=endpointUrl, ~apiToken, ~onHeight, ~onStatus),
   }
 }

package/src/sources/EvmHyperSyncSource.res.mjs

@@ -4,6 +4,7 @@ import * as Source from "./Source.res.mjs";
 import * as Logging from "../Logging.res.mjs";
 import * as HyperSync from "./HyperSync.res.mjs";
 import * as Performance from "../bindings/Performance.res.mjs";
+import * as HyperSyncSSE from "./HyperSyncSSE.res.mjs";
 import * as Stdlib_JsExn from "@rescript/runtime/lib/es6/Stdlib_JsExn.js";
 import * as ErrorHandling from "../ErrorHandling.res.mjs";
 import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js";
@@ -11,7 +12,6 @@ import * as Stdlib_JsError from "@rescript/runtime/lib/es6/Stdlib_JsError.js";
 import * as HyperSyncClient from "./HyperSyncClient.res.mjs";
 import * as Primitive_option from "@rescript/runtime/lib/es6/Primitive_option.js";
 import * as Primitive_exceptions from "@rescript/runtime/lib/es6/Primitive_exceptions.js";
-import * as HyperSyncHeightStream from "./HyperSyncHeightStream.res.mjs";
 
 function isUnauthorizedError(message) {
   return message.includes("401 Unauthorized");
@@ -143,7 +143,7 @@ Learn more or get a free Envio API token at: https://envio.dev/app/api-tokens`);
     sourceFor: "Sync",
     chainId: chainId,
     poweredByHyperSync: true,
-    pollingInterval: 100,
+    pollingInterval: HyperSync.pollingInterval,
     getBlockHashes: getBlockHashes,
     getHeightOrThrow: async () => {
       let timerRef = Performance.now();
@@ -188,7 +188,7 @@ Learn more or get a free Envio API token at: https://envio.dev/app/api-tokens`);
       };
     },
     getItemsOrThrow: getItemsOrThrow,
-    createHeightSubscription: onHeight => HyperSyncHeightStream.subscribe(endpointUrl, apiToken$1, chainId, onHeight)
+    createHeightSubscription: (onHeight, onStatus) => HyperSyncSSE.subscribe(endpointUrl, apiToken$1, onHeight, onStatus)
   };
 }
 

package/src/sources/EvmRpcWs.res

@@ -0,0 +1,124 @@
+// newHeads carries no keep-alive, so this has to tolerate slow chains rather
+// than the seconds an SSE ping allows.
+let staleTimeout = 60_000
+
+// The id of the one request this stream ever sends, which is what tells a
+// response meant for it apart from one about anything else sharing the socket.
+let subscribeRequestId = 1
+let subscribeRequestIdText = subscribeRequestId->Int.toString
+
+type wsMessage =
+  | NewHead(int)
+  // A response to some request, carrying the id it answers when it names one.
+  | Response({id: option<string>, isError: bool})
+
+let subscribeRequestJson =
+  {"jsonrpc": "2.0", "id": subscribeRequestId, "method": "eth_subscribe", "params": ["newHeads"]}
+  ->(Utils.magic: {
+    "jsonrpc": string,
+    "id": int,
+    "method": string,
+    "params": array<string>,
+  } => JSON.t)
+  ->JSON.stringify
+
+// A JSON-RPC id is any scalar, and gateways do echo an int id back as a string.
+// Both spellings name the same request, so both are read as the one text.
+// Nullable, not merely optional: JSON-RPC answers a request whose id it could
+// not read with an explicit null.
+let responseIdSchema = S.nullable(
+  S.union([S.string, S.int->S.transform(_ => {parser: id => id->Int.toString})]),
+)
+
+// The subscribe is the only request this socket ever sends, so an error naming
+// its id is that subscription being refused. So is one naming no id at all —
+// which is how JSON-RPC answers a request it could not parse — but only until
+// the node has confirmed the subscription: after that the request has had its
+// answer, and a null id belongs to whatever else the provider wants to say,
+// throttling most of all. Tearing a delivering subscription down over that
+// would reconnect against an endpoint already asking for less.
+let refusesSubscribe = (id, ~subscribed) =>
+  switch id {
+  | None => !subscribed
+  | Some(id) => id === subscribeRequestIdText
+  }
+
+let wsMessageSchema = S.union([
+  S.object(s => {
+    let _ = s.field("method", S.literal("eth_subscription"))
+    NewHead(
+      s.field(
+        "params",
+        S.object(s => {
+          s.field(
+            "result",
+            S.object(
+              s => {
+                s.field("number", Rpc.hexIntSchema)
+              },
+            ),
+          )
+        }),
+      ),
+    )
+  }),
+  S.object(s => {
+    let id = s.field("id", responseIdSchema)
+    let _ = s.field("result", S.string)
+    Response({id, isError: false})
+  }),
+  S.object(s => {
+    let id = s.field("id", responseIdSchema)
+    // The error member as JSON-RPC defines it, rather than `S.unknown`: a
+    // missing field parses as `undefined`, which `S.unknown` accepts, and this
+    // arm would then match every frame the ones above turned down — including
+    // one nobody can read.
+    let _ = s.field("error", S.object(s => s.field("code", S.int)))
+    Response({id, isError: true})
+  }),
+])
+
+let subscribe = (~wsUrl, ~onHeight, ~onStatus) =>
+  HeightStream.subscribe(~staleTimeout, ~onHeight, ~onStatus, ~connect=driver => {
+    let ws = WebSocket.create(wsUrl)
+    let subscribed = ref(false)
+
+    ws->WebSocket.onopen(() => ws->WebSocket.send(subscribeRequestJson))
+
+    ws->WebSocket.onmessage(event =>
+      if event.data->typeof === #string {
+        let text = event.data->(Utils.magic: unknown => string)
+        let message = Utils.Option.catchToNone(
+          () => text->JSON.parseOrThrow->S.parseOrThrow(wsMessageSchema),
+        )
+        switch message {
+        | Some(NewHead(blockNumber)) => driver.onHeight(blockNumber)
+        // An open socket isn't usable until the node accepts the subscription,
+        // and only the answer to the subscribe says that it has. A success frame
+        // answering anything else would otherwise report a stream as live
+        // without one ever having been subscribed.
+        | Some(Response({id: Some(id), isError: false})) if id === subscribeRequestIdText =>
+          subscribed := true
+          driver.onConnected()
+        | Some(Response({id, isError: true})) if id->refusesSubscribe(~subscribed=subscribed.contents) =>
+          driver.onFailure(~reason=SubscribeRejected, ~detail=text)
+        // A provider talking about something other than the subscription this
+        // socket asked for — a rate limit, a request nobody here made. Read
+        // fine, so not unreadable, and not this stream's to tear itself down
+        // over.
+        | Some(Response(_)) => ()
+        | None => driver.onUnreadable(~detail=text)
+        }
+      } else {
+        // A binary frame from a peer that should only ever be sending JSON text.
+        driver.onUnreadable(~detail="binary frame")
+      }
+    )
+
+    ws->WebSocket.onerror(error =>
+      driver.onFailure(~reason=TransportError, ~detail=?error->JsExn.message)
+    )
+    ws->WebSocket.onclose(() => driver.onFailure(~reason=Closed))
+
+    () => ws->WebSocket.close
+  })