envio 3.9.0 → 3.10.0

package/src/sources/HeightFeed.res.mjs

@@ -0,0 +1,452 @@
+// Generated by ReScript, PLEASE EDIT WITH CARE
+
+import * as Utils from "../Utils.res.mjs";
+import * as Source from "./Source.res.mjs";
+import * as Logging from "../Logging.res.mjs";
+import * as Stdlib_Array from "@rescript/runtime/lib/es6/Stdlib_Array.js";
+import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js";
+import * as Stdlib_Promise from "@rescript/runtime/lib/es6/Stdlib_Promise.js";
+import * as Primitive_option from "@rescript/runtime/lib/es6/Primitive_option.js";
+import * as Primitive_string from "@rescript/runtime/lib/es6/Primitive_string.js";
+import * as Primitive_exceptions from "@rescript/runtime/lib/es6/Primitive_exceptions.js";
+
+function make(source, recordRequestStats, getHeightRetryInterval) {
+  return {
+    source: source,
+    logger: Logging.createChild({
+      chainId: source.chainId,
+      source: source.name
+    }),
+    recordRequestStats: recordRequestStats,
+    getHeightRetryInterval: getHeightRetryInterval,
+    knownHeight: 0,
+    waiters: [],
+    stream: "NeverEnabled",
+    generation: 0,
+    closeStream: undefined,
+    polling: false,
+    sleeping: undefined,
+    pollRetry: 0,
+    connects: 0,
+    disconnects: {},
+    stopped: false,
+    unreadableWarned: false,
+    subscribeRejectedWarned: false
+  };
+}
+
+function knownHeight(feed) {
+  return feed.knownHeight;
+}
+
+function sample(feed) {
+  let match = feed.stream;
+  if (typeof match !== "object" && match === "NeverEnabled") {
+    return;
+  }
+  return {
+    connectCount: feed.connects,
+    disconnectsByReason: Object.entries(feed.disconnects).toSorted((param, param$1) => Primitive_string.compare(param[0], param$1[0]))
+  };
+}
+
+function shouldPoll(feed) {
+  if (feed.waiters.length === 0) {
+    return false;
+  }
+  let match = feed.stream;
+  if (typeof match !== "object" || !match.proven) {
+    return true;
+  } else {
+    return feed.waiters.some(waiter => waiter.distrustsStream);
+  }
+}
+
+function wakeCadence(feed) {
+  let match = feed.sleeping;
+  if (match === undefined) {
+    return;
+  }
+  let match$1 = match.kind;
+  if (match$1 === "Cadence") {
+    return match.wake();
+  }
+}
+
+function clearBackoff(feed) {
+  feed.pollRetry = 0;
+  let sleeping = feed.sleeping;
+  if (sleeping !== undefined) {
+    sleeping.kind = "Cadence";
+    return;
+  }
+}
+
+function fireWaiters(feed, height) {
+  let isSatisfied = waiter => height > waiter.knownHeight;
+  let satisfied = feed.waiters.filter(isSatisfied);
+  if (satisfied.length !== 0) {
+    feed.waiters = feed.waiters.filter(waiter => height <= waiter.knownHeight);
+    satisfied.forEach(waiter => {
+      try {
+        return waiter.onHeight(height);
+      } catch (raw_exn) {
+        let exn = Primitive_exceptions.internalToException(raw_exn);
+        return Logging.childError(feed.logger, {
+          msg: "A height waiter threw. Dropping it rather than losing the height for the others.",
+          err: Utils.prettifyExn(exn)
+        });
+      }
+    });
+    return wakeCadence(feed);
+  }
+}
+
+function trustStreamAgain(feed) {
+  feed.waiters.forEach(waiter => {
+    waiter.distrustsStream = false;
+  });
+}
+
+function recordHeight(feed, height) {
+  if (height > feed.knownHeight) {
+    feed.knownHeight = height;
+    return fireWaiters(feed, height);
+  }
+}
+
+function recordAnswer(feed, generation, res) {
+  feed.recordRequestStats(res.requestStats);
+  clearBackoff(feed);
+  let match = feed.stream;
+  if (typeof match !== "object") {
+    match === "NeverEnabled";
+  } else if (generation === feed.generation) {
+    feed.stream = {
+      TAG: "Connected",
+      proven: true
+    };
+  }
+  recordHeight(feed, res.height);
+  wakeCadence(feed);
+}
+
+function recordLateAnswer(feed, res) {
+  feed.recordRequestStats(res.requestStats);
+  recordHeight(feed, res.height);
+}
+
+function heightWithin(feed) {
+  let timeoutId = {
+    contents: undefined
+  };
+  let timedOut = {
+    contents: false
+  };
+  let request = feed.source.getHeightOrThrow();
+  Stdlib_Promise.$$catch(request.then(res => {
+    if (timedOut.contents) {
+      return recordLateAnswer(feed, res);
+    }
+  }), param => Promise.resolve());
+  return Promise.race([
+    Stdlib_Promise.$$catch(request.then(res => ({
+      TAG: "Answered",
+      _0: res
+    })), exn => Promise.resolve({
+      TAG: "Failed",
+      _0: exn
+    })),
+    new Promise((resolve, _reject) => {
+      timeoutId.contents = Primitive_option.some(setTimeout(() => {
+        timedOut.contents = true;
+        resolve("TimedOut");
+      }, 60000));
+    })
+  ]).then(outcome => {
+    Utils.clearTimeoutRef(timeoutId);
+    return outcome;
+  });
+}
+
+function sleep(feed, kind, millis) {
+  return new Promise((resolve, _reject) => {
+    let timeoutId = setTimeout(() => {
+      feed.sleeping = undefined;
+      resolve();
+    }, millis);
+    feed.sleeping = {
+      kind: kind,
+      wake: () => {
+        clearTimeout(timeoutId);
+        feed.sleeping = undefined;
+        resolve();
+      }
+    };
+  });
+}
+
+function currentInterval(feed) {
+  return Stdlib_Option.getOr(Stdlib_Array.reduce(feed.waiters, undefined, (shortest, waiter) => Utils.$$Math.minOptInt(shortest, waiter.interval())), feed.source.pollingInterval);
+}
+
+function nextRetryInterval(feed) {
+  let retryInterval = feed.getHeightRetryInterval(feed.pollRetry);
+  feed.pollRetry = feed.pollRetry + 1 | 0;
+  return retryInterval;
+}
+
+function backOff(feed, what, exn) {
+  let retryInterval = nextRetryInterval(feed);
+  Logging.childTrace(feed.logger, {
+    msg: `Height retrieval from ` + feed.source.name + ` source ` + what + `. Retrying in ` + retryInterval.toString() + `ms.`,
+    err: Stdlib_Option.map(exn, Utils.prettifyExn)
+  });
+  return [
+    retryInterval,
+    "Backoff"
+  ];
+}
+
+async function pollOnce(feed) {
+  let generation = feed.generation;
+  let res = await heightWithin(feed);
+  if (typeof res !== "object") {
+    return backOff(feed, `did not answer within ` + (60).toString() + `s`, undefined);
+  }
+  if (res.TAG !== "Answered") {
+    return backOff(feed, "failed", res._0);
+  }
+  recordAnswer(feed, generation, res._0);
+  return [
+    currentInterval(feed),
+    "Cadence"
+  ];
+}
+
+async function runPollLoop(feed) {
+  while (shouldPoll(feed)) {
+    let match;
+    try {
+      match = await pollOnce(feed);
+    } catch (raw_exn) {
+      let exn = Primitive_exceptions.internalToException(raw_exn);
+      Logging.childError(feed.logger, {
+        msg: "The height poll cycle threw. Backing off and carrying on rather than leaving the source uncovered.",
+        err: Utils.prettifyExn(exn)
+      });
+      match = [
+        nextRetryInterval(feed),
+        "Backoff"
+      ];
+    }
+    if (shouldPoll(feed)) {
+      await sleep(feed, match[1], match[0]);
+    }
+  };
+  feed.polling = false;
+}
+
+function syncPolling(feed) {
+  if (shouldPoll(feed) && !feed.polling) {
+    feed.polling = true;
+    runPollLoop(feed);
+    return;
+  } else {
+    return wakeCadence(feed);
+  }
+}
+
+async function catchUpWhileIdle(feed) {
+  let generation = feed.generation;
+  let res = await heightWithin(feed);
+  if (typeof res !== "object" || res.TAG !== "Answered") {
+    return;
+  } else {
+    return recordAnswer(feed, generation, res._0);
+  }
+}
+
+function handlePushedHeight(feed, height) {
+  let advances = height > feed.knownHeight;
+  feed.recordRequestStats([{
+      method: advances ? "heightPush" : "heightPushIgnored",
+      seconds: 0
+    }]);
+  clearBackoff(feed);
+  let match = feed.stream;
+  if (typeof match !== "object") {
+    match === "NeverEnabled";
+  } else if (advances) {
+    feed.stream = {
+      TAG: "Connected",
+      proven: true
+    };
+  }
+  trustStreamAgain(feed);
+  recordHeight(feed, height);
+  wakeCadence(feed);
+}
+
+function markStreamDown(feed, reason) {
+  let match = feed.stream;
+  let hadConnection;
+  hadConnection = typeof match === "object";
+  feed.stream = "Disconnected";
+  if (hadConnection) {
+    Utils.Dict.incrementBy(feed.disconnects, Source.downReasonLabel(reason), 1);
+    trustStreamAgain(feed);
+    return syncPolling(feed);
+  }
+}
+
+function handleStatus(feed, status) {
+  if (typeof status !== "object") {
+    let match = feed.stream;
+    if (typeof match === "object") {
+      return;
+    }
+    match === "NeverEnabled";
+    feed.stream = {
+      TAG: "Connected",
+      proven: false
+    };
+    feed.generation = feed.generation + 1 | 0;
+    feed.connects = feed.connects + 1 | 0;
+    syncPolling(feed);
+    if (!feed.polling) {
+      catchUpWhileIdle(feed);
+      return;
+    } else {
+      return;
+    }
+  }
+  let reason = status.reason;
+  markStreamDown(feed, reason);
+  let log;
+  if (typeof reason !== "object") {
+    switch (reason) {
+      case "Unreadable" :
+        if (feed.unreadableWarned) {
+          log = Logging.childTrace;
+        } else {
+          feed.unreadableWarned = true;
+          log = Logging.childWarn;
+        }
+        break;
+      case "SubscribeRejected" :
+        if (!feed.subscribeRejectedWarned && feed.connects === 0) {
+          feed.subscribeRejectedWarned = true;
+          log = Logging.childWarn;
+        } else {
+          log = Logging.childTrace;
+        }
+        break;
+      default:
+        log = Logging.childTrace;
+    }
+  } else {
+    log = Logging.childTrace;
+  }
+  let label = Source.downReasonLabel(reason);
+  log(feed.logger, {
+    msg: `Height subscription for ` + feed.source.name + ` source went down (` + label + `). Polling for the height until it reconnects.`,
+    reason: label,
+    detail: status.detail
+  });
+}
+
+function enableStream(feed) {
+  let match = feed.source.createHeightSubscription;
+  let match$1 = feed.closeStream;
+  if (match === undefined) {
+    return;
+  }
+  if (match$1 !== undefined) {
+    return;
+  }
+  if (feed.stopped) {
+    return;
+  }
+  let match$2 = feed.stream;
+  if (typeof match$2 !== "object" && match$2 === "NeverEnabled") {
+    feed.stream = "Disconnected";
+  }
+  feed.closeStream = match(height => handlePushedHeight(feed, height), status => handleStatus(feed, status));
+}
+
+function stop(feed) {
+  feed.stopped = true;
+  let closeStream = feed.closeStream;
+  if (closeStream !== undefined) {
+    closeStream();
+    feed.closeStream = undefined;
+    return markStreamDown(feed, "Unsubscribed");
+  }
+}
+
+function onHeightAbove(feed, knownHeight, interval, onHeight) {
+  if (feed.knownHeight > knownHeight) {
+    onHeight(feed.knownHeight);
+    return {
+      unsubscribe: () => {},
+      distrustStream: () => {}
+    };
+  }
+  let waiter = {
+    knownHeight: knownHeight,
+    onHeight: onHeight,
+    interval: interval,
+    distrustsStream: false
+  };
+  feed.waiters.push(waiter);
+  syncPolling(feed);
+  return {
+    unsubscribe: () => {
+      feed.waiters = feed.waiters.filter(w => w !== waiter);
+      wakeCadence(feed);
+    },
+    distrustStream: () => {
+      let match = feed.stream;
+      if (typeof match !== "object" || waiter.distrustsStream) {
+        return;
+      } else {
+        waiter.distrustsStream = true;
+        return syncPolling(feed);
+      }
+    }
+  };
+}
+
+let pollTimeoutMillis = 60000;
+
+export {
+  make,
+  knownHeight,
+  sample,
+  shouldPoll,
+  wakeCadence,
+  clearBackoff,
+  fireWaiters,
+  trustStreamAgain,
+  recordHeight,
+  recordAnswer,
+  recordLateAnswer,
+  pollTimeoutMillis,
+  heightWithin,
+  sleep,
+  currentInterval,
+  nextRetryInterval,
+  backOff,
+  pollOnce,
+  runPollLoop,
+  syncPolling,
+  catchUpWhileIdle,
+  handlePushedHeight,
+  markStreamDown,
+  handleStatus,
+  enableStream,
+  stop,
+  onHeightAbove,
+}
+/* Utils Not a pure module */

package/src/sources/HeightStream.res

@@ -0,0 +1,257 @@
+/*
+Reconnect driver shared by every height subscription transport. A transport
+supplies `connect`, which wires its socket to the driver callbacks and returns a
+close function; the driver owns retries, staleness detection and status
+reporting so SSE and WebSocket streams behave identically.
+
+Nothing here logs. A height stream failing is not something an operator has to
+act on — the indexer polls for the height instead — so a failure travels out
+through `onStatus`, where its reason becomes a metric label and its detail the
+one log line that says what the provider actually sent.
+*/
+
+type driver = {
+  // The stream is usable: for SSE the connection opened, for WebSocket the
+  // subscription was confirmed.
+  onConnected: unit => unit,
+  onKeepAlive: unit => unit,
+  onHeight: int => unit,
+  // A message the transport couldn't read. Not a failure by itself, but from
+  // then on only a height we can read counts as the connection still being
+  // useful, so a stream of them goes quiet and fails as "unreadable", carrying
+  // the frame that started it.
+  onUnreadable: (~detail: string) => unit,
+  onFailure: (~reason: Source.heightSubscriptionDownReason, ~detail: string=?) => unit,
+}
+
+// While a stream is down its consumer polls instead, so the retry delay is what
+// that fallback costs: a load balancer rotating a connection heals on the first
+// retry, and every polling interval spent waiting for it is an avoidable
+// request. Small first step, with the doubling to protect an endpoint that is
+// genuinely in trouble.
+let baseRetryMillis = 250
+let maxRetryMillis = 60_000
+// Floor for how long a connection has to serve to have been worth making, as a
+// fraction of the window we would wait before calling it dead. Both transports
+// deliver something the moment they connect — HyperSync sends the head, a
+// WebSocket confirms the subscription — so an endpoint that accepts and drops
+// produces everything a working one would have by then, and only staying up
+// tells them apart. A fixed floor can't do it: one low enough to clear on a
+// healthy connection is one a connection that dies seconds in clears too.
+let provenStaleFraction = 4
+// A frame nobody could read ends up in a log line, so cap what a provider can
+// put there.
+let maxDetailLength = 200
+
+// The wait that follows `count` consecutive failures. Zero for a first
+// connection, which nothing preceded. Deterministic: the bar a connection has
+// to clear to count as proven is derived from it, and a jittered bar would make
+// the rotated/closed split depend on a coin flip.
+let retryDelayFor = count =>
+  count <= 0
+    ? 0
+    : Utils.expBackoff(~base=baseRetryMillis, ~exp=count - 1, ~maxMillis=maxRetryMillis)
+
+let truncateDetail = detail =>
+  detail->String.length > maxDetailLength
+    ? detail->String.slice(~start=0, ~end=maxDetailLength) ++ "..."
+    : detail
+
+let subscribe = (
+  ~staleTimeout: int,
+  ~onHeight: int => unit,
+  ~onStatus: Source.heightSubscriptionStatus => unit,
+  // Must not allocate anything it can then throw past: a throw leaves the driver
+  // without the close function, so only the transport can release what it took.
+  ~connect: driver => unit => unit,
+): (unit => unit) => {
+  let closeConnectionRef = ref(None)
+  let failureCount = ref(0)
+  // Bumped on every failure, reconnect and unsubscribe. Callbacks from a
+  // superseded connection compare against it and no-op, so a socket that
+  // reports an error and then a close only counts as one failure, and nothing a
+  // connection reports after unsubscribing is heard.
+  let generation = ref(0)
+  // A connection is either waiting for traffic or waiting to be retried, never
+  // both, so a single slot holds whichever timer is pending.
+  let timeoutId = ref(None)
+  // The frame that stopped this connection being readable, kept for the failure
+  // it will eventually be named after.
+  let unreadableDetail = ref(None)
+  // When the current connection reported itself live, if it ever did. Time
+  // before that is not service: a handshake that takes seconds to fail, or a
+  // socket that is accepted and then hangs until the staleness timer, would
+  // otherwise be indistinguishable from a connection that worked.
+  let liveAt = ref(None)
+  let minProvenMillis = staleTimeout / provenStaleFraction
+
+  let clearPendingTimeout = () => timeoutId->Utils.clearTimeoutRef
+
+  // A transport that throws while closing must not be able to stop the retry
+  // that follows, nor escape a timer callback and take the process down. Every
+  // close goes through here, including the one for a connection superseded
+  // while connect was still running.
+  let closeSafely = close =>
+    try close() catch {
+    | _ => ()
+    }
+
+  let closeConnection = () => {
+    switch closeConnectionRef.contents {
+    | Some(close) => closeSafely(close)
+    | None => ()
+    }
+    closeConnectionRef := None
+  }
+
+  let rec armStaleTimeout = () => {
+    clearPendingTimeout()
+    timeoutId := Some(setTimeout(() => {
+          timeoutId := None
+          // Distinguishes a provider whose message shape we don't understand
+          // from a chain that simply has nothing to report.
+          switch unreadableDetail.contents {
+          | Some(detail) => fail(~reason=Source.Unreadable, ~detail)
+          | None => fail(~reason=Source.Stale)
+          }
+        }, staleTimeout))
+  }
+  and fail = (~reason: Source.heightSubscriptionDownReason, ~detail=?) => {
+    generation := generation.contents + 1
+    clearPendingTimeout()
+    closeConnection()
+
+    // A connection that served for longer than the wait it cost was worth
+    // making, whatever ended it, so the backoff starts over. That covers a load
+    // balancer rotating connections and a chain whose blocks are further apart
+    // than the stale timeout alike, without either having to be recognised. The
+    // bar rises with the backoff, so an endpoint that always dies at the same
+    // age keeps escalating rather than settling into a loop at a fixed delay.
+    let servedMillis = switch liveAt.contents {
+    | Some(since) => (Performance.now() -. since)->Float.toInt
+    | None => 0
+    }
+    // Against the schedule, not against the jittered wait actually served. The
+    // spread exists to keep indexers from retrying in lockstep, and letting it
+    // reach this would put a coin flip in the rotated/closed split an operator
+    // reads. Judging a connection by the step of the ramp it was born on is
+    // conservative in the same direction every time.
+    let provenMillis = Pervasives.max(retryDelayFor(failureCount.contents), minProvenMillis)
+    failureCount := if servedMillis >= provenMillis {
+        1
+      } else {
+        failureCount.contents + 1
+      }
+
+    let retryMillis = retryDelayFor(failureCount.contents)->Utils.jitter
+
+    // A clean end to a connection that had proven itself is a rotation: routine,
+    // and worth telling apart from the same clean end arriving early, which is a
+    // server dropping connections it should be serving. Neither transport can
+    // tell which it was, so the driver decides from how long the connection
+    // lasted — the same measure the backoff already trusts.
+    let reason: Source.heightSubscriptionDownReason = switch reason {
+    | Closed if servedMillis >= provenMillis => Rotated
+    | reason => reason
+    }
+
+    // Scheduled before reporting, so a consumer that throws can't be what makes
+    // the stream give up.
+    timeoutId := Some(setTimeout(() => {
+          timeoutId := None
+          start()
+        }, retryMillis))
+
+    onStatus(Down({reason, detail: ?(detail->Option.map(truncateDetail))}))
+  }
+  and start = () => {
+    generation := generation.contents + 1
+    unreadableDetail := None
+    liveAt := None
+    let connectionGeneration = generation.contents
+    let isCurrent = () => generation.contents === connectionGeneration
+
+    // Armed before connecting, so it covers a connect that never reports
+    // anything, and so a synchronous failure inside connect replaces it with
+    // the retry timeout rather than the other way around.
+    armStaleTimeout()
+
+    // A height is proof the stream is delivering, so it reports Live as well as
+    // onConnected does. Without that, a transport that never reports the
+    // connection usable would leave its consumer polling at full rate next to a
+    // stream that works.
+    let goLive = () => {
+      armStaleTimeout()
+      if liveAt.contents->Option.isNone {
+        liveAt := Some(Performance.now())
+        onStatus(Live)
+      }
+    }
+
+    let driver = {
+      onConnected: () =>
+        if isCurrent() {
+          goLive()
+        },
+      onKeepAlive: () =>
+        // Stops counting once anything unreadable has arrived, on the first one
+        // rather than on some run of them: a stream whose heights are all
+        // malformed is indistinguishable from one that has sent a single
+        // garbled frame until a readable height arrives, and the keep-alives
+        // between malformed heights would otherwise hold the connection open
+        // indefinitely, never failing and so never reporting anything, while
+        // its consumer sat on the staleness backstop. Being wrong about a lone
+        // glitch costs one reconnect.
+        if isCurrent() && unreadableDetail.contents->Option.isNone {
+          armStaleTimeout()
+        },
+      onHeight: height =>
+        if isCurrent() {
+          // A height we could read proves the shape is fine again. A keep-alive
+          // deliberately doesn't: on a stream whose heights are all malformed,
+          // the pings between them would clear this and the staleness that
+          // followed would look like a chain with nothing to report.
+          unreadableDetail := None
+          goLive()
+          onHeight(height)
+        },
+      onUnreadable: (~detail) =>
+        if isCurrent() {
+          unreadableDetail := Some(detail)
+        },
+      onFailure: (~reason, ~detail=?) =>
+        if isCurrent() {
+          fail(~reason, ~detail?)
+        },
+    }
+
+    // A transport constructor can throw on a malformed url. Left to escape it
+    // would reach a timer callback on the next retry and take the process down.
+    // What it says is the only account of why the url can't be connected to, and
+    // this retries forever, so it goes out as the failure's detail.
+    let connected = try Ok(connect(driver)) catch {
+    | exn => Error(exn)
+    }
+
+    switch connected {
+    | Ok(closeCurrentConnection) =>
+      if isCurrent() {
+        closeConnectionRef := Some(closeCurrentConnection)
+      } else {
+        closeSafely(closeCurrentConnection)
+      }
+    | Error(exn) =>
+      if isCurrent() {
+        fail(~reason=ConnectFailed, ~detail=?exn->Utils.exnMessage)
+      }
+    }
+  }
+
+  start()
+
+  () => {
+    generation := generation.contents + 1
+    clearPendingTimeout()
+    closeConnection()
+  }
+}