@effectstream/runtime 0.101.0 → 0.102.0

package/LICENSE-APACHE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright 2024-2026 Midnight Foundation
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

package/LICENSE-MIT

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024-2026 Midnight Foundation
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@effectstream/runtime",
-  "version": "0.101.0",
-  "license": "MIT",
+  "version": "0.102.0",
+  "license": "(MIT OR Apache-2.0)",
   "homepage": "https://effectstream.github.io/docs/",
   "repository": {
     "type": "git",
@@ -13,17 +13,17 @@
   },
   "exports": "./src/mod.ts",
   "dependencies": {
-    "@effectstream/concise": "0.101.0",
-    "@effectstream/config": "0.101.0",
-    "@effectstream/coroutine": "0.101.0",
-    "@effectstream/crypto": "0.101.0",
-    "@effectstream/db": "0.101.0",
-    "@effectstream/event-client": "0.101.0",
-    "@effectstream/event-server": "0.101.0",
-    "@effectstream/log": "0.101.0",
-    "@effectstream/sm": "0.101.0",
-    "@effectstream/sync": "0.101.0",
-    "@effectstream/utils": "0.101.0",
+    "@effectstream/concise": "0.102.0",
+    "@effectstream/config": "0.102.0",
+    "@effectstream/coroutine": "0.102.0",
+    "@effectstream/crypto": "0.102.0",
+    "@effectstream/db": "0.102.0",
+    "@effectstream/event-client": "0.102.0",
+    "@effectstream/event-server": "0.102.0",
+    "@effectstream/log": "0.102.0",
+    "@effectstream/sm": "0.102.0",
+    "@effectstream/sync": "0.102.0",
+    "@effectstream/utils": "0.102.0",
     "@fastify/cors": "^11.0.1",
     "@metamask/json-rpc-engine": "^10.0.3",
     "pg": "8.21.0",

package/src/api/health.ts

@@ -0,0 +1,235 @@
+/**
+ * Sync liveness for `/health`.
+ *
+ * `/health` used to report database reachability only. That misses every
+ * failure this service actually suffers: a blackholed RPC, a producer whose
+ * stream ended, a chain that stopped advancing. All of them end the same way —
+ * the merge blocks on one chain's page and block production stops — while the
+ * database stays perfectly healthy. A node that has not applied a block in an
+ * hour would answer `200 {"status":"ok"}`.
+ *
+ * Two deliberate choices:
+ *
+ * 1. **Liveness is measured in wall-clock time since the last APPLIED block**,
+ *    not in block-timestamp lag. A chain replaying history is legitimately
+ *    hours behind in block time while being perfectly healthy; what matters is
+ *    whether the pipeline is still moving.
+ *
+ * 2. **503 is reserved for "not doing its job"** — database unreachable, or no
+ *    block applied within the threshold. A chain that is erroring or restarting
+ *    its producer while blocks still flow reports `degraded` with a 200, so an
+ *    orchestrator does not kill a node that is recovering on its own.
+ */
+import type { AllSyncProtocols } from "@effectstream/sync";
+import { poolErrors } from "@effectstream/db";
+import { appliedBlockStatus } from "./apply-status.ts";
+import { finalizedStreamStatus } from "./stream-status.ts";
+
+/**
+ * Node-level rollup. `ok` and `degraded` answer 200; `starting`, `stalled` and
+ * `db-unreachable` answer 503 (not ready to serve).
+ */
+export type HealthStatus =
+  | "ok"
+  | "degraded"
+  | "starting"
+  | "stalled"
+  | "db-unreachable";
+
+/** Per-protocol rollup. `blockingMerge` is reported separately. */
+export type ProtocolStatus = "ok" | "starting" | "erroring" | "wedged";
+
+export type ProtocolHealth = {
+  name: string;
+  status: ProtocolStatus;
+  /** Highest block of this chain that has been merged into a produced block. */
+  ownBlockNumber: number | null;
+  /** ms since this chain's fetch loop last completed a pass; null if never. */
+  sinceLastPollMs: number | null;
+  /** ms since `readData` last returned successfully; null if never. */
+  sinceLastSuccessfulFetchMs: number | null;
+  consecutiveErrors: number;
+  /** Streaming producers only: restarts since boot. A climb means flapping. */
+  producerRestarts: number;
+  /** Producer failures, counted independently of `consecutiveErrors`. */
+  producerErrors: number;
+  /** ms since the last producer failure; null if there has never been one. */
+  sinceLastProducerErrorMs: number | null;
+  /**
+   * True when the merge is currently waiting for THIS chain's page to advance.
+   * When the node is stalled, this is the field that says where to look.
+   */
+  blockingMerge: boolean;
+  /** In-memory fetched-not-yet-merged data, and the backpressure cap. */
+  buffered: number;
+  bufferCap: number;
+  paused: boolean;
+};
+
+export type HealthReport = {
+  status: HealthStatus;
+  db: ReturnType<typeof poolErrors.state>;
+  apply: {
+    blockHeight: number | null;
+    /**
+     * ms since the node last applied ANY block — the liveness signal. Never
+     * null: before the first block it measures from process start instead.
+     */
+    sinceLastAppliedMs: number;
+    /** Block-time lag. Informational: large during legitimate catch-up. */
+    lagMs: number | null;
+  };
+  finalizedStream: {
+    produced: number;
+    consumed: number;
+    inFlight: number;
+  };
+  protocols: ProtocolHealth[];
+};
+
+/**
+ * A fetch loop is "wedged" once its heartbeat is this many polling intervals
+ * stale. Generous, because the cost of a false positive (a flapping 503) is
+ * worse than noticing a genuine hang a minute late — and a hung request is now
+ * bounded by `requestTimeoutMs` anyway, so a healthy loop always cycles.
+ */
+const WEDGED_POLL_MULTIPLE = 20;
+/** Floor for the above, so fast-polling chains don't trip on a single hiccup.
+ *
+ * Caveat worth knowing before alerting on `wedged`: one `readData` pass fetches
+ * a whole chunk, which can be many sequential RPCs, so during deep catch-up a
+ * legitimately-progressing chain can exceed this without returning to the top of
+ * the loop and briefly read as `wedged`. Node-level status stays 200 while
+ * blocks keep applying, so this is cosmetic — but alert on the node's `status`,
+ * not on a single protocol's. */
+const WEDGED_MIN_MS = 60_000;
+
+/**
+ * How long after a producer failure the chain still reports `erroring`. Twice
+ * the supervisor's max backoff, so a producer that recovers on its first or
+ * second restart clears the flag on its own.
+ */
+const PRODUCER_FLAP_WINDOW_MS = 60_000;
+
+function protocolHealth(
+  protocol: AllSyncProtocols,
+  now: number,
+): ProtocolHealth {
+  const sinceLastPollMs = protocol.lastPollAtMs > 0
+    ? now - protocol.lastPollAtMs
+    : null;
+  const sinceLastSuccessfulFetchMs = protocol.lastSuccessfulFetchMs > 0
+    ? now - protocol.lastSuccessfulFetchMs
+    : null;
+
+  // `pollingIntervalMs` is published by startSync, so this and the loop agree on
+  // one value. It is 0 until startSync stamps it; fall back until then.
+  const wedgedAfterMs = Math.max(
+    WEDGED_POLL_MULTIPLE * (protocol.pollingIntervalMs || 1_000),
+    WEDGED_MIN_MS,
+  );
+
+  const sinceLastProducerErrorMs = protocol.lastProducerErrorMs > 0
+    ? now - protocol.lastProducerErrorMs
+    : null;
+
+  let status: ProtocolStatus;
+  if (sinceLastPollMs != null && sinceLastPollMs > wedgedAfterMs) {
+    // The loop itself stopped cycling: stuck inside a call that never returned.
+    status = "wedged";
+  } else if (protocol.consecutiveErrors > 0) {
+    status = "erroring";
+  } else if (
+    sinceLastProducerErrorMs != null &&
+    sinceLastProducerErrorMs < PRODUCER_FLAP_WINDOW_MS
+  ) {
+    // A producer that failed recently but has been restarted. Reported so a
+    // flapping stream is visible, and time-bounded so a single old blip does
+    // not pin the chain to `erroring` forever (which is what sharing
+    // `consecutiveErrors` used to do for an idle streaming chain).
+    status = "erroring";
+  } else if (protocol.lastPage == null) {
+    status = "starting";
+  } else {
+    status = "ok";
+  }
+
+  return {
+    name: protocol.name,
+    status,
+    ownBlockNumber: protocol.lastPage?.ownBlockNumber ?? null,
+    sinceLastPollMs,
+    sinceLastSuccessfulFetchMs,
+    consecutiveErrors: protocol.consecutiveErrors,
+    producerRestarts: protocol.producerRestarts,
+    producerErrors: protocol.producerErrors,
+    sinceLastProducerErrorMs,
+    blockingMerge: protocol.mergeWaitingForPage,
+    buffered: protocol.bufferedData.size(),
+    bufferCap: protocol.bufferCap,
+    paused: protocol.pausedNow,
+  };
+}
+
+/**
+ * @param stallThresholdMs How long without applying a block counts as stalled.
+ *   Callers pass the same value used for lag logging and coalescing (20× the
+ *   main clock's block time, or `EFFECTSTREAM_LAG_THRESHOLD_MS`).
+ */
+export function buildHealthReport(
+  syncProtocols: AllSyncProtocols[],
+  stallThresholdMs: number,
+): HealthReport {
+  const now = Date.now();
+  const db = poolErrors.state();
+  const protocols = syncProtocols.map((p) => protocolHealth(p, now));
+
+  const appliedAtMs = appliedBlockStatus.appliedAtMs;
+  // Before the first block, measure from process start instead — a node that
+  // boots and never applies anything is stalled too, and that is exactly what a
+  // chain wedged from its very first request looks like.
+  const sinceLastAppliedMs = appliedAtMs != null
+    ? now - appliedAtMs
+    : Math.round(process.uptime() * 1000);
+  const appliedTs = appliedBlockStatus.timestamp;
+  const neverApplied = appliedAtMs == null;
+
+  let status: HealthStatus;
+  if (db.sustained) {
+    status = "db-unreachable";
+  } else if (neverApplied && sinceLastAppliedMs <= stallThresholdMs) {
+    // Booting: migrations, genesis sync, first merge. Not ready to serve, but
+    // distinct from `stalled` so an operator can tell "still coming up" from
+    // "was up and stopped".
+    status = "starting";
+  } else if (sinceLastAppliedMs > stallThresholdMs) {
+    status = "stalled";
+  } else if (protocols.some((p) => p.status !== "ok" && p.status !== "starting")) {
+    // Something is unhealthy but blocks are still flowing: worth alerting on,
+    // not worth restarting the node over.
+    status = "degraded";
+  } else {
+    status = "ok";
+  }
+
+  return {
+    status,
+    db,
+    apply: {
+      blockHeight: appliedBlockStatus.blockNumber,
+      sinceLastAppliedMs,
+      lagMs: appliedTs != null ? now - appliedTs : null,
+    },
+    finalizedStream: {
+      produced: finalizedStreamStatus.produced,
+      consumed: finalizedStreamStatus.consumed,
+      inFlight: finalizedStreamStatus.produced - finalizedStreamStatus.consumed,
+    },
+    protocols,
+  };
+}
+
+/** 503 only when the node is not (yet) doing its job; see the note at the top. */
+export function healthHttpStatus(status: HealthStatus): 200 | 503 {
+  return status === "ok" || status === "degraded" ? 200 : 503;
+}

package/src/api/http-server.ts

@@ -2,6 +2,7 @@ import fastify, { type FastifyInstance, type FastifyReply, type FastifyRequest }
 import { evmRpcEngine } from "./rpc-evm/eip1193.ts";
 import { appliedBlockStatus } from "./apply-status.ts";
 import { finalizedStreamStatus } from "./stream-status.ts";
+import { buildHealthReport, healthHttpStatus } from "./health.ts";
 import type { Pool } from "pg";
 import cors from "@fastify/cors";
 import { run, until } from "effection";
@@ -159,6 +160,8 @@ function* registerOpenApiDocumentation(
 export const startHttpServer = function* (
   dbConn: Pool,
   syncProtocols: AllSyncProtocols[],
+  /** How long without applying a block counts as stalled on `/health`. */
+  stallThresholdMs: number,
   apiRouter?: StartConfigApiRouter,
   grammar?: GrammarDefinition,
 ) {
@@ -373,9 +376,35 @@ export const startHttpServer = function* (
     sustainedDurationMs: Type.Number(),
     sustained: Type.Boolean(),
   });
+  const HealthProtocolSchema = Type.Object({
+    name: Type.String(),
+    status: Type.String(),
+    ownBlockNumber: Type.Union([Type.Number(), Type.Null()]),
+    sinceLastPollMs: Type.Union([Type.Number(), Type.Null()]),
+    sinceLastSuccessfulFetchMs: Type.Union([Type.Number(), Type.Null()]),
+    consecutiveErrors: Type.Number(),
+    producerRestarts: Type.Number(),
+    producerErrors: Type.Number(),
+    sinceLastProducerErrorMs: Type.Union([Type.Number(), Type.Null()]),
+    blockingMerge: Type.Boolean(),
+    buffered: Type.Number(),
+    bufferCap: Type.Number(),
+    paused: Type.Boolean(),
+  });
   const HealthResponseSchema = Type.Object({
     status: Type.String(),
     db: HealthDbSchema,
+    apply: Type.Object({
+      blockHeight: Type.Union([Type.Number(), Type.Null()]),
+      sinceLastAppliedMs: Type.Number(),
+      lagMs: Type.Union([Type.Number(), Type.Null()]),
+    }),
+    finalizedStream: Type.Object({
+      produced: Type.Number(),
+      consumed: Type.Number(),
+      inFlight: Type.Number(),
+    }),
+    protocols: Type.Array(HealthProtocolSchema),
   });
   server.get("/health", {
     schema: {
@@ -386,10 +415,10 @@ export const startHttpServer = function* (
       },
     },
   }, (_req, reply) => {
-    const db = poolErrors.state();
-    return reply
-      .status(db.sustained ? 503 : 200)
-      .send({ status: db.sustained ? "db-unreachable" : "ok", db });
+    // Reports sync liveness, not just database reachability — a stalled merge
+    // leaves the database perfectly healthy. See ./health.ts.
+    const report = buildHealthReport(syncProtocols, stallThresholdMs);
+    return reply.status(healthHttpStatus(report.status)).send(report);
   });
 
   server.get("/addresses", {

package/src/config-snapshot.ts

@@ -20,7 +20,7 @@ import { getEnv } from "@effectstream/utils/runtime";
  *   Cardano CARP               → syncProtocol.startSlot
  *   Cardano UTXOrpc            → syncProtocol.startChainPoint
  *   EVM, Mina, Avail, Midnight,
- *   Bitcoin, Celestia          → syncProtocol.startBlockHeight
+ *   Bitcoin, Celestia, NEAR, Solana → syncProtocol.startBlockHeight
  */
 function extractImmutableConfig(protocol: SyncProtocolWithNetwork): Record<string, unknown> {
   // NTP and the synthetic TEST chain both derive blocks arithmetically from
@@ -49,7 +49,7 @@ function extractImmutableConfig(protocol: SyncProtocolWithNetwork): Record<strin
     return { startChainPoint: sp.startChainPoint };
   }
 
-  // EVM, Mina, Avail, Midnight, Bitcoin, Celestia
+  // EVM, Mina, Avail, Midnight, Bitcoin, Celestia, NEAR, Solana
   const knownBlockHeightProtocols: string[] = [
     ConfigSyncProtocolType.EVM_RPC_PARALLEL,
     ConfigSyncProtocolType.MINA_PARALLEL,
@@ -58,6 +58,7 @@ function extractImmutableConfig(protocol: SyncProtocolWithNetwork): Record<strin
     ConfigSyncProtocolType.BITCOIN_RPC_PARALLEL,
     ConfigSyncProtocolType.CELESTIA_PARALLEL,
     ConfigSyncProtocolType.NEAR_RPC_PARALLEL,
+    ConfigSyncProtocolType.SOLANA_RPC_PARALLEL,
   ];
   if (!knownBlockHeightProtocols.includes(protocol.syncProtocolType)) {
     throw new Error(

package/src/main.ts

@@ -90,18 +90,11 @@ export function* start(config: StartConfig): Operation<void> {
     new EventBroker("effectstream-engine").createServer();
   }
 
-  yield* spawn(function* () {
-    yield* startHttpServer(
-      dbConn,
-      syncProtocols,
-      config.apiRouter,
-      config.grammar,
-    );
-  });
-
   // 20× main clock block time (NTP if present, else protocol 0).
   // Falls back to 60 s so coalescing is not silently disabled for chains that
   // don't expose a blockTimeMS on their network config (e.g. Midnight-as-main).
+  // Computed before the HTTP server starts because /health uses the same
+  // threshold to decide whether the node counts as stalled.
   const ntpConfig = syncInfo.find(s => s.networkType === ConfigNetworkType.NTP);
   const clockBlockTimeMS =
     (ntpConfig?.network as { blockTimeMS?: number } | undefined)?.blockTimeMS ??
@@ -109,6 +102,16 @@ export function* start(config: StartConfig): Operation<void> {
   const lagThresholdMs = ENV.EFFECTSTREAM_LAG_THRESHOLD_MS ??
     (clockBlockTimeMS != null ? clockBlockTimeMS * 20 : 60_000);
 
+  yield* spawn(function* () {
+    yield* startHttpServer(
+      dbConn,
+      syncProtocols,
+      lagThresholdMs,
+      config.apiRouter,
+      config.grammar,
+    );
+  });
+
   // Bounded hand-off queue between the merge and the apply loop. Backpressure caps
   // the in-memory queue so deep catch-up can't grow it toward the whole backlog
   // (see finalized-stream.ts / sync/CLAUDE.md Finding #1).

package/test/reproduction/determinism.test.ts

@@ -0,0 +1,207 @@
+/**
+ * Timing must not change state.
+ *
+ * The whole pipeline rests on one property: two nodes syncing the same chains
+ * reach byte-identical state, regardless of how fast they got there. Block `n`
+ * always holds the same data on every machine, so the block-hash chain — and
+ * everything downstream of it — agrees.
+ *
+ * That property is easy to break by accident, and three changes in this branch
+ * touch the fetch path: per-request timeouts, unconditional loop pacing, and
+ * producer supervision. All three alter *when* a fetch happens, so this test
+ * varies exactly those knobs and asserts the resulting databases are identical.
+ *
+ * Why it holds, and what would break it:
+ *
+ *   - Pacing changes only how often the loop asks. A block's content is a pure
+ *     function of its height (the main clock derives timestamps arithmetically;
+ *     parallel data merges into the first root block whose timestamp is >= its
+ *     own), so polling faster only reaches the same answer sooner.
+ *
+ *   - A timeout makes `readData` throw, and `lastPage` only advances on a
+ *     COMPLETE `DataFetched` — so a timeout re-fetches the same range rather
+ *     than skipping it. A slower machine that times out where a faster one does
+ *     not therefore converges on the same data. It would NOT be safe if a
+ *     partial result were ever committed, which is why the fetchers build their
+ *     whole page-range before returning.
+ *
+ * The synthetic chain is deterministic by construction, so any difference here
+ * comes from the engine rather than from the chain.
+ */
+import { afterEach, beforeEach, expect, test } from "bun:test";
+import {
+  ConfigBuilder,
+  ConfigNetworkType,
+  ConfigSyncProtocolType,
+} from "@effectstream/config";
+import { TestChainControl } from "@effectstream/sync";
+import {
+  compareConsistencySnapshots,
+  type ConsistencySnapshot,
+  dumpConsistencySnapshot,
+  SYNC_BOOKKEEPING_TABLES,
+} from "./consistency-snapshot.ts";
+import { type Harness, setupHarness, TEST_PRIMITIVE_TYPE } from "./harness.ts";
+
+const START_TIME = 1_700_000_000_000;
+const BLOCK_TIME_MS = 1000;
+const TARGET = 60;
+/**
+ * The parallel chain is deliberately fetched further than the main clock. The
+ * merge only folds a parallel chain's data into root timestamp T once that
+ * chain's page is STRICTLY past T, so a parallel tip equal to the main tip
+ * stalls production one block short of the target (see the merge-boundary note
+ * in sync/CLAUDE.md).
+ */
+const PARALLEL_TIP = TARGET + 40;
+
+const MAIN = "detMain";
+const PAR = "detParallel";
+
+/** Events spread across the range so several blocks carry state. */
+const EVENTS = [
+  { atBlock: 12, payload: { v: "a" } },
+  { atBlock: 25, payload: { v: "b" } },
+  { atBlock: 44, payload: { v: "c" } },
+];
+
+type Timing = {
+  pollingInterval: number;
+  stepSize: number;
+  requestTimeoutMs: number;
+};
+
+function buildConfig(timing: Timing) {
+  return new ConfigBuilder()
+    .setNamespace((b) => b.setSecurityNamespace("sync-determinism"))
+    .buildNetworks((b) =>
+      b
+        .addNetwork({
+          name: "clock",
+          type: ConfigNetworkType.TEST,
+          startTime: START_TIME,
+          blockTimeMS: BLOCK_TIME_MS,
+        })
+        .addNetwork({
+          name: "chainP",
+          type: ConfigNetworkType.TEST,
+          startTime: START_TIME,
+          blockTimeMS: BLOCK_TIME_MS,
+        })
+    )
+    .buildDeployments((b) => b)
+    .buildSyncProtocols((b) =>
+      b
+        .addMain(
+          (n) => n.clock,
+          () => ({
+            name: MAIN,
+            type: ConfigSyncProtocolType.TEST_MAIN,
+            startBlockHeight: 1,
+            pollingInterval: timing.pollingInterval,
+            stepSize: timing.stepSize,
+            requestTimeoutMs: timing.requestTimeoutMs,
+          }),
+        )
+        .addParallel(
+          (n) => (n as any).chainP,
+          () => ({
+            name: PAR,
+            type: ConfigSyncProtocolType.TEST_PARALLEL,
+            startBlockHeight: 1,
+            pollingInterval: timing.pollingInterval,
+            stepSize: timing.stepSize,
+            requestTimeoutMs: timing.requestTimeoutMs,
+            delayMs: 0,
+            events: EVENTS,
+          }),
+        )
+    )
+    .buildPrimitives((b) =>
+      b.addPrimitive(
+        (sp) => (sp as any)[PAR],
+        () => ({ name: "detEvt", type: TEST_PRIMITIVE_TYPE, startBlockHeight: 1 }),
+      )
+    )
+    .build();
+}
+
+/** Sync a fresh database to TARGET under the given timing, then fingerprint it. */
+async function syncUnder(
+  timing: Timing,
+  apiPort: number,
+): Promise<ConsistencySnapshot> {
+  const h: Harness = await setupHarness();
+  try {
+    TestChainControl.clear();
+    TestChainControl.setTip(MAIN, TARGET);
+    TestChainControl.setTip(PAR, PARALLEL_TIP);
+
+    await h.runToHeight({
+      events: [],
+      // Keyed by THIS config's protocol names; a mismatch would leave the
+      // synthetic chain on its wall-clock fallback tip and sync to "now".
+      tips: { [MAIN]: TARGET, [PAR]: PARALLEL_TIP },
+      target: TARGET,
+      apiPort,
+      config: buildConfig(timing),
+    });
+
+    // Bookkeeping tables legitimately differ by path (the page queue records
+    // where a run happened to be), so compare observable state.
+    return await dumpConsistencySnapshot(h.pool, {
+      excludeTables: SYNC_BOOKKEEPING_TABLES,
+    });
+  } finally {
+    await h.teardown();
+  }
+}
+
+beforeEach(() => {
+  TestChainControl.clear();
+});
+afterEach(() => {
+  TestChainControl.clear();
+});
+
+test("state is identical regardless of polling speed, chunk size and timeouts", async () => {
+  // Fast + fine-grained: many small fetches, tight deadline.
+  const fast = await syncUnder(
+    { pollingInterval: 5, stepSize: 5, requestTimeoutMs: 1_000 },
+    19201,
+  );
+
+  // Slow + coarse: fewer, larger fetches, generous deadline. A different
+  // machine under different load is somewhere between these two.
+  const slow = await syncUnder(
+    { pollingInterval: 60, stepSize: 100, requestTimeoutMs: 30_000 },
+    19202,
+  );
+
+  const { diffs } = compareConsistencySnapshots(fast, slow);
+  expect(diffs).toEqual([]);
+
+  // Guard against the assertion passing vacuously on two empty databases.
+  const rows = [...fast.values()].reduce((sum, f) => sum + f.rowCount, 0);
+  expect(rows).toBeGreaterThan(0);
+}, 180_000);
+
+test("the block hash chain is identical across timings", async () => {
+  const a = await syncUnder(
+    { pollingInterval: 5, stepSize: 5, requestTimeoutMs: 1_000 },
+    19203,
+  );
+  const b = await syncUnder(
+    { pollingInterval: 60, stepSize: 100, requestTimeoutMs: 30_000 },
+    19204,
+  );
+
+  // effectstream_blocks carries block_height + effectstream_block_hash, so an
+  // identical fingerprint here means every block hashed the same on both runs —
+  // the property consensus actually depends on. Called out separately from the
+  // whole-database check so a failure points straight at the hash chain.
+  const key = "effectstream.effectstream_blocks";
+  expect(a.get(key)).toBeDefined();
+  expect(a.get(key)!.rowCount).toBeGreaterThan(0);
+  expect(b.get(key)?.fingerprint).toBe(a.get(key)!.fingerprint);
+}, 180_000);

package/test/reproduction/harness.ts

@@ -317,6 +317,16 @@ export type RunToHeightOpts = {
   securityNamespace?: string;
   /** Enable/disable empty-block coalescing. */
   coalesce?: boolean;
+  /**
+   * Full ConfigBuilder output, for tests that need protocols the default
+   * scenario doesn't provide. When omitted the node-runner builds the standard
+   * two-protocol scenario from `events`/`parallelStepSize`.
+   *
+   * NOTE: `tips` must then be keyed by THIS config's protocol names — a
+   * mismatch leaves the synthetic chain on its wall-clock fallback tip, which
+   * silently syncs to "now" instead of the intended height.
+   */
+  config?: any;
 };
 
 const RUNNER = join(import.meta.dir, "node-runner.ts");
@@ -342,6 +352,9 @@ function makeRunToHeight(
       apiPort: opts.apiPort,
       waitPages: opts.waitPages,
       coalesce: opts.coalesce,
+      // Forwarded so a caller can supply its own protocols; node-runner already
+      // prefers `spec.config` over the default scenario.
+      config: opts.config,
     };
     return new Promise<void>((resolve, reject) => {
       const child = spawn("bun", [RUNNER, JSON.stringify(spec)], {
@@ -458,8 +471,14 @@ function makeRunNode(
       }
     })();
 
+    // Readiness here means "the HTTP server is listening", not "the node is
+    // healthy": /health answers 503 while the node is still `starting` (no
+    // block applied yet) or `stalled`, and several tests deliberately boot a
+    // node into exactly those states.
     const up = await pollUntil(
-      () => fetch(`http://127.0.0.1:${opts.apiPort}/health`).then((r) => r.ok),
+      () =>
+        fetch(`http://127.0.0.1:${opts.apiPort}/health`)
+          .then((r) => r.status === 200 || r.status === 503),
       10_000,
     );
     if (!up) {

package/test/reproduction/health.test.ts

@@ -0,0 +1,229 @@
+/**
+ * Sync liveness on `/health` (sync/CLAUDE.md Finding #6).
+ *
+ * `/health` used to report only `poolErrors.state()` — database reachability.
+ * Every failure mode found in this investigation is invisible to that check:
+ * a blackholed RPC, a producer whose stream ended, or simply one chain that
+ * stopped advancing all end the same way — the merge blocks on that chain's
+ * page and block production stops — while the database stays perfectly
+ * healthy and `/health` keeps answering `200 {"status":"ok"}`.
+ *
+ * This test stalls the merge deterministically (a parallel chain pinned far
+ * behind the main clock, the head-of-line case from `buffering.test.ts` 1b) and
+ * asserts that `/health` now:
+ *   - reports 503 with `status: "stalled"`,
+ *   - names the protocol the merge is waiting on (`blockingMerge`),
+ *   - and still reports the healthy chain as `ok`.
+ *
+ * The apply-lag threshold is driven by EFFECTSTREAM_LAG_THRESHOLD_MS so this
+ * runs in seconds rather than the 20× block-time default.
+ */
+import { afterEach, beforeEach, expect, test } from "bun:test";
+import {
+  ConfigBuilder,
+  ConfigNetworkType,
+  ConfigSyncProtocolType,
+} from "@effectstream/config";
+import { TestChainControl } from "@effectstream/sync";
+import {
+  type Harness,
+  type RunningNode,
+  setupHarness,
+  sleep,
+  TEST_PRIMITIVE_TYPE,
+} from "./harness.ts";
+
+const START_TIME = 1_700_000_000_000;
+const BLOCK_TIME_MS = 1000;
+const STEP_SIZE = 1000;
+
+const MAIN = "healthMain";
+const PAR_OK = "healthParallelOk";
+const PAR_STALLED = "healthParallelStalled";
+
+/** Main clock runs far ahead; the stalled chain is pinned just above genesis. */
+const MAIN_TIP = 5_000;
+const OK_TIP = 5_000;
+const STALLED_TIP = 5;
+
+/** Well below the time this test spends waiting, so "stalled" latches quickly. */
+const LAG_THRESHOLD_MS = 2_000;
+
+function buildConfig() {
+  return new ConfigBuilder()
+    .setNamespace((b) => b.setSecurityNamespace("sync-health"))
+    .buildNetworks((b) =>
+      b
+        .addNetwork({
+          name: "clock",
+          type: ConfigNetworkType.TEST,
+          startTime: START_TIME,
+          blockTimeMS: BLOCK_TIME_MS,
+        })
+        .addNetwork({
+          name: "chainOk",
+          type: ConfigNetworkType.TEST,
+          startTime: START_TIME,
+          blockTimeMS: BLOCK_TIME_MS,
+        })
+        .addNetwork({
+          name: "chainStalled",
+          type: ConfigNetworkType.TEST,
+          startTime: START_TIME,
+          blockTimeMS: BLOCK_TIME_MS,
+        })
+    )
+    .buildDeployments((b) => b)
+    .buildSyncProtocols((b) =>
+      b
+        .addMain(
+          (n) => n.clock,
+          () => ({
+            name: MAIN,
+            type: ConfigSyncProtocolType.TEST_MAIN,
+            startBlockHeight: 1,
+            pollingInterval: 30,
+            stepSize: STEP_SIZE,
+          }),
+        )
+        .addParallel(
+          (n) => (n as any).chainOk,
+          () => ({
+            name: PAR_OK,
+            type: ConfigSyncProtocolType.TEST_PARALLEL,
+            startBlockHeight: 1,
+            pollingInterval: 30,
+            stepSize: STEP_SIZE,
+            delayMs: 0,
+            events: [],
+          }),
+        )
+        .addParallel(
+          (n) => (n as any).chainStalled,
+          () => ({
+            name: PAR_STALLED,
+            type: ConfigSyncProtocolType.TEST_PARALLEL,
+            startBlockHeight: 1,
+            pollingInterval: 30,
+            stepSize: STEP_SIZE,
+            delayMs: 0,
+            events: [],
+          }),
+        )
+    )
+    .buildPrimitives((b) =>
+      b.addPrimitive(
+        (sp) => (sp as any)[PAR_OK],
+        () => ({ name: "noEvtHealth", type: TEST_PRIMITIVE_TYPE, startBlockHeight: 1 }),
+      )
+    )
+    .build();
+}
+
+type HealthProtocol = {
+  name: string;
+  status: string;
+  blockingMerge: boolean;
+  ownBlockNumber: number | null;
+  consecutiveErrors: number;
+  producerRestarts: number;
+};
+type HealthBody = {
+  status: string;
+  db: { sustained: boolean };
+  apply: {
+    blockHeight: number | null;
+    sinceLastAppliedMs: number | null;
+    lagMs: number | null;
+  };
+  protocols: HealthProtocol[];
+};
+
+async function getHealth(
+  port: number,
+): Promise<{ httpStatus: number; body: HealthBody }> {
+  const res = await fetch(`http://127.0.0.1:${port}/health`);
+  return { httpStatus: res.status, body: (await res.json()) as HealthBody };
+}
+
+let h: Harness;
+let node: RunningNode | undefined;
+
+beforeEach(async () => {
+  TestChainControl.clear();
+  process.env.EFFECTSTREAM_LAG_THRESHOLD_MS = String(LAG_THRESHOLD_MS);
+  h = await setupHarness();
+});
+afterEach(async () => {
+  await node?.stop();
+  node = undefined;
+  delete process.env.EFFECTSTREAM_LAG_THRESHOLD_MS;
+  await h?.teardown();
+});
+
+test("/health reports a stalled merge instead of a bare 'ok'", async () => {
+  TestChainControl.setTip(MAIN, MAIN_TIP);
+  TestChainControl.setTip(PAR_OK, OK_TIP);
+  // Pinned far behind: once the root timestamp passes this chain's tip, the
+  // merge blocks on its page and block production stops for good.
+  TestChainControl.setTip(PAR_STALLED, STALLED_TIP);
+
+  node = await h.runNode({ config: buildConfig(), apiPort: 19171 });
+
+  // Let the node apply what it can, then sit past the lag threshold.
+  await sleep(LAG_THRESHOLD_MS * 3);
+
+  const { httpStatus, body } = await getHealth(node.apiPort);
+
+  // The database is fine — this is exactly why the old db-only check passed.
+  expect(body.db.sustained).toBe(false);
+
+  // ...but the node has not applied a block in a long time. Measured in
+  // wall-clock time since the last apply, not block-timestamp lag: the
+  // synthetic chain's blocks are dated 2023, and a node replaying history is
+  // legitimately far behind in block time while being perfectly healthy.
+  expect(body.status).toBe("stalled");
+  expect(httpStatus).toBe(503);
+  expect(body.apply.sinceLastAppliedMs).toBeGreaterThan(LAG_THRESHOLD_MS);
+
+  // The report must name WHICH chain is holding up the merge, otherwise an
+  // operator has a stalled node and no idea where to look.
+  const stalled = body.protocols.find((p) => p.name === PAR_STALLED);
+  expect(stalled).toBeDefined();
+  expect(stalled!.blockingMerge).toBe(true);
+
+  // The healthy chains are still reported healthy — "stalled" is a property of
+  // the pipeline, not of every protocol in it.
+  const ok = body.protocols.find((p) => p.name === PAR_OK);
+  expect(ok).toBeDefined();
+  expect(ok!.blockingMerge).toBe(false);
+  expect(ok!.consecutiveErrors).toBe(0);
+}, 120_000);
+
+test("/health reports ok while blocks are flowing", async () => {
+  // All chains advancing: the merge is never gated, blocks keep applying.
+  TestChainControl.setTip(MAIN, MAIN_TIP);
+  TestChainControl.setTip(PAR_OK, OK_TIP);
+  TestChainControl.setTip(PAR_STALLED, OK_TIP);
+
+  node = await h.runNode({ config: buildConfig(), apiPort: 19172 });
+
+  // Poll: the node needs a moment to apply its first block, and once it does
+  // the status must be ok (not merely "not stalled").
+  let body: HealthBody | undefined;
+  let httpStatus = 0;
+  const deadline = Date.now() + 30_000;
+  while (Date.now() < deadline) {
+    ({ httpStatus, body } = await getHealth(node.apiPort));
+    if (body.status === "ok") break;
+    await sleep(100);
+  }
+
+  expect(body!.status).toBe("ok");
+  expect(httpStatus).toBe(200);
+  expect(body!.apply.blockHeight).toBeGreaterThan(0);
+  for (const p of body!.protocols) {
+    expect(p.consecutiveErrors).toBe(0);
+    expect(p.producerRestarts).toBe(0);
+  }
+}, 120_000);