@effectstream/sync 0.101.1 → 0.103.0

package/src/sync-protocols/near/NearClient.ts

@@ -1,3 +1,5 @@
+import { DEFAULT_REQUEST_TIMEOUT_MS, fetchWithTimeout } from "../common/http.ts";
+
 // ============================
 // NEAR JSON-RPC type responses
 // ============================
@@ -107,17 +109,25 @@ export function parseNep297Log(log: string): Nep297Event | null {
 
 export class NearClient {
   private readonly rpcUrl: string;
+  /** Per-request deadline; see `sync-protocols/common/http.ts`. */
+  private readonly requestTimeoutMs: number;
 
-  constructor(rpcUrl: string) {
+  constructor(rpcUrl: string, requestTimeoutMs = DEFAULT_REQUEST_TIMEOUT_MS) {
     this.rpcUrl = rpcUrl;
+    this.requestTimeoutMs = requestTimeoutMs;
   }
 
   private async rpc<T>(method: string, params: Record<string, unknown>): Promise<T> {
-    const res = await fetch(this.rpcUrl, {
-      method: "POST",
-      headers: { "Content-Type": "application/json" },
-      body: JSON.stringify({ jsonrpc: "2.0", id: 1, method, params }),
-    });
+    const res = await fetchWithTimeout(
+      this.rpcUrl,
+      {
+        method: "POST",
+        headers: { "Content-Type": "application/json" },
+        body: JSON.stringify({ jsonrpc: "2.0", id: 1, method, params }),
+      },
+      `NEAR ${method}`,
+      this.requestTimeoutMs,
+    );
 
     const json = await res.json();
     if (json.error) {

package/src/sync-protocols/near/fetcher.ts

@@ -4,6 +4,7 @@ import {
 } from "@effectstream/config";
 import { BaseDataFetcher } from "../base/fetcher.ts";
 import type { DataFetched } from "../base/fetcher.ts";
+import { requestTimeoutOf } from "../common/http.ts";
 import type {
   LastPage,
   OutputAndCleanup,
@@ -39,7 +40,10 @@ export class NearFetcher extends BaseDataFetcher<
     readonly config: ConfigType,
   ) {
     super(config.syncProtocol.name);
-    this.client = new NearClient(config.network.rpcUrl);
+    this.client = new NearClient(
+      config.network.rpcUrl,
+      requestTimeoutOf(config.syncProtocol),
+    );
   }
 
   @bound

package/src/sync-protocols/near/state.ts

@@ -11,6 +11,7 @@ import { getPage } from "@effectstream/db";
 import { NearClient } from "./NearClient.ts";
 import { applyDelay } from "../common/utils.ts";
 import { bufferAtCap } from "../common/page-helpers.ts";
+import { requestTimeoutOf } from "../common/http.ts";
 
 export class NearSyncState extends SyncState<
   Input,
@@ -134,7 +135,10 @@ export class NearSyncState extends SyncState<
       page,
       config,
       fetcher,
-      new NearClient(config.network.rpcUrl),
+      new NearClient(
+        config.network.rpcUrl,
+        requestTimeoutOf(config.syncProtocol),
+      ),
       dbConn,
     );
   }

package/src/sync-protocols/orchestration/sync.ts

@@ -4,19 +4,113 @@ import { ComponentNames, log, SeverityNumber } from "@effectstream/log";
 import type { AllSyncProtocols, ISyncProtocol } from "../types.ts";
 import { tryYield } from "@effectstream/utils";
 
+/**
+ * Used when a protocol config carries no `pollingInterval`.
+ *
+ * Every polling protocol declares one via `PollingSyncProtocol`, so this should
+ * be unreachable — it exists because the cost of getting it wrong is not a slow
+ * loop but a dead node. A pass through this loop that neither fetches nor
+ * sleeps never yields to the macrotask queue, which starves the entire process:
+ * no timers, no HTTP server, no other chain's fetch loop, and (for streaming
+ * chains) none of the stream callbacks that would let this loop make progress.
+ * See `sync/test/poll-loop-spin.test.ts`.
+ */
+const FALLBACK_POLLING_INTERVAL_MS = 1_000;
+
+/** This protocol's polling interval, or the fallback above. */
+function pollingIntervalOf(config: { syncProtocol: unknown }): number {
+  const syncProtocol = config.syncProtocol as { pollingInterval?: number };
+  return syncProtocol?.pollingInterval ?? FALLBACK_POLLING_INTERVAL_MS;
+}
+
+/** Backoff bounds for restarting a dead streaming producer. */
+const PRODUCER_MIN_BACKOFF_MS = 1_000;
+const PRODUCER_MAX_BACKOFF_MS = 30_000;
+
 export function* startSync(
   state: AllSyncProtocols,
 ): Operation<void> {
   const iState: ISyncProtocol = state as ISyncProtocol;
 
-  // spawn async task
+  // Resolve once and publish it, so `/health` and anything else reads the same
+  // value this loop paces itself with instead of re-deriving it.
+  state.pollingIntervalMs = pollingIntervalOf(state.fetcher.config);
+
   yield* spawn(function* () {
-    yield* iState.startAsync();
+    if (!iState.hasAsyncProducer) {
+      // Polled chain: startAsync is the base-class no-op. Run it once, as
+      // before — supervising it would mean restarting a no-op forever.
+      yield* iState.startAsync();
+      return;
+    }
+
+    // Streaming chain: all of this protocol's data arrives through the
+    // producer, so if it dies the chain goes silent and the merge blocks on its
+    // page forever. Both of its exit modes used to be unhandled — a throw tore
+    // down the enclosing scope (in production, `start()`, taking every other
+    // chain with it), and a clean return went entirely unnoticed. Supervise
+    // both. See `sync/test/start-async-supervision.test.ts`.
+    let attempt = 0;
+    while (true) {
+      const startedAt = Date.now();
+      const result = yield* tryYield(iState.startAsync());
+      const ranForMs = Date.now() - startedAt;
+
+      if (result.error != null) {
+        // Producer failures are tracked separately from `consecutiveErrors`.
+        // That counter belongs to the fetch loop and is only cleared by a
+        // successful `readData`, so bumping it here left an idle streaming chain
+        // reporting `erroring` until its next block arrived — and conversely, a
+        // successful poll would erase the record of a flapping producer.
+        state.producerErrors++;
+        state.lastProducerErrorMs = Date.now();
+        log.remote(
+          ComponentNames.EFFECTSTREAM_SYNC,
+          [...state.getNamespace(), "startAsync"],
+          SeverityNumber.ERROR,
+          (l) => l(result.error),
+        );
+      } else {
+        // Returning is a failure for a producer: its stream ended. Treated the
+        // same as a throw, because the observable effect is identical — no
+        // more data.
+        log.remote(
+          ComponentNames.EFFECTSTREAM_SYNC,
+          [...state.getNamespace(), "startAsync"],
+          SeverityNumber.WARN,
+          (l) => l("producer returned: stream ended, restarting"),
+        );
+      }
+
+      state.producerRestarts++;
+
+      // De-escalate after a producer that clearly worked. Without this `attempt`
+      // only ever grows, so a stream that ran healthily for a day between
+      // incidents would still be waiting the 30s cap on its next restart —
+      // penalising a healthy chain for ancient history. (It also keeps
+      // `2 ** attempt` from reaching Infinity after ~1024 restarts, which
+      // `Math.min` currently absorbs silently.)
+      if (ranForMs >= PRODUCER_MAX_BACKOFF_MS) {
+        attempt = 0;
+      }
+
+      const backoff = Math.min(
+        PRODUCER_MIN_BACKOFF_MS * 2 ** attempt,
+        PRODUCER_MAX_BACKOFF_MS,
+      );
+      attempt++;
+      yield* sleep(backoff);
+    }
   });
 
   // spawn polling task
   yield* spawn(function* () {
     while (true) {
+      // Liveness heartbeat: stamped every pass regardless of outcome, so a
+      // stale value means the loop is wedged inside a call that never returns
+      // rather than merely idle. See SyncState.lastPollAtMs.
+      state.lastPollAtMs = Date.now();
+
       const inputResult = yield* tryYield(iState.stateToInput());
       if (inputResult.error != null) {
         state.consecutiveErrors++;
@@ -27,18 +121,14 @@ export function* startSync(
           SeverityNumber.ERROR,
           (log) => log(inputResult.error),
         );
-        const { config } = state.fetcher;
-        if ("pollingInterval" in config.syncProtocol) {
-          yield* sleep(config.syncProtocol.pollingInterval);
-        }
+        yield* sleep(state.pollingIntervalMs);
         continue;
       }
       const input = inputResult.data;
-      const { config } = state.fetcher;
       if (input == null) {
-        if ("pollingInterval" in config.syncProtocol) {
-          yield* sleep(config.syncProtocol.pollingInterval);
-        }
+        // Caught up. This is the ordinary steady state, and the pass that must
+        // never skip its sleep — see FALLBACK_POLLING_INTERVAL_MS.
+        yield* sleep(state.pollingIntervalMs);
         continue;
       }
 
@@ -52,9 +142,7 @@ export function* startSync(
           SeverityNumber.ERROR,
           (l) => l(result.error),
         );
-        if ("pollingInterval" in config.syncProtocol) {
-          yield* sleep(config.syncProtocol.pollingInterval);
-        }
+        yield* sleep(state.pollingIntervalMs);
         continue;
       }
       state.consecutiveErrors = 0;

package/src/sync-protocols/solana/SolanaClient.ts

@@ -0,0 +1,188 @@
+import {
+  DEFAULT_REQUEST_TIMEOUT_MS,
+  fetchWithTimeout,
+} from "../common/http.ts";
+
+// ==========================
+// Solana JSON-RPC type defs
+// ==========================
+
+export type SolanaBlock = {
+  blockhash: string;
+  blockTime: number | null;
+  blockHeight: number | null;
+  parentSlot: number;
+  previousBlockhash: string;
+  transactions: SolanaTransaction[];
+};
+
+export type SolanaTransaction = {
+  transaction: {
+    message: {
+      accountKeys: string[];
+      instructions: SolanaInstruction[];
+    };
+    signatures: string[];
+  };
+  /**
+   * Null when the RPC could not decode the transaction (e.g. a version newer
+   * than `maxSupportedTransactionVersion`). Callers must null-check.
+   */
+  meta: {
+    err: unknown | null;
+    logMessages: string[] | null;
+    preBalances: number[];
+    postBalances: number[];
+    /**
+     * Addresses a versioned (v0) transaction pulled in through an address
+     * lookup table. These are NOT in `message.accountKeys`, which carries only
+     * static keys — but `pre`/`postBalances` are indexed over the full list.
+     * See {@link resolveAccountKeys} for the ordering.
+     */
+    loadedAddresses?: {
+      writable: string[];
+      readonly: string[];
+    } | null;
+    /**
+     * SPL token balances after the transaction, one record per (token account,
+     * mint) pair it touched. Returned by `getBlock` with
+     * `transactionDetails: "full"`; absent on transactions that touched no token
+     * account, and on validators old enough not to report them.
+     *
+     * `accountIndex` indexes the SAME resolved list as `pre`/`postBalances` — see
+     * {@link resolveAccountKeys} — so a token account reached through a lookup
+     * table is only findable after resolution.
+     */
+    postTokenBalances?: SolanaTokenBalance[] | null;
+    /** Pre-state counterpart of {@link postTokenBalances}. */
+    preTokenBalances?: SolanaTokenBalance[] | null;
+  } | null;
+};
+
+export type SolanaTokenBalance = {
+  /** Index into the resolved account list, NOT into `message.accountKeys` alone. */
+  accountIndex: number;
+  mint: string;
+  /** Optional: older validators omit it. */
+  owner?: string;
+  /** The owning token program, i.e. SPL Token or Token-2022. Optional for the same reason. */
+  programId?: string;
+  uiTokenAmount: {
+    /** Raw u64 in base units, as a string. */
+    amount: string;
+    decimals: number;
+    uiAmount: number | null;
+    uiAmountString?: string;
+  };
+};
+
+/**
+ * The account list `pre`/`postBalances` are indexed against: static message
+ * keys first, then lookup-table writable addresses, then lookup-table readonly
+ * ones. Legacy transactions have no `loadedAddresses`, so this is just the
+ * static keys.
+ */
+export function resolveAccountKeys(
+  accountKeys: string[],
+  loadedAddresses?: { writable: string[]; readonly: string[] } | null,
+): string[] {
+  if (!loadedAddresses) return accountKeys;
+  return [
+    ...accountKeys,
+    ...(loadedAddresses.writable ?? []),
+    ...(loadedAddresses.readonly ?? []),
+  ];
+}
+
+export type SolanaInstruction = {
+  programId: string;
+  accounts: string[];
+  data: string;
+};
+
+// ===========
+// RPC Client
+// ===========
+
+export class SolanaClient {
+  private readonly rpcUrl: string;
+  /** Per-request deadline; see `sync-protocols/common/http.ts`. */
+  private readonly requestTimeoutMs: number;
+
+  constructor(rpcUrl: string, requestTimeoutMs = DEFAULT_REQUEST_TIMEOUT_MS) {
+    this.rpcUrl = rpcUrl;
+    this.requestTimeoutMs = requestTimeoutMs;
+  }
+
+  private async rpc<T>(
+    method: string,
+    params: unknown[] = [],
+  ): Promise<T> {
+    // Bounded: a blackholed endpoint would otherwise hang readData forever,
+    // freezing block production with every health counter clean (sync
+    // CLAUDE.md finding #4).
+    const res = await fetchWithTimeout(
+      this.rpcUrl,
+      {
+        method: "POST",
+        headers: { "Content-Type": "application/json" },
+        body: JSON.stringify({
+          jsonrpc: "2.0",
+          id: 1,
+          method,
+          params,
+        }),
+      },
+      `Solana ${method}`,
+      this.requestTimeoutMs,
+    );
+
+    const json = await res.json();
+    if (json.error) {
+      const err = new Error(
+        `[Solana] RPC error [${method}]: ${json.error.message ?? JSON.stringify(json.error)}`,
+      ) as Error & { rpcCode?: number };
+      // Preserve the JSON-RPC code so callers can branch on it instead of
+      // pattern-matching human-readable messages.
+      err.rpcCode = typeof json.error.code === "number" ? json.error.code : undefined;
+      throw err;
+    }
+    return json.result as T;
+  }
+
+  async getSlot(): Promise<number> {
+    return this.rpc<number>("getSlot", [
+      { commitment: "confirmed" },
+    ]);
+  }
+
+  async getBlock(
+    slot: number,
+  ): Promise<SolanaBlock | null> {
+    try {
+      return await this.rpc<SolanaBlock | null>("getBlock", [
+        slot,
+        {
+          encoding: "json",
+          transactionDetails: "full",
+          rewards: false,
+          maxSupportedTransactionVersion: 0,
+          commitment: "confirmed",
+        },
+      ]);
+    } catch (e) {
+      // A skipped slot is normal on Solana: no block was produced. Branch on the
+      // JSON-RPC code rather than the message text —
+      //   -32007 SLOT_SKIPPED, -32009 LONG_TERM_STORAGE_SLOT_SKIPPED.
+      // Deliberately NOT -32004 (block not available yet): that is a transient
+      // "ask again" and must keep throwing so the fetcher retries rather than
+      // treating the slot as permanently empty.
+      const code = (e as { rpcCode?: number }).rpcCode;
+      if (code === -32007 || code === -32009) return null;
+      // Fall back to the message for RPCs that omit or remap the code.
+      if (e instanceof Error && e.message.includes("was skipped")) return null;
+      throw e;
+    }
+  }
+
+}