envio 3.13.0 → 3.14.0-subgraph

package/src/subgraph/hosts.ts

@@ -0,0 +1,158 @@
+/**
+ * The graph-node host ops that reach outside the chain data envio already has:
+ * IPFS, Arweave, ENS, `eth_getBalance`/`eth_getCode`, and a block handler's
+ * timestamp. Each is an envio effect, so preload batches and dedupes them and
+ * the sync bridge suspends the mapping until the answer lands.
+ */
+
+import { rpcClient } from "./rpc.ts";
+import * as Sury from "rescript-schema";
+import { createEffect } from "../Envio.res.mjs";
+import { configureBlockTimestamps, requestBlockTimestamp } from "./blocks.ts";
+import { missingRpcMessage } from "./errors.ts";
+
+const IPFS_GATEWAY = "https://ipfs.io/ipfs/";
+const ARWEAVE_GATEWAY = "https://arweave.net/";
+/** graph-node heals a hash from its rainbow table; this is the public one. */
+const ENS_RAINBOW = "https://api.ensrainbow.io/v1/heal/";
+
+const nullableString = Sury.union([Sury.string, null]);
+
+/** A stalled gateway would hold the whole batch, which waits on the mapping. */
+const GATEWAY_TIMEOUT_MS = 30_000;
+
+/**
+ * Only "the gateway doesn't have it" is an answer, and a cached one; anything
+ * else — a timeout, a 5xx, a rate limit — throws, so envio retries the batch
+ * instead of caching a null for good.
+ */
+async function fetchBase64(url: string): Promise<string | null> {
+  const response = await fetch(url, { signal: AbortSignal.timeout(GATEWAY_TIMEOUT_MS) });
+  if (response.status === 404) {
+    return null;
+  }
+  if (!response.ok) {
+    throw new Error(`Envio Subgraph's fetch of ${url} failed: ${response.status} ${response.statusText}`);
+  }
+  const buffer = new Uint8Array(await response.arrayBuffer());
+  return Buffer.from(buffer).toString("base64");
+}
+
+export type HostEffects = ReturnType<typeof makeHostEffects>;
+
+export function makeHostEffects(rpcUrls: string[]) {
+  configureBlockTimestamps(rpcUrls);
+
+  const clientOrThrow = (callSite: string) => {
+    if (rpcUrls.length === 0) {
+      throw new Error(missingRpcMessage(callSite));
+    }
+    return rpcClient(rpcUrls);
+  };
+
+  const ipfsCat = createEffect(
+    {
+      name: "envio_subgraph_ipfs_cat",
+      input: Sury.string,
+      output: nullableString,
+      rateLimit: false,
+      cache: true,
+    },
+    async ({ input }: { input: string }) => fetchBase64(IPFS_GATEWAY + input),
+  );
+
+  const arweaveData = createEffect(
+    {
+      name: "envio_subgraph_arweave",
+      input: Sury.string,
+      output: nullableString,
+      rateLimit: false,
+      cache: true,
+    },
+    async ({ input }: { input: string }) => fetchBase64(ARWEAVE_GATEWAY + input),
+  );
+
+  // graph-node returns null when its rainbow table doesn't hold the hash.
+  // ENSRainbow answers that as a 404, and a hash that isn't one as a 400;
+  // anything else is the service failing, which must not be cached as a miss.
+  const ensName = createEffect(
+    {
+      name: "envio_subgraph_ens_name",
+      input: Sury.string,
+      output: nullableString,
+      rateLimit: false,
+      cache: true,
+    },
+    async ({ input }: { input: string }) => {
+      const url = ENS_RAINBOW + input;
+      const response = await fetch(url, { signal: AbortSignal.timeout(GATEWAY_TIMEOUT_MS) });
+      if (response.status === 404 || response.status === 400) {
+        return null;
+      }
+      if (!response.ok) {
+        throw new Error(`Envio Subgraph's lookup of ${url} failed: ${response.status} ${response.statusText}`);
+      }
+      const body = (await response.json()) as { label?: string };
+      return body.label ?? null;
+    },
+  );
+
+  const getBalance = createEffect(
+    {
+      name: "envio_subgraph_eth_balance",
+      input: Sury.string,
+      output: Sury.string,
+      rateLimit: false,
+      cache: true,
+      crossChain: false,
+    },
+    async ({ input }: { input: string }) => {
+      const { address, blockNumber } = JSON.parse(input);
+      const balance = await clientOrThrow("ethereum.getBalance").getBalance({
+        address,
+        blockNumber: BigInt(blockNumber),
+      });
+      return balance.toString();
+    },
+  );
+
+  const hasCode = createEffect(
+    {
+      name: "envio_subgraph_has_code",
+      input: Sury.string,
+      output: Sury.boolean,
+      rateLimit: false,
+      cache: true,
+      crossChain: false,
+    },
+    async ({ input }: { input: string }) => {
+      const { address, blockNumber } = JSON.parse(input);
+      const code = await clientOrThrow("ethereum.hasCode").getCode({
+        address,
+        blockNumber: BigInt(blockNumber),
+      });
+      return code !== undefined && code !== "0x";
+    },
+  );
+
+  // Uncached on purpose: a block's timestamp is read exactly once, by that
+  // block's own handler invocation, so a persisted row per indexed block would
+  // be bloat with no reuse. The in-memory memo still covers replay rounds and
+  // the preload -> execute transition.
+  const blockTimestamp = createEffect(
+    {
+      name: "envio_subgraph_block_timestamp",
+      input: Sury.number,
+      output: Sury.string,
+      rateLimit: false,
+      cache: false,
+      crossChain: false,
+    },
+    async ({ input, context }: { input: number; context: { chain: { id: number } } }) => {
+      const timestamp = await requestBlockTimestamp(context.chain.id, input);
+      return timestamp.toString();
+    },
+  );
+
+  return { ipfsCat, arweaveData, ensName, getBalance, hasCode, blockTimestamp };
+}

package/src/subgraph/rpc.ts

@@ -0,0 +1,65 @@
+/**
+ * The one client every RPC read of the subgraph runtime goes through: contract
+ * calls, `getBalance`/`hasCode` and block timestamps.
+ *
+ * Preload runs a batch's handlers at once, and with them every read they make.
+ * Unbounded, that is one open connection per read — thousands on a factory's
+ * backfill, until the endpoint stops answering and the batch never completes.
+ * So requests beyond a fixed number wait for one in flight to finish; each is
+ * short, and a queue drains far faster than a refused connection retries.
+ */
+
+import { createPublicClient, fallback, http, type Transport } from "viem";
+
+const MAX_IN_FLIGHT = 16;
+
+/** Runs at most `MAX_IN_FLIGHT` requests at once, queueing the rest. */
+function gate() {
+  let inFlight = 0;
+  const queued: (() => void)[] = [];
+  return async <T>(request: () => Promise<T>): Promise<T> => {
+    if (inFlight < MAX_IN_FLIGHT) inFlight++;
+    // A finishing request hands its slot straight over.
+    else await new Promise<void>((resolve) => queued.push(resolve));
+    try {
+      return await request();
+    } finally {
+      const next = queued.shift();
+      if (next) next();
+      else inFlight--;
+    }
+  };
+}
+
+/** Bounded per set of endpoints, so one chain's slow RPC can't stall another's. */
+function boundedTransport(transport: Transport): Transport {
+  const bounded = gate();
+  return (options) => {
+    const inner = transport(options);
+    return {
+      ...inner,
+      request: ((args) => bounded(() => inner.request(args))) as typeof inner.request,
+    };
+  };
+}
+
+const clients = new Map<string, ReturnType<typeof createPublicClient>>();
+
+export function rpcClient(rpcUrls: string[]) {
+  const key = rpcUrls.join("|");
+  let client = clients.get(key);
+  if (!client) {
+    client = createPublicClient({
+      // Retrying is envio's job: a failed read becomes a handler error and
+      // the batch is retried with the effect's dedup still in place.
+      transport: boundedTransport(fallback(rpcUrls.map((url) => http(url, { retryCount: 0 })))),
+    });
+    clients.set(key, client);
+  }
+  return client;
+}
+
+/** Reset between test indexers, which each bring their own endpoints. */
+export function resetRpcClients() {
+  clients.clear();
+}