envio 3.15.0-subgraph → 3.16.0

package/src/subgraph/calls.ts

@@ -1,258 +0,0 @@
-/**
- * `Contract.bind(x).foo()` -> an envio effect, encoded and decoded with
- * alloy, as graph-node does.
- *
- * graph-node evaluates a mapping's contract calls against the block the event
- * came from, so the block number is part of the effect's input — and therefore
- * of its cache key, which is what makes a cached call safe to reuse.
- */
-
-import * as Sury from "rescript-schema";
-import { subgraphAbiDecode, subgraphAbiEncodeCall } from "../Core.res.mjs";
-import { createEffect } from "../Envio.res.mjs";
-import { missingRpcMessage } from "./errors.ts";
-import { rpcClient } from "./rpc.ts";
-import { parseSignature } from "./abi-types.ts";
-import { isRevert, untilAnswered } from "./transient.ts";
-import { info } from "./log.ts";
-
-export type CallInput = {
-  chainId: number;
-  address: string;
-  signature: string;
-  args: unknown[];
-  blockNumber: number;
-};
-
-export type CallOutput = {
-  reverted: boolean;
-  values: unknown[] | null;
-};
-
-/**
- * A batch commits only once every contract call its handlers make is in, and a
- * call-heavy backfill's first batch can wait minutes on thousands of them with
- * nothing else to show, which reads as a hang. While calls are
- * outstanding, how many were answered is logged now and then.
- */
-const progress = { started: 0, answered: 0, reportedAnswered: 0, timer: undefined as NodeJS.Timeout | undefined };
-
-function progressIntervalMs(): number {
-  const value = Number(process.env.ENVIO_SUBGRAPH_PROGRESS_LOG_MS);
-  return Number.isFinite(value) && value > 0 ? value : 30_000;
-}
-
-function reportProgress() {
-  const waiting = progress.started - progress.answered;
-  if (waiting <= 0) {
-    clearInterval(progress.timer);
-    progress.timer = undefined;
-    return;
-  }
-  const interval = progressIntervalMs();
-  const span = interval >= 1000 ? `${Math.round(interval / 1000)}s` : `${interval}ms`;
-  info(
-    `Envio Subgraph: ${progress.answered - progress.reportedAnswered} contract calls answered in the last ` +
-      `${span}, ${waiting} waiting. The batch is written once its handlers' calls are all in.`,
-  );
-  progress.reportedAnswered = progress.answered;
-}
-
-function trackCall<T>(call: Promise<T>): Promise<T> {
-  progress.started++;
-  if (!progress.timer) {
-    progress.timer = setInterval(reportProgress, progressIntervalMs());
-    progress.timer.unref();
-  }
-  const settle = () => {
-    progress.answered++;
-  };
-  call.then(settle, settle);
-  return call;
-}
-
-/**
- * Functions whose answer a contract does not change once it has given one that
- * isn't empty: token metadata, a pair's tokens, a registry's factory. A mapping
- * asks them again at every block it handles (a token's `decimals()` on every
- * swap of every pair that holds it, a factory's `getPair(a, b)` on every sync)
- * and the effect cache keys a call by block, so each ask was an RPC request.
- * Once one has been answered at block B with a value, any block from B on gets
- * that value without a request. Only a non-empty answer counts: a zero address
- * or an empty string is "not there yet" (`getPair` before the pair exists,
- * `symbol()` before the token is deployed), and a revert is never shared.
- * A proxy that changes its `name()` later is the one case this gets wrong;
- * ENVIO_SUBGRAPH_EXACT_CALLS=1 asks every call at its own block instead.
- */
-export const STABLE_FUNCTIONS = new Set([
-  "name",
-  "symbol",
-  "decimals",
-  "NAME",
-  "SYMBOL",
-  "DECIMALS",
-  "token0",
-  "token1",
-  "factory",
-  "getPair",
-  "comptroller",
-  "underlying",
-]);
-
-type Shared = { block: number; output: string };
-
-/** The call without its block: what stays true from the block it was first answered at. */
-function stableKey(input: CallInput): string | undefined {
-  if (process.env.ENVIO_SUBGRAPH_EXACT_CALLS === "1") return undefined;
-  if (!STABLE_FUNCTIONS.has(parseSignature(input.signature).name)) return undefined;
-  // An address argument is the same one whatever its checksum casing.
-  const args = input.args.map((arg) => (/^s:0x[0-9a-fA-F]*$/.test(String(arg)) ? String(arg).toLowerCase() : arg));
-  return JSON.stringify([input.chainId, input.address.toLowerCase(), input.signature, args]);
-}
-
-const sharedAnswers = new Map<string, Shared>();
-const sharedInFlight = new Map<string, { block: number; settled: Promise<void> }>();
-const SHARED_ANSWERS_LIMIT = 200_000;
-
-/** Reset between test indexers, which each bring their own chain. */
-export function resetSharedCalls() {
-  sharedAnswers.clear();
-  sharedInFlight.clear();
-}
-
-function remember(key: string, block: number, output: string) {
-  const answer = JSON.parse(output) as CallOutput;
-  // A zero address or an empty string is "not there yet"; a revert says nothing.
-  const empty = (answer.values ?? []).every((v) => typeof v === "string" && /^s:(0x0*)?$/i.test(v));
-  if (answer.reverted || !answer.values || empty) return;
-  const earlier = sharedAnswers.get(key);
-  if (earlier && earlier.block <= block) return;
-  if (sharedAnswers.size >= SHARED_ANSWERS_LIMIT) sharedAnswers.clear();
-  sharedAnswers.set(key, { block, output });
-}
-
-/**
- * A stable function's answer from an earlier block, one still being asked for
- * (waited on, then reused if it turns out to be a value), or `ask`.
- */
-export async function shareAcrossBlocks(input: CallInput, ask: () => Promise<string>): Promise<string> {
-  const key = stableKey(input);
-  if (key === undefined) return ask();
-  const known = sharedAnswers.get(key);
-  if (known && known.block <= input.blockNumber) return known.output;
-
-  const running = sharedInFlight.get(key);
-  if (running && running.block <= input.blockNumber) {
-    await running.settled;
-    const after = sharedAnswers.get(key);
-    if (after && after.block <= input.blockNumber) return after.output;
-  }
-
-  const asked = ask();
-  const entry = { block: input.blockNumber, settled: asked.then(() => undefined, () => undefined) };
-  if (!sharedInFlight.has(key) || sharedInFlight.get(key)!.block > input.blockNumber) {
-    sharedInFlight.set(key, entry);
-  }
-  try {
-    const output = await asked;
-    remember(key, input.blockNumber, output);
-    return output;
-  } finally {
-    if (sharedInFlight.get(key) === entry) sharedInFlight.delete(key);
-  }
-}
-
-/** An argument as the addon reads it: an integer as its decimal string. */
-function jsonArg(value: unknown): unknown {
-  if (typeof value === "bigint") return value.toString();
-  if (Array.isArray(value)) return value.map(jsonArg);
-  return value;
-}
-
-const hexBytes = (hex: string) => Buffer.from(hex.replace(/^0x/, ""), "hex");
-
-async function askNode(rpcUrls: string[], input: CallInput): Promise<string> {
-  const { name, inputs, outputs } = parseSignature(input.signature);
-  const data = subgraphAbiEncodeCall(name, `(${inputs})`, input.args.map(decodeArg).map(jsonArg));
-
-  const result = await trackCall(
-    untilAnswered(
-      `${input.signature} on ${input.address} at block ${input.blockNumber}`,
-      async () => {
-        try {
-          return await rpcClient(rpcUrls).call({ to: input.address, data, blockNumber: input.blockNumber });
-        } catch (error) {
-          if (isRevert(error)) return null;
-          throw error;
-        }
-      },
-      { deadline: false },
-    ),
-  );
-  if (result === null) {
-    return JSON.stringify({ reverted: true, values: null });
-  }
-
-  try {
-    // graph-ts returns one `ethereum.Value` per ABI output. A single
-    // tuple/struct is therefore `[tuple]`, not the flattened fields —
-    // generated bindings call `result[0].toTuple()`.
-    const values = subgraphAbiDecode(`(${outputs})`, hexBytes(result), true) as unknown[];
-    return JSON.stringify({ reverted: false, values: values.map(encodeArg) });
-  } catch {
-    // Output the declared signature can't decode counts as a failed call,
-    // which is what lets a mapping retry through a different ABI — the
-    // bytes32-vs-string ERC20 name is the case every token subgraph hits. A
-    // function answering no bytes at all is the same: the account has no code.
-    return JSON.stringify({ reverted: true, values: null });
-  }
-}
-
-export function makeCallEffect(rpcUrls: string[], recall: (encoded: string) => string | undefined = () => undefined) {
-  return createEffect(
-    {
-      name: "envio_subgraph_eth_call",
-      // Both sides travel as JSON text: the cache key is then the exact call
-      // description, and the cached row is readable.
-      input: Sury.string,
-      output: Sury.string,
-      rateLimit: false,
-      cache: true,
-      crossChain: false,
-    },
-    async ({ input: encoded }: { input: string }) => {
-      // The register pass asked the same question at fetch time; its answer
-      // is the same one, and asking twice doubled a factory's calls.
-      const recalled = recall(encoded);
-      if (recalled !== undefined) return recalled;
-      const input = JSON.parse(encoded) as CallInput;
-      if (rpcUrls.length === 0) {
-        throw new Error(missingRpcMessage(input.signature));
-      }
-      return shareAcrossBlocks(input, () => askNode(rpcUrls, input));
-    },
-  );
-}
-
-/** Effect inputs/outputs travel as strings, tagged so the type survives. */
-export function encodeArg(value: unknown): string {
-  if (typeof value === "bigint") return `i:${value.toString()}`;
-  if (typeof value === "boolean") return `b:${value ? "1" : "0"}`;
-  if (Array.isArray(value)) return `a:${JSON.stringify(value.map(encodeArg))}`;
-  return `s:${String(value)}`;
-}
-
-export function decodeArg(value: string): unknown {
-  const tag = value.slice(0, 2);
-  const body = value.slice(2);
-  switch (tag) {
-    case "i:":
-      return BigInt(body);
-    case "b:":
-      return body === "1";
-    case "a:":
-      return (JSON.parse(body) as string[]).map(decodeArg);
-    default:
-      return body;
-  }
-}

package/src/subgraph/conformance.ts

@@ -1,106 +0,0 @@
-/**
- * The shim, checked against the types a subgraph developer codes against.
- *
- * A subgraph's mappings are type-checked by `asc` against
- * `@graphprotocol/graph-ts`, and then run by this shim. Nothing else makes the
- * two agree, so a gap shows up as green types and a runtime error — the worst
- * feedback loop there is. This file closes it.
- *
- * The declarations come from the real package (see
- * `scripts/generate-graph-ts-types.mjs`). Every export is walked, and within
- * each class and namespace every static and every instance member, so a gap is
- * named down to the member — `ethereum.Value.fromAddress` — rather than hidden
- * behind a suppression covering a whole namespace. This file is type-checked,
- * never run.
- */
-
-import type * as GraphTs from "./graph-ts-types/index.js";
-import type * as Shim from "./graph-ts.ts";
-
-/** A class's statics: a mapped type drops the construct signature (see `Constructs`). */
-type Statics<T> = { [K in keyof T as K extends "prototype" ? never : K]: T[K] };
-
-type Agrees<Real, Ours, Path extends string> = [Ours] extends [Real] ? never : Path;
-
-/**
- * A public constructor the shim's has to accept, named `Path()` — a mapping
- * can build an `ethereum.Event` itself. A protected one isn't a construct
- * signature, so it is never compared.
- */
-type Constructs<Real, Ours, Path extends string> = [Real] extends [abstract new (...args: infer Args) => any]
-  ? [Ours] extends [abstract new (...args: infer OursArgs) => any]
-    ? // By arity: a shorter parameter list is assignable to a longer one, and
-      // would take the arguments a mapping passes without a word. The types
-      // themselves can't be compared this way round — graph-ts' `BigInt` is a
-      // `Uint8Array`, the shim's isn't — and the members already are.
-      number extends OursArgs["length"]
-      ? never
-      : OursArgs["length"] extends Args["length"]
-        ? never
-        : `${Path}()`
-    : `${Path}()`
-  : never;
-
-/**
- * Deep enough to reach `ethereum.Value.fromAddress`, and no deeper: a member
- * past that is compared whole, which still catches any disagreement inside it.
- */
-type Member<Real, Ours, Path extends string, Depth extends unknown[]> = Depth["length"] extends 2
-  ? Agrees<Real, Ours, Path>
-  : // `any` would take every branch below at once.
-    0 extends 1 & Real
-    ? Agrees<Real, Ours, Path>
-    : // Bracketed so a union — an enum, `boolean` — is compared whole.
-      [Real] extends [(...args: any) => any]
-    ? Agrees<Real, Ours, Path>
-    : // A class, even one whose constructor is protected.
-      [Real] extends [{ prototype: infer RealInstance }]
-      ? [Ours] extends [{ prototype: infer OursInstance }]
-        ?
-            | Constructs<Real, Ours, Path>
-            | Gaps<Statics<Real>, Statics<Ours>, `${Path}.`, [...Depth, unknown]>
-            | Gaps<RealInstance, OursInstance, `${Path}#`, [...Depth, unknown]>
-        : Path
-      : [Real] extends [object]
-        ? Gaps<Real, Ours, `${Path}.`, [...Depth, unknown]>
-        : Agrees<Real, Ours, Path>;
-
-/** Where `Ours` lacks or disagrees with `Real`, by path. */
-type Gaps<Real, Ours, Path extends string, Depth extends unknown[] = []> = {
-  [K in keyof Real & string]-?: K extends keyof Ours
-    ? Member<Real[K], Ours[K], `${Path}${K}`, Depth>
-    : `${Path}${K}`;
-}[keyof Real & string];
-
-type Found = Gaps<typeof GraphTs, typeof Shim, "">;
-
-/** Every gap the shim knows it has, and why. */
-type KnownGap =
-  // Chains envio doesn't index in subgraph mode: their namespaces refuse a read.
-  | "arweave.Block"
-  | "arweave.ProofOfAccess"
-  | "arweave.Tag"
-  | "arweave.Transaction"
-  | "arweave.TransactionWithBlockPtr"
-  // Refused at runtime: a YAML parser isn't shipped with the shim.
-  | `yaml.${string}`
-  | `YAMLValue${string}`
-  | `YAMLTaggedValue${string}`
-  // graph-ts declares `CallResult._value` private, which no other declaration
-  // can match; the shim's is structurally the same.
-  | "ethereum.CallResult.fromValue"
-  | "ethereum.SmartContract#tryCall"
-  // Constructors only the shim itself calls. graph-ts' BigInt inherits
-  // Uint8Array's and is built through its statics; the others are declared
-  // without one, and a mapping gets each from a call, a parse or `fromValue`.
-  | "BigInt()"
-  | "ethereum.CallResult()"
-  | "Result()"
-  | "JSONValue()";
-
-/** A known gap no longer found, whose entry should go. */
-type Closed<Gap> = Gap extends unknown ? ([Extract<Found, Gap>] extends [never] ? Gap : never) : never;
-
-// Keyed so a failure lists each path as a missing property.
-export const unlisted: Record<Exclude<Found, KnownGap>, "a gap KnownGap doesn't list"> = {};
-export const closed: Record<Closed<KnownGap>, "a KnownGap entry that no longer applies"> = {};

package/src/subgraph/errors.ts

@@ -1,85 +0,0 @@
-const ISSUES_URL = "https://github.com/enviodev/hyperindex/issues";
-
-/**
- * Subgraph mode ships under its own npm dist-tag (see `publish.yml`), so
- * `envio@latest` would move a project off it.
- */
-const UPGRADE_COMMAND = "pnpm add -D envio@subgraph";
-
-/** A feature we recognise and deliberately don't implement. */
-export function unsupported(feature: string, location: string, why?: string): Error {
-  return new Error(
-    `Envio Subgraph doesn't support ${feature} yet${why ? `: ${why}` : ""}.\n` +
-      `  Found in ${location}.\n` +
-      `First, make sure you're on the newest Envio Subgraph release — support may have landed:\n` +
-      `  ${UPGRADE_COMMAND}\n` +
-      `If you're up to date and need this feature, please open an issue (existing\n` +
-      `issues welcome a 👍 — demand drives prioritization):\n  ${ISSUES_URL}`,
-  );
-}
-
-/** Something we don't recognise at all: newer than us, or a typo. */
-export function unknown(thing: string, location: string): Error {
-  return new Error(
-    `Envio Subgraph doesn't know ${thing}.\n` +
-      `  Found in ${location}.\n` +
-      `This may be a feature newer than this envio version understands, or a typo.\n` +
-      `First, make sure you're on the newest Envio Subgraph release:\n` +
-      `  ${UPGRADE_COMMAND}\n` +
-      `If you're up to date and this is a real subgraph feature, please open an\n` +
-      `issue so we can add it: ${ISSUES_URL}`,
-  );
-}
-
-const RPC_ENV_VAR = "ENVIO_SUBGRAPH_RPC";
-
-/** §6b: contract calls need an endpoint HyperSync and HyperRPC don't serve. */
-export function missingRpcMessage(callSite: string): string {
-  return (
-    `This subgraph performs contract calls (${callSite}), which need an\n` +
-    `RPC endpoint — HyperSync and HyperRPC serve logs and blocks, not eth_call.\n` +
-    `Set one in .env or the environment:\n` +
-    `  ${RPC_ENV_VAR}=https://...\n` +
-    `Advanced (matches envio's rpc config; single entry or array):\n` +
-    `  ${RPC_ENV_VAR}={"url":"https://...","for":"fallback","headers":{...}}`
-  );
-}
-
-/**
- * Names a trap must answer for rather than refuse. AS-compiled mappings never
- * feature-detect, so nothing legitimate reaches these — but `console.log`,
- * `await` and `JSON.stringify` do, and they must not explode.
- */
-const PASSTHROUGH = new Set([
-  "then",
-  "toJSON",
-  "constructor",
-  "valueOf",
-  "toString",
-  "inspect",
-  "nodeType",
-  "$$typeof",
-  "@@__IMMUTABLE_ITERABLE__@@",
-]);
-
-function isPassthrough(prop: PropertyKey): boolean {
-  return typeof prop === "symbol" || PASSTHROUGH.has(prop as string);
-}
-
-export const PROTOTYPE_PASSTHROUGH = PASSTHROUGH;
-
-/**
- * Full Proxy for a graph-ts namespace object. Future graph-ts APIs are
- * unenumerable, so only a `get` trap can catch them; these are cold paths, so
- * the trap cost doesn't matter.
- */
-export function strictNamespace<T extends object>(name: string, target: T): T {
-  return new Proxy(target, {
-    get(target, prop, receiver) {
-      if (prop in target || isPassthrough(prop)) {
-        return Reflect.get(target, prop, receiver);
-      }
-      throw unknown(`the graph-ts API ${name}.${String(prop)}`, `a mapping handler`);
-    },
-  });
-}

package/src/subgraph/graft.ts

@@ -1,73 +0,0 @@
-export type Graft = { base: string; block: number };
-
-type Manifest = { startBlocks: number[]; graft: Graft | null };
-
-const GATEWAYS = [
-  "https://api.thegraph.com/ipfs/api/v0/cat?arg=",
-  "https://ipfs.network.thegraph.com/api/v0/cat?arg=",
-];
-const MAX_CHAIN = 8;
-
-async function fetchFromGateways(hash: string): Promise<string | null> {
-  for (const gateway of GATEWAYS) {
-    try {
-      const response = await fetch(gateway + hash, {
-        method: "POST",
-        signal: AbortSignal.timeout(8000),
-      });
-      if (response.ok) return await response.text();
-    } catch {
-      // try the next gateway
-    }
-  }
-  return null;
-}
-
-/** The start blocks and graft of a deployment manifest, read without a YAML parser. */
-export function readManifest(text: string): Manifest {
-  const startBlocks = [...text.matchAll(/^\s*startBlock:\s*["']?(\d+)/gm)].map((m) => Number(m[1]));
-  const block = /^graft:\s*\n((?:[ \t]+.*\n?)+)/m.exec(text)?.[1] ?? "";
-  const base = /^\s*base:\s*["']?(Qm[A-Za-z0-9]+)/m.exec(block)?.[1];
-  const at = /^\s*block:\s*["']?(\d+)/m.exec(block)?.[1];
-  return { startBlocks, graft: base && at ? { base, block: Number(at) } : null };
-}
-
-/**
- * graph-node copies the base deployment's data up to the graft block, and the
- * manifest on IPFS is the only record of where that base started. If it
- * started earlier than any data source here does, a re-index from these start
- * blocks is missing everything in between; the numbers go in the warning.
- * Returns null when the base can't be read (offline, pruned from IPFS).
- */
-export async function graftBaseNotice(
-  graft: Graft,
-  ownStartBlocks: number[],
-  fetchManifest: (hash: string) => Promise<string | null> = fetchFromGateways,
-): Promise<string | null> {
-  let earliest: number | null = null;
-  let hash: string | null = graft.base;
-  for (let depth = 0; hash && depth < MAX_CHAIN; depth++) {
-    const text = await fetchManifest(hash);
-    if (text === null) return null;
-    const manifest = readManifest(text);
-    if (manifest.startBlocks.length > 0) {
-      const lowest = Math.min(...manifest.startBlocks);
-      earliest = earliest === null ? lowest : Math.min(earliest, lowest);
-    }
-    hash = manifest.graft?.base ?? null;
-  }
-  if (earliest === null) return null;
-  const own = ownStartBlocks.length > 0 ? Math.min(...ownStartBlocks) : 0;
-  if (earliest < own) {
-    return (
-      `WARNING, DATA IS MISSING: the graft base ${graft.base} (read from IPFS) starts at block ` +
-      `${earliest}, but the earliest startBlock here is ${own}. Blocks ${earliest} to ${own - 1} ` +
-      `are in the grafted deployment and will not be indexed. Lower the startBlock of the data ` +
-      `sources to ${earliest} to rebuild that history.`
-    );
-  }
-  return (
-    `The graft base ${graft.base} (read from IPFS) starts at block ${earliest}, not after this ` +
-    `manifest's earliest startBlock ${own}: a re-index covers the same blocks.`
-  );
-}

package/src/subgraph/graph-cli-worker.mjs

@@ -1,35 +0,0 @@
-// Runs one graph-cli command, as its own bin would, on the thread
-// `graph-cli.ts` sized for it. Plain JavaScript: the TypeScript loader envio
-// registers doesn't reach worker threads.
-import { parentPort, workerData } from "node:worker_threads";
-import path from "node:path";
-import { pathToFileURL } from "node:url";
-
-const { packageDir, command, argv } = workerData;
-
-let ok = true;
-try {
-  const commandFile = path.join(packageDir, "dist", "commands", `${command}.js`);
-  const exported = await import(pathToFileURL(commandFile).href);
-  // graph-cli before 0.93 is CommonJS compiled from TypeScript: an ES import
-  // hands back `module.exports`, so the class is its `default`, one level down.
-  const Command = [exported.default, exported.default?.default].find(
-    (candidate) => typeof candidate?.run === "function",
-  );
-  if (Command === undefined) {
-    throw new Error(
-      `Envio Subgraph couldn't find a runnable \`graph ${command}\` in ${commandFile}`,
-    );
-  }
-  await Command.run(argv, packageDir);
-  // A generation failure is reported through the exit code, not a throw.
-  ok = process.exitCode === undefined || process.exitCode === 0;
-} catch (error) {
-  // oclif ends a successful command early by throwing an exit of 0.
-  if (error?.oclif?.exit !== 0) {
-    console.error(error?.message ?? error);
-    ok = false;
-  }
-}
-process.exitCode = 0;
-parentPort.postMessage(ok);