envio 3.15.0 → 3.16.0-subgraph

package/src/subgraph/receipts.ts

@@ -0,0 +1,184 @@
+/**
+ * `event.receipt.logs`: every log of the event's transaction.
+ *
+ * envio hands a handler the event and, for a `receipt: true` handler, the
+ * receipt's scalar fields as transaction fields — but not the other logs of the
+ * transaction. They are read from HyperSync: the logs of the event's block, of
+ * which the transaction's are kept. Every handler invocation in a batch asks
+ * within the same microtask, so one query answers every transaction of a block,
+ * and blocks run side by side. A chain HyperSync doesn't serve, or a failing
+ * HyperSync, falls back to `eth_getTransactionReceipt` when an RPC is set.
+ */
+
+import { rpcClient } from "./rpc.ts";
+import { HttpStatusError, untilAnswered, withDeadline } from "./transient.ts";
+
+const API_TOKEN_ENV_VAR = "ENVIO_API_TOKEN";
+const MAX_QUERIES_IN_FLIGHT = 8;
+/** Blocks whose logs are kept for the transactions that ask later. */
+const KEEP_BLOCKS = 128;
+
+const hypersyncUrl = (chainId: number) => `https://${chainId}.hypersync.xyz/query`;
+
+/** A log as the runtime serves it: hex strings, plain numbers. */
+export type ReceiptLog = {
+  address: string;
+  topics: string[];
+  data: string;
+  logIndex: number;
+  transactionIndex: number;
+  transactionHash: string;
+};
+
+export type ReceiptRequest = { chainId: number; blockNumber: number; transactionIndex: number; transactionHash: string };
+
+let rpcUrls: string[] = [];
+/** Chains HyperSync doesn't serve, which RPC answers alone. */
+let rpcOnly = new Set<number>();
+/** Logs of recently read blocks, by chain then number; insertion order is age. */
+let recent = new Map<string, Promise<ReceiptLog[]>>();
+let inFlight = 0;
+const queued: (() => void)[] = [];
+
+export function configureReceiptLogs(urls: string[], rpcOnlyChains: number[] = []) {
+  rpcUrls = urls;
+  rpcOnly = new Set(rpcOnlyChains);
+  recent = new Map();
+}
+
+async function slot<T>(run: () => Promise<T>): Promise<T> {
+  if (inFlight >= MAX_QUERIES_IN_FLIGHT) await new Promise<void>((resolve) => queued.push(resolve));
+  else inFlight++;
+  try {
+    return await run();
+  } finally {
+    const next = queued.shift();
+    if (next) next();
+    else inFlight--;
+  }
+}
+
+type HyperSyncLogs = {
+  data: { logs?: Record<string, string | number | null>[] }[];
+  next_block: number;
+};
+
+const num = (value: unknown) => Number(value);
+
+async function blockLogsFromHyperSync(chainId: number, blockNumber: number): Promise<ReceiptLog[]> {
+  const token = process.env[API_TOKEN_ENV_VAR];
+  const url = hypersyncUrl(chainId);
+  const logs: ReceiptLog[] = [];
+  let cursor = blockNumber;
+  // A block is answered whole, but resume if the response says otherwise.
+  while (cursor <= blockNumber) {
+    const from = cursor;
+    const ask = rpcUrls.length > 0 ? withDeadline : untilAnswered;
+    const body = await ask(
+      `HyperSync's logs of block ${blockNumber}`,
+      async (signal): Promise<HyperSyncLogs> => {
+        const response = await fetch(url, {
+          method: "POST",
+          signal,
+          headers: {
+            "content-type": "application/json",
+            ...(token ? { authorization: `Bearer ${token}` } : {}),
+          },
+          body: JSON.stringify({
+            from_block: from,
+            to_block: blockNumber + 1,
+            logs: [{}],
+            field_selection: {
+              log: [
+                "block_number",
+                "log_index",
+                "transaction_index",
+                "transaction_hash",
+                "address",
+                "data",
+                "topic0",
+                "topic1",
+                "topic2",
+                "topic3",
+              ],
+            },
+          }),
+        });
+        if (!response.ok) {
+          throw new HttpStatusError(url, response.status, await response.text());
+        }
+        return (await response.json()) as HyperSyncLogs;
+      },
+    );
+    for (const page of body.data ?? []) {
+      for (const log of page.logs ?? []) {
+        if (num(log.block_number) !== blockNumber) continue;
+        logs.push({
+          address: String(log.address),
+          topics: [log.topic0, log.topic1, log.topic2, log.topic3].filter(
+            (topic): topic is string => typeof topic === "string",
+          ),
+          data: String(log.data ?? "0x"),
+          logIndex: num(log.log_index),
+          transactionIndex: num(log.transaction_index),
+          transactionHash: String(log.transaction_hash),
+        });
+      }
+    }
+    if (!body.next_block || body.next_block <= cursor) break;
+    cursor = body.next_block;
+  }
+  return logs.sort((a, b) => a.logIndex - b.logIndex);
+}
+
+function blockLogs(chainId: number, blockNumber: number): Promise<ReceiptLog[]> {
+  const key = `${chainId}:${blockNumber}`;
+  let logs = recent.get(key);
+  if (!logs) {
+    logs = slot(() => blockLogsFromHyperSync(chainId, blockNumber));
+    recent.set(key, logs);
+    // A failed read is not kept: the retry should ask again.
+    logs.catch(() => recent.delete(key));
+    if (recent.size > KEEP_BLOCKS) recent.delete(recent.keys().next().value as string);
+  }
+  return logs;
+}
+
+async function fromRpc(request: ReceiptRequest): Promise<ReceiptLog[]> {
+  const client = rpcClient(rpcUrls);
+  const receipt = await untilAnswered(
+    `the receipt of ${request.transactionHash} from RPC`,
+    () => client.getTransactionReceipt({ hash: request.transactionHash }),
+    { deadline: false },
+  );
+  return (receipt.logs as Record<string, any>[]).map((log) => ({
+    address: log.address,
+    topics: [...log.topics],
+    data: log.data,
+    logIndex: Number(log.logIndex),
+    transactionIndex: Number(log.transactionIndex),
+    transactionHash: log.transactionHash,
+  }));
+}
+
+/** The logs of one transaction, in log order. */
+export async function requestReceiptLogs(request: ReceiptRequest): Promise<ReceiptLog[]> {
+  if (rpcOnly.has(request.chainId) && rpcUrls.length > 0) return fromRpc(request);
+  try {
+    const logs = await blockLogs(request.chainId, request.blockNumber);
+    const mine = logs.filter((log) => log.transactionIndex === request.transactionIndex);
+    // An event of this transaction has to be among them; if not, HyperSync
+    // is behind or answered another fork, and RPC is asked instead.
+    if (mine.length > 0 || rpcUrls.length === 0) return mine;
+  } catch (error) {
+    if (rpcUrls.length === 0) {
+      const message = error instanceof Error ? error.message : String(error);
+      throw new Error(
+        `Envio Subgraph couldn't read the logs of transaction ${request.transactionHash} from HyperSync: ${message}\n` +
+          `Set ${API_TOKEN_ENV_VAR} in .env or the environment — create one at\n` +
+          `https://envio.dev/app/api-tokens — or set ENVIO_SUBGRAPH_RPC to fall back to RPC.`,
+      );
+    }
+  }
+  return fromRpc(request);
+}

package/src/subgraph/register-slots.ts

@@ -0,0 +1,93 @@
+/**
+ * Bounds how many register passes run at once.
+ *
+ * envio's fetch loop starts a `contractRegister` for every item of a response
+ * and awaits them all. A response can hold tens of thousands of items (a chain
+ * kept open by a template rule is fetched far ahead of the handlers), and each
+ * pass is a live mapping run with its own scope, replay rounds and pending
+ * contract calls: Venus on BSC held 10,000 at once and the heap passed 4 GB
+ * while the RPC answered a few calls a second. A pass now needs a slot first;
+ * the rest wait as plain queue entries, which is also the back-pressure the
+ * fetch loop was missing, since it awaits its passes before the next response.
+ *
+ * Slots go to the lowest position first, the order graph-node would process
+ * them in. A pass that waits on an earlier pass (see `runRegisterRounds`) gives
+ * its slot back for the wait — otherwise every slot could be held by passes
+ * waiting on one that never got a slot. Waiting on an RPC answer keeps it: the
+ * answer depends on nothing else here.
+ */
+
+const DEFAULT_REGISTER_CONCURRENCY = 256;
+
+export function registerConcurrencyFromEnv(): number {
+  const value = Number(process.env.ENVIO_SUBGRAPH_REGISTER_CONCURRENCY);
+  return Number.isInteger(value) && value > 0 ? value : DEFAULT_REGISTER_CONCURRENCY;
+}
+
+export type RegisterSlot = {
+  /** Gives the slot back while `wait` is pending and takes one again after. */
+  pause<T>(wait: Promise<T>): Promise<T>;
+  release(): void;
+};
+
+export type RegisterSlots = {
+  acquire(position: number): Promise<RegisterSlot>;
+  /** Passes holding a slot, and passes queued for one. */
+  stats(): { active: number; queued: number };
+};
+
+export function createRegisterSlots(limit = registerConcurrencyFromEnv()): RegisterSlots {
+  let active = 0;
+  const queue: { position: number; grant: () => void }[] = [];
+
+  const take = (position: number): Promise<void> => {
+    if (active < limit && queue.length === 0) {
+      active++;
+      return Promise.resolve();
+    }
+    return new Promise<void>((grant) => {
+      let at = queue.length;
+      while (at > 0 && queue[at - 1].position > position) at--;
+      queue.splice(at, 0, { position, grant });
+    });
+  };
+
+  // A finishing pass hands its slot straight to the lowest queued position.
+  const give = () => {
+    const next = queue.shift();
+    if (next) next.grant();
+    else active--;
+  };
+
+  const slotFor = (position: number): RegisterSlot => {
+    let held = true;
+    return {
+      async pause(wait) {
+        if (!held) return wait;
+        held = false;
+        give();
+        try {
+          return await wait;
+        } finally {
+          await take(position);
+          held = true;
+        }
+      },
+      release() {
+        if (!held) return;
+        held = false;
+        give();
+      },
+    };
+  };
+
+  return {
+    async acquire(position) {
+      await take(position);
+      return slotFor(position);
+    },
+    stats: () => ({ active, queued: queue.length }),
+  };
+}
+
+export const registerSlots = createRegisterSlots();

package/src/subgraph/rollup.ts

@@ -0,0 +1,109 @@
+import {
+  aggregationEntity,
+  bucketsBetween,
+  compileAggregation,
+  INTERVAL_MICROS,
+  ROLLUP_STATE_ENTITY,
+  rollupBucket,
+  storedValue,
+  type CompiledAggregation,
+  type PrevState,
+  type SourceFields,
+} from "./aggregations.ts";
+import type { Scope } from "./scope.ts";
+
+const CLOCK_ID = "clock";
+const compiledBySchema = new WeakMap<object, CompiledAggregation[]>();
+
+function compiledFor(scope: Scope): CompiledAggregation[] {
+  let compiled = compiledBySchema.get(scope.schema);
+  if (!compiled) {
+    compiled = scope.schema.aggregations.map((def) => {
+      const declared = scope.schema.entityFieldTypes[def.source] ?? {};
+      const fields: SourceFields = {
+        kinds: Object.fromEntries(Object.entries(declared).map(([field, type]) => [field, type.kind])),
+        refs: new Set(scope.schema.entityRefFields[def.source] ?? []),
+      };
+      return compileAggregation(def, fields);
+    });
+    compiledBySchema.set(scope.schema, compiled);
+  }
+  return compiled;
+}
+
+function table(scope: Scope, entity: string) {
+  const accessor = scope.schema.entityAccessors[entity] ?? entity;
+  const found = scope.context[accessor];
+  if (!found) throw new Error(`Envio Subgraph has no table for the aggregation entity ${entity}.`);
+  return found;
+}
+
+/**
+ * graph-node rolls up when a block's time passes the end of the bucket the
+ * previous block's time was in (`Layout::rollup`): only that first bucket can
+ * hold rows, since nothing was written in between. The last block time lives in
+ * an entity, so a restart or a reorg rollback finds it where it left it.
+ */
+export function advanceRollup(scope: Scope, now: bigint): void {
+  if (scope.mode !== "handler" || scope.schema.aggregations.length === 0) return;
+  const state = scope.context[ROLLUP_STATE_ENTITY];
+  const stored = state.getSync(CLOCK_ID);
+  // Preload drops writes, so every handler there reads the clock the last batch
+  // left and would roll that bucket up again; the execute pass rolls up once.
+  if (scope.context.isPreload) return;
+  const last: bigint | null = stored ? BigInt(JSON.parse(stored.payload).t) : null;
+  if (last !== null && now <= last) return;
+  if (last !== null) {
+    for (const compiled of compiledFor(scope)) {
+      for (const interval of compiled.def.intervals) {
+        const [bucket] = bucketsBetween(last, now, interval);
+        if (bucket) rollup(scope, compiled, interval, bucket.start, bucket.end);
+      }
+    }
+  }
+  state.set({ id: CLOCK_ID, aggregation: CLOCK_ID, payload: JSON.stringify({ t: now.toString() }) });
+}
+
+function rollup(scope: Scope, compiled: CompiledAggregation, interval: string, start: bigint, end: bigint) {
+  const { def } = compiled;
+  const source = table(scope, def.source);
+  const rows = source.getWhereSync({
+    timestamp: { _gte: new Date(Number(start / 1000n)), _lt: new Date(Number(end / 1000n)) },
+  });
+  if (rows.length === 0) return;
+  const state = scope.context[ROLLUP_STATE_ENTITY];
+  const group = `${def.name}|${interval}`;
+  const stateId = (dimKey: string) => `prev|${group}|${dimKey}`;
+  const hasCumulative = def.aggregates.some((aggregate) => aggregate.cumulative);
+  // Every group's carried values in one read: read one group at a time, the
+  // sync bridge would replay the handler once per group in the bucket.
+  const carried = new Map<string, PrevState>();
+  if (hasCumulative) {
+    for (const row of state.getWhereSync({ aggregation: { _eq: group } })) {
+      carried.set(row.id, JSON.parse(row.payload) as PrevState);
+    }
+  }
+  const target = table(scope, aggregationEntity(def.name, interval));
+  const timestamp = new Date(Number(start / 1000n));
+  for (const bucket of rollupBucket(compiled, rows, (dimKey) => carried.get(stateId(dimKey)) ?? null)) {
+    const row: Record<string, unknown> = { id: bucket.id.toString(), timestamp, ...bucket.dimensions };
+    for (const aggregate of def.aggregates) {
+      row[aggregate.column ?? aggregate.field] = storedValue(bucket.values[aggregate.field], aggregate.kind);
+    }
+    target.set(row);
+    if (hasCumulative) {
+      state.set({ id: stateId(bucket.dimKey), aggregation: group, payload: JSON.stringify(bucket.next) });
+    }
+  }
+}
+
+/**
+ * graph-node's `SeqGenerator::id` for an Int8: the block number in the upper 32
+ * bits and the block's id sequence below, shared by its handlers in the order
+ * they run.
+ */
+export function timeseriesId(blockNumber: number, sequence: number): string {
+  return ((BigInt(blockNumber) << 32n) + BigInt(sequence)).toString();
+}
+
+export { INTERVAL_MICROS };