envio 3.12.1 → 3.13.0-rc.0

package/src/Supervisor.res.mjs

@@ -0,0 +1,455 @@
+// Generated by ReScript, PLEASE EDIT WITH CARE
+
+import * as Env from "./Env.res.mjs";
+import * as Tui from "./tui/Tui.res.mjs";
+import * as Utils from "./Utils.res.mjs";
+import * as Config from "./Config.res.mjs";
+import * as Server from "./Server.res.mjs";
+import * as Worker from "./Worker.res.mjs";
+import * as ChainId from "./ChainId.res.mjs";
+import * as Logging from "./Logging.res.mjs";
+import * as Metrics from "./Metrics.res.mjs";
+import * as Process from "process";
+import * as ChainMap from "./ChainMap.res.mjs";
+import * as PgStorage from "./PgStorage.res.mjs";
+import * as Performance from "./bindings/Performance.res.mjs";
+import * as Persistence from "./Persistence.res.mjs";
+import * as Stdlib_Array from "@rescript/runtime/lib/es6/Stdlib_Array.js";
+import * as Primitive_int from "@rescript/runtime/lib/es6/Primitive_int.js";
+import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js";
+import * as Child_process from "child_process";
+import * as Stdlib_JsError from "@rescript/runtime/lib/es6/Stdlib_JsError.js";
+import * as CrossChainState from "./CrossChainState.res.mjs";
+import * as Primitive_option from "@rescript/runtime/lib/es6/Primitive_option.js";
+import * as S$RescriptSchema from "rescript-schema/src/S.res.mjs";
+
+function plan(chainIds, maxConnections) {
+  let workerCount = Stdlib_Array.reduce([
+    chainIds.length,
+    maxConnections / 2 | 0,
+    4
+  ], 4, Primitive_int.min);
+  if (workerCount < 2) {
+    return;
+  }
+  let evenShare = Primitive_int.div(maxConnections, workerCount);
+  let remainder = Primitive_int.mod_(maxConnections, workerCount);
+  return Stdlib_Array.fromInitializer(workerCount, workerIndex => ({
+    chainIds: chainIds.filter((param, dealIndex) => {
+      let position = Primitive_int.mod_(dealIndex, workerCount);
+      let isReversePass = Primitive_int.div(dealIndex, workerCount) % 2 === 1;
+      return (
+        isReversePass ? (workerCount - 1 | 0) - position | 0 : position
+      ) === workerIndex;
+    }),
+    maxConnections: evenShare + (
+      workerIndex < remainder ? 1 : 0
+    ) | 0
+  }));
+}
+
+function planForRun(config, maxConnectionsOpt) {
+  let maxConnections = maxConnectionsOpt !== undefined ? maxConnectionsOpt : Env.Db.maxConnections;
+  if (config.isolated || !Config.isPerChain(config)) {
+    return;
+  } else {
+    return plan(ChainMap.values(config.chainMap).map(chain => chain.id), maxConnections);
+  }
+}
+
+function name(worker) {
+  return worker.chainIds.map(ChainId.toString).join(";");
+}
+
+function label(worker) {
+  return `[chain ` + name(worker) + `]`;
+}
+
+function logFilePath(workerIndex, pathOpt) {
+  let path = pathOpt !== undefined ? pathOpt : Env.logFilePath;
+  let suffix = `.worker-` + workerIndex.toString();
+  let dot = path.lastIndexOf(".");
+  if (dot > path.lastIndexOf("/")) {
+    return path.slice(0, dot) + suffix + path.slice(dot, path.length);
+  } else {
+    return path + suffix;
+  }
+}
+
+function readLines(onLine) {
+  let pending = {
+    contents: ""
+  };
+  let read = chunk => {
+    let parts = (pending.contents + chunk).split("\n");
+    pending.contents = Stdlib_Option.getOr(parts.pop(), "");
+    parts.forEach(onLine);
+  };
+  let flush = () => {
+    let line = pending.contents;
+    if (line === "") {
+      return;
+    }
+    pending.contents = "";
+    onLine(line);
+  };
+  return [
+    read,
+    flush
+  ];
+}
+
+function memoryBudgets(workerCount) {
+  return [
+    [
+      "ENVIO_INDEXING_MAX_BUFFER_SIZE",
+      CrossChainState.calculateTargetBufferSize()
+    ],
+    [
+      "ENVIO_IN_MEMORY_OBJECTS_TARGET",
+      Env.inMemoryObjectsTarget | 0
+    ]
+  ].map(param => [
+    param[0],
+    Primitive_int.max(1, Primitive_int.div(param[1], workerCount)).toString()
+  ]);
+}
+
+function fork(worker, workerIndex, workerCount, holdRealtime, isDev, entryPathOpt, pipeOutputOpt, onOutputOpt, onErrorOutputOpt, onSnapshotOpt) {
+  let entryPath = entryPathOpt !== undefined ? entryPathOpt : process.argv[1];
+  let pipeOutput = pipeOutputOpt !== undefined ? pipeOutputOpt : false;
+  let onOutput = onOutputOpt !== undefined ? onOutputOpt : prim => {
+      console.log(prim);
+    };
+  let onErrorOutput = onErrorOutputOpt !== undefined ? onErrorOutputOpt : prim => {
+      console.error(prim);
+    };
+  let onSnapshot = onSnapshotOpt !== undefined ? onSnapshotOpt : () => {};
+  let env = Object.assign({}, Process.env);
+  env[Worker.envVar] = S$RescriptSchema.reverseConvertToJsonStringOrThrow({
+    chainIds: worker.chainIds,
+    holdRealtime: holdRealtime,
+    isDev: isDev
+  }, Worker.configSchema, undefined);
+  env["ENVIO_PG_MAX_CONNECTIONS"] = worker.maxConnections.toString();
+  memoryBudgets(workerCount).forEach(param => {
+    env[param[0]] = param[1];
+  });
+  env["LOG_FILE"] = logFilePath(workerIndex, undefined);
+  if (pipeOutput && Stdlib_Option.getOr((process.stdout.isTTY == null) ? undefined : Primitive_option.some(process.stdout.isTTY), false)) {
+    env["FORCE_COLOR"] = "1";
+  }
+  let child = Child_process.fork(entryPath, [], {
+    env: env,
+    serialization: "advanced",
+    stdio: pipeOutput ? [
+        "inherit",
+        "pipe",
+        "pipe",
+        "ipc"
+      ] : [
+        "inherit",
+        "inherit",
+        "inherit",
+        "ipc"
+      ]
+  });
+  if (pipeOutput) {
+    [
+      [
+        child.stdout,
+        onOutput
+      ],
+      [
+        child.stderr,
+        onErrorOutput
+      ]
+    ].forEach(param => {
+      let stream = param[0];
+      if (stream === null) {
+        return;
+      }
+      let match = readLines(param[1]);
+      stream.setEncoding("utf8");
+      stream.on("data", match[0]);
+      stream.on("end", match[1]);
+    });
+  }
+  let running = {
+    worker: worker,
+    child: child,
+    snapshot: undefined,
+    runtime: undefined,
+    settled: false
+  };
+  child.on("message", message => {
+    running.snapshot = message.metrics;
+    running.runtime = message.runtime;
+    onSnapshot();
+  });
+  return running;
+}
+
+function stop(group) {
+  group.stopping = true;
+  group.running.forEach(r => {
+    r.child.kill("SIGTERM");
+  });
+}
+
+let inFlight = {
+  contents: undefined
+};
+
+function syncCache(dump) {
+  let dumping = inFlight.contents;
+  if (dumping !== undefined) {
+    return dumping;
+  }
+  let dumping$1 = dump().finally(() => {
+    inFlight.contents = undefined;
+  });
+  inFlight.contents = dumping$1;
+  return dumping$1;
+}
+
+function dumpCache(config) {
+  let storage = PgStorage.makeStorageFromEnv(config, Primitive_option.some(PgStorage.makeClient(1)), undefined, undefined);
+  return storage.dumpEffectCache().finally(() => {
+    storage.close();
+  });
+}
+
+function classifyExit(code, signal, stopping) {
+  if (stopping) {
+    return "Expected";
+  }
+  if (code === 0) {
+    return "Expected";
+  }
+  if (signal === "SIGTERM") {
+    return "Stopping";
+  } else {
+    return "Failed";
+  }
+}
+
+async function awaitExit(group) {
+  let failed = {
+    contents: false
+  };
+  let alive = {
+    contents: group.running.length
+  };
+  await new Promise((resolve, param) => {
+    let onGone = (r, ending) => {
+      if (r.settled) {
+        return;
+      }
+      r.settled = true;
+      switch (ending) {
+        case "Expected" :
+          break;
+        case "Stopping" :
+          stop(group);
+          break;
+        case "Failed" :
+          failed.contents = true;
+          stop(group);
+          break;
+      }
+      alive.contents = alive.contents - 1 | 0;
+      if (alive.contents === 0) {
+        return resolve();
+      }
+    };
+    group.running.forEach(r => {
+      r.child.on("exit", (code, signal) => onGone(r, classifyExit(code, signal, group.stopping)));
+      r.child.on("error", exn => {
+        Logging.errorWithExn(exn, label(r.worker) + ` failed to start`);
+        onGone(r, "Failed");
+      });
+    });
+  });
+  if (failed.contents) {
+    Stdlib_JsError.throwWithMessage("An indexer process exited with a failure. Stopped the others.");
+  }
+  if (group.stopping) {
+    return "Stopped";
+  } else {
+    return "Finished";
+  }
+}
+
+function isRunAtHead(running) {
+  if (Utils.$$Array.notEmpty(running)) {
+    return running.every(r => {
+      if (r.child.connected) {
+        return Stdlib_Option.mapOr(r.snapshot, false, snapshot => snapshot.hasArrivedAtHead);
+      } else {
+        return false;
+      }
+    });
+  } else {
+    return false;
+  }
+}
+
+function releaseIfAtHead(group) {
+  if (group.holdingRealtime && !group.stopping && isRunAtHead(group.running)) {
+    group.holdingRealtime = false;
+    group.running.forEach(r => {
+      r.child.send("release-realtime");
+    });
+    return;
+  }
+}
+
+function configuredChains(config) {
+  return ChainMap.values(config.chainMap).map(chain => {
+    let match = chain.sourceConfig;
+    let tmp;
+    switch (match.TAG) {
+      case "EvmSourceConfig" :
+        tmp = Stdlib_Option.isSome(match.hypersync);
+        break;
+      case "FuelSourceConfig" :
+      case "SvmSourceConfig" :
+        tmp = true;
+        break;
+      case "SimulateSourceConfig" :
+      case "CustomSources" :
+        tmp = false;
+        break;
+    }
+    let block = chain.startBlock;
+    let tmp$1;
+    tmp$1 = block === "latest" ? 0 : block;
+    return {
+      chainId: chain.id,
+      poweredByHyperSync: tmp,
+      firstEventBlockNumber: undefined,
+      latestProcessedBlock: undefined,
+      timestampCaughtUpToHeadOrEndblock: undefined,
+      numEventsProcessed: 0,
+      latestFetchedBlockNumber: 0,
+      knownHeight: 0,
+      numBatchesFetched: 0,
+      startBlock: tmp$1,
+      endBlock: chain.endBlock,
+      numAddresses: 0,
+      addressesByContract: [],
+      isReady: false,
+      sourceBlockNumber: 0,
+      progressBlockNumber: -1,
+      progressLatencyMs: undefined,
+      progressBlockTime: undefined,
+      concurrency: 0,
+      partitionsCount: 0,
+      bufferSize: 0,
+      bufferBlockNumber: -1,
+      idleSeconds: 0,
+      waitingForNewBlockSeconds: 0,
+      queryingSeconds: 0,
+      blockRangeFetchSeconds: 0,
+      blockRangeParseSeconds: 0,
+      blockRangeFetchCount: 0,
+      blockRangeFetchedEvents: 0,
+      blockRangeFetchedBlocks: 0,
+      reorgCount: 0,
+      reorgDetectedBlock: undefined,
+      rollbackTargetBlock: undefined,
+      rateLimitTimeMs: 0,
+      rateLimitResetInMs: undefined
+    };
+  });
+}
+
+async function run(config, workers, reset) {
+  let persistence = PgStorage.makePersistenceFromConfig(config, undefined);
+  await Persistence.initForRun(persistence, config, reset, config.isDev, false);
+  await persistence.storage.close();
+  let startTime = new Date();
+  let startTimeRef = Performance.now();
+  Logging.info({
+    msg: "Indexing will be split across multiple processes for faster and more reliable indexing.",
+    chains: ChainMap.values(config.chainMap).length,
+    processes: workers.length,
+    maxConnections: Env.Db.maxConnections
+  });
+  let shouldUseTui = Tui.shouldUse(undefined, undefined);
+  let holdRealtime = Persistence.getInitializedState(persistence).chains.some(chain => Stdlib_Option.isNone(chain.timestampCaughtUpToHeadOrEndblock));
+  let group = {
+    running: [],
+    stopping: false,
+    holdingRealtime: holdRealtime
+  };
+  group.running = workers.map((worker, workerIndex) => fork(worker, workerIndex, workers.length, holdRealtime, config.isDev, undefined, shouldUseTui, undefined, undefined, () => releaseIfAtHead(group)));
+  let reported = () => Stdlib_Array.filterMap(group.running, r => r.snapshot);
+  let merge = snapshots => Metrics.merge(snapshots, startTime, new Date(), Performance.secondsSince(startTimeRef), CrossChainState.calculateTargetBufferSize());
+  Server.startServer(() => {
+    let snapshots = reported();
+    if (snapshots.length !== 0) {
+      return merge(snapshots);
+    }
+  }, Utils.EnvioPackage.value.version, () => syncCache(() => dumpCache(config)), () => Metrics.renderRuntime(Stdlib_Array.filterMap(group.running, r => Stdlib_Option.map(r.runtime, runtime => [
+    `worker="` + name(r.worker) + `"`,
+    runtime
+  ]))), config.isDev);
+  if (shouldUseTui) {
+    Tui.start(config, () => {
+      let snapshots = reported();
+      if (snapshots.length !== 0) {
+        return merge(snapshots);
+      }
+      let newrecord = {...merge([])};
+      newrecord.chains = configuredChains(config);
+      return newrecord;
+    });
+  }
+  process.on("SIGTERM", () => stop(group));
+  process.on("SIGINT", () => stop(group));
+  let outcome = await awaitExit(group);
+  if (outcome === "Finished") {
+    if (shouldUseTui) {
+      process.on("SIGTERM", () => {
+        Process.exit(0);
+      });
+      process.on("SIGINT", () => {
+        Process.exit(0);
+      });
+    } else {
+      Logging.info("Exiting with success");
+      Process.exit(0);
+    }
+    return;
+  }
+  Process.exit(0);
+}
+
+let minConnectionsPerWorker = 2;
+
+let maxWorkers = 4;
+
+export {
+  minConnectionsPerWorker,
+  maxWorkers,
+  plan,
+  planForRun,
+  name,
+  label,
+  logFilePath,
+  readLines,
+  memoryBudgets,
+  fork,
+  stop,
+  syncCache,
+  dumpCache,
+  classifyExit,
+  awaitExit,
+  isRunAtHead,
+  releaseIfAtHead,
+  configuredChains,
+  run,
+}
+/* Env Not a pure module */

package/src/TestIndexer.res

@@ -948,7 +948,7 @@ let createTestIndexer = (): t<'processConfig> => {
                 ~exitAfterFirstEventBlock,
                 ~onError=errHandler => {
                   errHandler->ErrorHandling.log
-                  reject(errHandler.exn->Utils.prettifyExn)
+                  reject(errHandler->ErrorHandling.toExn)
                 },
                 // Caught up: resolve the run instead of exiting the process.
                 ~onExit=() => resolve(),

package/src/TestIndexer.res.mjs

@@ -639,9 +639,9 @@ function createTestIndexer() {
       };
       try {
         await new Promise((resolve, reject) => {
-          let indexerState = IndexerState.makeFromDbState(runConfig, persistence, initialState, registrationsByChainId, undefined, undefined, exitAfterFirstEventBlock, undefined, undefined, errHandler => {
+          let indexerState = IndexerState.makeFromDbState(runConfig, persistence, initialState, registrationsByChainId, undefined, undefined, exitAfterFirstEventBlock, undefined, undefined, undefined, errHandler => {
             ErrorHandling.log(errHandler);
-            reject(Utils.prettifyExn(errHandler.exn));
+            reject(ErrorHandling.toExn(errHandler));
           }, () => resolve());
           indexerStateRef.contents = Primitive_option.some(indexerState);
           IndexerLoop.start(indexerState);

package/src/Worker.res

@@ -0,0 +1,101 @@
+// The worker side of a supervised run: a process the supervisor forked to drive
+// a subset of the chains. It has no server and no TUI of its own — it reports
+// through the IPC channel, and the supervisor is the one operational surface.
+
+// Set by a supervisor in the environment of the workers it forks. Internal:
+// it counts only together with the fork's own channel, so a copy left in a
+// shell starts nothing, and an indexer a user starts themselves takes every
+// path it takes today.
+let envVar = "ENVIO_INTERNAL_WORKER"
+
+// What the supervisor decided about this worker, handed over in the spawn
+// environment rather than over the channel: it is settled before the process
+// starts, and the worker needs it before it can load its own config.
+type config = {
+  chainIds: array<ChainId.t>,
+  // The chains this worker drives may reach the head while chains in another
+  // process are still backfilling, and an indexer goes realtime as a whole or
+  // not at all. Cleared by the supervisor's `ReleaseRealtime`.
+  holdRealtime: bool,
+  // Which command the run was started by. A worker re-parses the project's
+  // files, which say nothing about that, so it can only be told — and a worker
+  // that took a dev run for a plain one would exit at its end block and leave
+  // the console it was still serving with a chain missing.
+  isDev: bool,
+}
+
+let configSchema = S.object((s): config => {
+  chainIds: s.field("chainIds", S.array(ChainId.schema)),
+  holdRealtime: s.fieldOr("holdRealtime", S.bool, false),
+  isDev: s.field("isDev", S.bool),
+})
+
+// Read as this module loads, which is before anything that could catch a bare
+// schema error and say where it came from.
+let detect = (~env: dict<string>, ~hasChannel) =>
+  switch (hasChannel, env->Dict.get(envVar)) {
+  | (true, Some(json)) =>
+    switch json->S.parseJsonStringOrThrow(configSchema) {
+    | config => Some(config)
+    | exception S.Raised(error) =>
+      JsError.throwWithMessage(
+        `Invalid ${envVar}: ${error->S.Error.message}. It is set by an indexer supervisor for the processes it forks, and isn't meant to be set by hand.`,
+      )
+    }
+  | _ => None
+  }
+
+let config = detect(
+  ~env=NodeJs.Process.process.env,
+  ~hasChannel=NodeJs.Process.channel->Nullable.toOption->Option.isSome,
+)
+
+let isEnabled = config->Option.isSome
+
+@tag("kind")
+type parentMessage =
+  // Every chain in the run has reached the head, so this worker may enter the
+  // reorg threshold and switch to realtime with the rest of them.
+  | @as("release-realtime") ReleaseRealtime
+
+@tag("kind")
+type workerMessage =
+  | @as("snapshot") Snapshot({metrics: Metrics.t, runtime: Metrics.runtimeSample})
+
+// How often a worker reports. Matches the TUI's own refresh, so the supervised
+// display moves at the same rate an unsplit run's does.
+%%private(let snapshotIntervalMillis = 500)
+
+// The supervisor is the one that stops a worker, and the one whose absence
+// ends it. A terminal's interrupt reaches the whole group at once, so the
+// worker leaves it to the supervisor, which stops every worker in turn; without
+// that, a worker gone on its own would read as a failure to the supervisor
+// still deciding what the interrupt meant. A supervisor that dies can't tear
+// the group down, so losing the channel is what ends the worker then.
+let bindToSupervisor = () => {
+  NodeJs.Process.onSignal("SIGINT", () => ())
+  NodeJs.Process.onDisconnect(() => {
+    Logging.error("The indexer supervisor is gone. Stopping this chain's process.")
+    NodeJs.process->NodeJs.exitWithCode(Failure)
+  })
+}
+
+%%private(let send = (message: workerMessage) => NodeJs.Process.sendToParent(message)->ignore)
+
+// Reports this process's chains and its own runtime for as long as it runs, so
+// the supervisor can merge every worker's into the one snapshot the run serves,
+// and listens for the one decision the supervisor makes on the run's behalf.
+// Does nothing in a process nobody forked.
+let bindRun = (~getMetrics: unit => Metrics.t, ~onReleaseRealtime: unit => unit) =>
+  if isEnabled {
+    Metrics.startRuntimeCollectors()
+    let _intervalId = setInterval(
+      () => send(Snapshot({metrics: getMetrics(), runtime: Metrics.sampleRuntime()})),
+      snapshotIntervalMillis,
+    )
+    NodeJs.Process.onMessage((message: parentMessage) =>
+      switch message {
+      | ReleaseRealtime => onReleaseRealtime()
+      }
+    )
+  }

package/src/Worker.res.mjs

@@ -0,0 +1,81 @@
+// Generated by ReScript, PLEASE EDIT WITH CARE
+
+import * as ChainId from "./ChainId.res.mjs";
+import * as Logging from "./Logging.res.mjs";
+import * as Metrics from "./Metrics.res.mjs";
+import * as Process from "process";
+import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js";
+import * as Stdlib_JsError from "@rescript/runtime/lib/es6/Stdlib_JsError.js";
+import * as Primitive_option from "@rescript/runtime/lib/es6/Primitive_option.js";
+import * as S$RescriptSchema from "rescript-schema/src/S.res.mjs";
+import * as Primitive_exceptions from "@rescript/runtime/lib/es6/Primitive_exceptions.js";
+
+let envVar = "ENVIO_INTERNAL_WORKER";
+
+let configSchema = S$RescriptSchema.object(s => ({
+  chainIds: s.f("chainIds", S$RescriptSchema.array(ChainId.schema)),
+  holdRealtime: s.fieldOr("holdRealtime", S$RescriptSchema.bool, false),
+  isDev: s.f("isDev", S$RescriptSchema.bool)
+}));
+
+function detect(env, hasChannel) {
+  let match = env[envVar];
+  if (!hasChannel) {
+    return;
+  }
+  if (match === undefined) {
+    return;
+  }
+  let config;
+  try {
+    config = S$RescriptSchema.parseJsonStringOrThrow(match, configSchema);
+  } catch (raw_error) {
+    let error = Primitive_exceptions.internalToException(raw_error);
+    if (error.RE_EXN_ID === S$RescriptSchema.Raised) {
+      return Stdlib_JsError.throwWithMessage(`Invalid ` + envVar + `: ` + S$RescriptSchema.$$Error.message(error._1) + `. It is set by an indexer supervisor for the processes it forks, and isn't meant to be set by hand.`);
+    }
+    throw error;
+  }
+  return config;
+}
+
+let config = detect(Process.env, Stdlib_Option.isSome(Primitive_option.fromNullable(process.channel)));
+
+let isEnabled = Stdlib_Option.isSome(config);
+
+function bindToSupervisor() {
+  process.on("SIGINT", () => {});
+  process.on("disconnect", () => {
+    Logging.error("The indexer supervisor is gone. Stopping this chain's process.");
+    Process.exit(1);
+  });
+}
+
+function bindRun(getMetrics, onReleaseRealtime) {
+  if (isEnabled) {
+    Metrics.startRuntimeCollectors();
+    setInterval(() => {
+      let message_0 = getMetrics();
+      let message_1 = Metrics.sampleRuntime();
+      let message = {
+        kind: "snapshot",
+        metrics: message_0,
+        runtime: message_1
+      };
+      process.send(message);
+    }, 500);
+    process.on("message", message => onReleaseRealtime());
+    return;
+  }
+}
+
+export {
+  envVar,
+  configSchema,
+  detect,
+  config,
+  isEnabled,
+  bindToSupervisor,
+  bindRun,
+}
+/* configSchema Not a pure module */