envio 3.15.0 → 3.16.0-subgraph

package/src/SubgraphLog.res

@@ -0,0 +1,19 @@
+// Every line the subgraph runtime logs, and every line a mapping logs through
+// graph-ts (`uinfo`, `uwarn`, ...), as it is logged. A test indexer's own logger
+// is silent, so this is how a test reads them. Compiled to JavaScript so a test
+// can import it from an installed envio, where TypeScript isn't stripped.
+
+type listener = (string, string) => unit
+
+let listeners: array<listener> = []
+
+let onLog = (listener: listener) => {
+  listeners->Array.push(listener)
+  () =>
+    switch listeners->Array.indexOf(listener) {
+    | -1 => ()
+    | index => listeners->Array.splice(~start=index, ~remove=1, ~insert=[])
+    }
+}
+
+let notify = (level, message) => listeners->Array.forEach(listener => listener(level, message))

package/src/SubgraphLog.res.mjs

@@ -0,0 +1,26 @@
+// Generated by ReScript, PLEASE EDIT WITH CARE
+
+
+let listeners = [];
+
+function onLog(listener) {
+  listeners.push(listener);
+  return () => {
+    let index = listeners.indexOf(listener);
+    if (index !== -1) {
+      listeners.splice(index, 1);
+      return;
+    }
+  };
+}
+
+function notify(level, message) {
+  listeners.forEach(listener => listener(level, message));
+}
+
+export {
+  listeners,
+  onLog,
+  notify,
+}
+/* No side effect */

package/src/Supervisor.res

@@ -374,74 +374,93 @@ let releaseIfAtHead = group =>
     )
   }
 
-// The run's chains as an unsplit indexer reports them before it has fetched
-// anything: at their configured blocks, with nothing indexed. A display draws
-// these for a worker it hasn't heard from yet.
-let configuredChains = (config: Config.t): array<Metrics.chainMetrics> =>
+// The run's chains as a worker reports them before it has fetched anything:
+// where `ChainState.makeFromDbState` resumes them. A display draws these for a
+// worker it hasn't heard from yet, and its first sight of a chain is what it
+// measures the sync pace from, so a resumed chain drawn from its start block
+// would pass the earlier runs' progress off as this one's.
+let resumedChains = (config: Config.t, ~resumed: array<Persistence.initialChainState>): array<
+  Metrics.chainMetrics,
+> =>
   config.chainMap
   ->ChainMap.values
   ->Array.map((chain): Metrics.chainMetrics => {
-    chainId: chain.id,
-    poweredByHyperSync: switch chain.sourceConfig {
-    | EvmSourceConfig({hypersync}) => hypersync->Option.isSome
-    | FuelSourceConfig(_) | SvmSourceConfig(_) => true
-    | SimulateSourceConfig(_) | CustomSources(_) => false
-    },
-    firstEventBlockNumber: None,
-    latestProcessedBlock: None,
-    timestampCaughtUpToHeadOrEndblock: None,
-    numEventsProcessed: 0.,
-    latestFetchedBlockNumber: 0,
-    knownHeight: 0,
-    numBatchesFetched: 0,
-    // A chain resolves `start_block: latest` against its own head as it starts,
-    // which is a worker's to do and no supervisor's to guess.
-    startBlock: switch chain.startBlock {
-    | Block(block) => block
-    | Latest => 0
-    },
-    endBlock: chain.endBlock,
-    numAddresses: 0,
-    addressesByContract: [],
-    isReady: false,
-    sourceBlockNumber: 0,
-    progressBlockNumber: -1,
-    progressLatencyMs: None,
-    progressBlockTime: None,
-    concurrency: 0,
-    partitionsCount: 0,
-    bufferSize: 0,
-    bufferBlockNumber: -1,
-    idleSeconds: 0.,
-    waitingForNewBlockSeconds: 0.,
-    queryingSeconds: 0.,
-    blockRangeFetchSeconds: 0.,
-    blockRangeParseSeconds: 0.,
-    blockRangeFetchCount: 0.,
-    blockRangeFetchedEvents: 0.,
-    blockRangeFetchedBlocks: 0.,
-    reorgCount: 0,
-    reorgDetectedBlock: None,
-    rollbackTargetBlock: None,
-    rateLimitTimeMs: 0.,
-    rateLimitResetInMs: None,
+    let row = resumed->Array.find(row => row.id == chain.id)
+    let startBlock = row->Option.mapOr(chain->Config.startBlockOrZero, row => row.startBlock)
+    let endBlock = row->Option.mapOr(chain.endBlock, row => row.endBlock)
+    let progressBlockNumber = switch row {
+    | Some({progressBlockNumber}) if progressBlockNumber >= 0 => progressBlockNumber
+    | _ => startBlock - 1
+    }
+    let sourceBlockNumber = row->Option.mapOr(0, row => row.sourceBlockNumber)
+    let timestampCaughtUpToHeadOrEndblock =
+      row->Option.flatMap(row => row.timestampCaughtUpToHeadOrEndblock)
+    let metrics: Metrics.chainMetrics = {
+      chainId: chain.id,
+      poweredByHyperSync: switch chain.sourceConfig {
+      | EvmSourceConfig({hypersync}) => hypersync->Option.isSome
+      | FuelSourceConfig(_) | SvmSourceConfig(_) => true
+      | SimulateSourceConfig(_) | CustomSources(_) => false
+      },
+      firstEventBlockNumber: row->Option.flatMap(row => row.firstEventBlockNumber),
+      latestProcessedBlock: progressBlockNumber === -1 ? None : Some(progressBlockNumber),
+      timestampCaughtUpToHeadOrEndblock,
+      numEventsProcessed: row->Option.mapOr(0., row => row.numEventsProcessed),
+      latestFetchedBlockNumber: Pervasives.max(progressBlockNumber, 0),
+      knownHeight: sourceBlockNumber,
+      numBatchesFetched: 0,
+      startBlock,
+      endBlock,
+      numAddresses: 0,
+      addressesByContract: [],
+      isReady: timestampCaughtUpToHeadOrEndblock->Option.isSome,
+      sourceBlockNumber,
+      progressBlockNumber,
+      progressLatencyMs: None,
+      progressBlockTime: row->Option.flatMap(row => row.progressBlockTime),
+      concurrency: 0,
+      partitionsCount: 0,
+      bufferSize: 0,
+      bufferBlockNumber: progressBlockNumber,
+      idleSeconds: 0.,
+      waitingForNewBlockSeconds: 0.,
+      queryingSeconds: 0.,
+      blockRangeFetchSeconds: 0.,
+      blockRangeParseSeconds: 0.,
+      blockRangeFetchCount: 0.,
+      blockRangeFetchedEvents: 0.,
+      blockRangeFetchedBlocks: 0.,
+      reorgCount: 0,
+      reorgDetectedBlock: None,
+      rollbackTargetBlock: None,
+      rateLimitTimeMs: 0.,
+      rateLimitResetInMs: None,
+    }
+    switch (metrics->Metrics.hasProcessedToEndblock, endBlock) {
+    | (true, Some(endBlock)) => {...metrics, knownHeight: endBlock}
+    | _ => metrics
+    }
   })
 
 // Per worker rather than all or nothing: a worker done within its first report
 // interval is heard from only as it exits, and a display holding just its
 // siblings' finished chains would read as synced while it still indexes.
-let displayedSnapshots = (running: array<running>, ~config, ~empty: Metrics.t) => {
-  let configured = configuredChains(config)
+let displayedSnapshots = (
+  running: array<running>,
+  ~resumedChains: array<Metrics.chainMetrics>,
+  ~empty: Metrics.t,
+) =>
   running->Array.map(r =>
     switch r.snapshot {
     | Some(snapshot) => snapshot
     | None => {
         ...empty,
-        chains: configured->Array.filter(chain => r.worker.chainIds->Array.includes(chain.chainId)),
+        chains: resumedChains->Array.filter(chain =>
+          r.worker.chainIds->Array.includes(chain.chainId)
+        ),
       }
     }
   )
-}
 
 // Runs the group: creates the schema for every chain, forks a worker per plan
 // entry, and serves the run's metrics, console and display from what they
@@ -481,10 +500,9 @@ let run = async (~config: Config.t, ~logger, ~workers: array<worker>, ~reset) =>
   let shouldUseTui = Tui.shouldUse()
   // A run that resumed with every chain already caught up owes nobody a wait:
   // its workers start realtime and there is no barrier to open.
+  let resumed = (persistence->Persistence.getInitializedState).chains
   let holdRealtime =
-    (persistence->Persistence.getInitializedState).chains->Array.some(chain =>
-      chain.timestampCaughtUpToHeadOrEndblock->Option.isNone
-    )
+    resumed->Array.some(chain => chain.timestampCaughtUpToHeadOrEndblock->Option.isNone)
   let group = {running: [], stopping: false, holdingRealtime: holdRealtime}
   group.running =
     workers->Array.mapWithIndex((worker, workerIndex) =>
@@ -534,8 +552,9 @@ let run = async (~config: Config.t, ~logger, ~workers: array<worker>, ~reset) =>
   )
 
   if shouldUseTui {
+    let resumedChains = resumedChains(config, ~resumed)
     Tui.start(~config, ~logger, ~getMetrics=() =>
-      group.running->displayedSnapshots(~config, ~empty=[]->merge)->merge
+      group.running->displayedSnapshots(~resumedChains, ~empty=[]->merge)->merge
     )
   }
 

package/src/Supervisor.res.mjs

@@ -22,6 +22,7 @@ import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js";
 import * as Child_process from "child_process";
 import * as Stdlib_JsError from "@rescript/runtime/lib/es6/Stdlib_JsError.js";
 import * as CrossChainState from "./CrossChainState.res.mjs";
+import * as Primitive_object from "@rescript/runtime/lib/es6/Primitive_object.js";
 import * as Primitive_option from "@rescript/runtime/lib/es6/Primitive_option.js";
 import * as S$RescriptSchema from "rescript-schema/src/S.res.mjs";
 
@@ -313,8 +314,20 @@ function releaseIfAtHead(group) {
   }
 }
 
-function configuredChains(config) {
+function resumedChains(config, resumed) {
   return ChainMap.values(config.chainMap).map(chain => {
+    let row = resumed.find(row => Primitive_object.equal(row.id, chain.id));
+    let startBlock = Stdlib_Option.mapOr(row, Config.startBlockOrZero(chain), row => row.startBlock);
+    let endBlock = Stdlib_Option.mapOr(row, chain.endBlock, row => row.endBlock);
+    let progressBlockNumber;
+    if (row !== undefined) {
+      let progressBlockNumber$1 = row.progressBlockNumber;
+      progressBlockNumber = progressBlockNumber$1 >= 0 ? progressBlockNumber$1 : startBlock - 1 | 0;
+    } else {
+      progressBlockNumber = startBlock - 1 | 0;
+    }
+    let sourceBlockNumber = Stdlib_Option.mapOr(row, 0, row => row.sourceBlockNumber);
+    let timestampCaughtUpToHeadOrEndblock = Stdlib_Option.flatMap(row, row => row.timestampCaughtUpToHeadOrEndblock);
     let match = chain.sourceConfig;
     let tmp;
     switch (match.TAG) {
@@ -330,32 +343,37 @@ function configuredChains(config) {
         tmp = false;
         break;
     }
-    let block = chain.startBlock;
-    let tmp$1;
-    tmp$1 = block === "latest" ? 0 : block;
-    return {
-      chainId: chain.id,
+    let metrics_chainId = chain.id;
+    let metrics_firstEventBlockNumber = Stdlib_Option.flatMap(row, row => row.firstEventBlockNumber);
+    let metrics_latestProcessedBlock = progressBlockNumber === -1 ? undefined : progressBlockNumber;
+    let metrics_numEventsProcessed = Stdlib_Option.mapOr(row, 0, row => row.numEventsProcessed);
+    let metrics_latestFetchedBlockNumber = Primitive_int.max(progressBlockNumber, 0);
+    let metrics_addressesByContract = [];
+    let metrics_isReady = Stdlib_Option.isSome(timestampCaughtUpToHeadOrEndblock);
+    let metrics_progressBlockTime = Stdlib_Option.flatMap(row, row => row.progressBlockTime);
+    let metrics = {
+      chainId: metrics_chainId,
       poweredByHyperSync: tmp,
-      firstEventBlockNumber: undefined,
-      latestProcessedBlock: undefined,
-      timestampCaughtUpToHeadOrEndblock: undefined,
-      numEventsProcessed: 0,
-      latestFetchedBlockNumber: 0,
-      knownHeight: 0,
+      firstEventBlockNumber: metrics_firstEventBlockNumber,
+      latestProcessedBlock: metrics_latestProcessedBlock,
+      timestampCaughtUpToHeadOrEndblock: timestampCaughtUpToHeadOrEndblock,
+      numEventsProcessed: metrics_numEventsProcessed,
+      latestFetchedBlockNumber: metrics_latestFetchedBlockNumber,
+      knownHeight: sourceBlockNumber,
       numBatchesFetched: 0,
-      startBlock: tmp$1,
-      endBlock: chain.endBlock,
+      startBlock: startBlock,
+      endBlock: endBlock,
       numAddresses: 0,
-      addressesByContract: [],
-      isReady: false,
-      sourceBlockNumber: 0,
-      progressBlockNumber: -1,
+      addressesByContract: metrics_addressesByContract,
+      isReady: metrics_isReady,
+      sourceBlockNumber: sourceBlockNumber,
+      progressBlockNumber: progressBlockNumber,
       progressLatencyMs: undefined,
-      progressBlockTime: undefined,
+      progressBlockTime: metrics_progressBlockTime,
       concurrency: 0,
       partitionsCount: 0,
       bufferSize: 0,
-      bufferBlockNumber: -1,
+      bufferBlockNumber: progressBlockNumber,
       idleSeconds: 0,
       waitingForNewBlockSeconds: 0,
       queryingSeconds: 0,
@@ -370,18 +388,27 @@ function configuredChains(config) {
       rateLimitTimeMs: 0,
       rateLimitResetInMs: undefined
     };
+    let match$1 = Metrics.hasProcessedToEndblock(metrics);
+    if (!match$1) {
+      return metrics;
+    }
+    if (endBlock === undefined) {
+      return metrics;
+    }
+    let newrecord = {...metrics};
+    newrecord.knownHeight = endBlock;
+    return newrecord;
   });
 }
 
-function displayedSnapshots(running, config, empty) {
-  let configured = configuredChains(config);
+function displayedSnapshots(running, resumedChains, empty) {
   return running.map(r => {
     let snapshot = r.snapshot;
     if (snapshot !== undefined) {
       return snapshot;
     }
     let newrecord = {...empty};
-    newrecord.chains = configured.filter(chain => r.worker.chainIds.includes(chain.chainId));
+    newrecord.chains = resumedChains.filter(chain => r.worker.chainIds.includes(chain.chainId));
     return newrecord;
   });
 }
@@ -402,7 +429,8 @@ async function run(config, logger, workers, reset) {
     maxConnections: Env.Db.maxConnections
   });
   let shouldUseTui = Tui.shouldUse(undefined, undefined);
-  let holdRealtime = Persistence.getInitializedState(persistence).chains.some(chain => Stdlib_Option.isNone(chain.timestampCaughtUpToHeadOrEndblock));
+  let resumed = Persistence.getInitializedState(persistence).chains;
+  let holdRealtime = resumed.some(chain => Stdlib_Option.isNone(chain.timestampCaughtUpToHeadOrEndblock));
   let group = {
     running: [],
     stopping: false,
@@ -421,7 +449,8 @@ async function run(config, logger, workers, reset) {
     runtime
   ]))), config.isDev);
   if (shouldUseTui) {
-    Tui.start(config, logger, () => merge(displayedSnapshots(group.running, config, merge([]))));
+    let resumedChains$1 = resumedChains(config, resumed);
+    Tui.start(config, logger, () => merge(displayedSnapshots(group.running, resumedChains$1, merge([]))));
   }
   process.on("SIGTERM", () => stop(group));
   process.on("SIGINT", () => stop(group));
@@ -465,7 +494,7 @@ export {
   awaitExit,
   isRunAtHead,
   releaseIfAtHead,
-  configuredChains,
+  resumedChains,
   displayedSnapshots,
   run,
 }

package/src/SyncWrites.res

@@ -0,0 +1,107 @@
+// The writes one round of a synchronous handler has made. A round that
+// suspends is replayed from the top, so its writes are held here rather than
+// applied: the replay has to see the store exactly as the first round did, or a
+// read-modify-write lands once per round.
+
+// A Map rather than a dict: every round makes a fresh one keyed by ids no
+// other round shares, and a dict would give each of them its own hidden class.
+type table = {
+  entityConfig: Internal.entityConfig,
+  scope: Internal.chainScope,
+  changes: Map.t<string, Change.t<Internal.entity>>,
+}
+
+// A handler runs on one chain, so an entity's scope is fixed for the whole
+// round and the entity name is enough to key its table.
+type t = dict<table>
+
+let make = (): t => Dict.make()
+
+let record = (writes: t, ~entityConfig: Internal.entityConfig, ~scope, change) => {
+  let table = switch writes->Utils.Dict.dangerouslyGetNonOption(entityConfig.name) {
+  | Some(table) => table
+  | None =>
+    let table = {entityConfig, scope, changes: Map.make()}
+    writes->Dict.set(entityConfig.name, table)
+    table
+  }
+  table.changes->Map.set(entityConfig.table.entityIdToKey(change->Change.getEntityId), change)
+}
+
+let changesOf = (writes: t, ~entityConfig: Internal.entityConfig) =>
+  switch writes->Utils.Dict.dangerouslyGetNonOption(entityConfig.name) {
+  | Some({changes}) => Some(changes)
+  | None => None
+  }
+
+// `Some(None)` is a delete this round made: the entity is known not to exist.
+let getById = (writes: t, ~entityConfig: Internal.entityConfig, ~entityId: string) =>
+  switch writes->changesOf(~entityConfig) {
+  | Some(changes) =>
+    changes
+    ->Map.get(entityConfig.table.entityIdToKey(entityId->EntityId.unsafeOfString))
+    ->Option.map(InMemoryTable.Entity.mapChangeToEntity)
+  | None => None
+  }
+
+// What the store answered, as it reads once this round's writes land: a row
+// the round rewrote is re-tested against the filter, and one it created joins.
+let overlayFilter = (
+  writes: t,
+  entities: array<Internal.entity>,
+  ~entityConfig: Internal.entityConfig,
+  ~filter: EntityFilter.t,
+) =>
+  switch writes->changesOf(~entityConfig) {
+  | None => entities
+  | Some(changes) =>
+    let matches = filter->EntityFilter.makeMatcher(~table=entityConfig.table)
+    let untouched =
+      entities->Array.filter(entity =>
+        !(
+          changes->Map.has(
+            entityConfig.table.entityIdToKey(
+              (entity->(Utils.magic: Internal.entity => {"id": EntityId.t}))["id"],
+            ),
+          )
+        )
+      )
+    let written = []
+    changes->Map.forEach(change =>
+      switch change->InMemoryTable.Entity.mapChangeToEntity {
+      | Some(entity) if matches(entity) => written->Array.push(entity)->ignore
+      | _ => ()
+      }
+    )
+    untouched->Array.concat(written)
+  }
+
+// How a change lands in the in-memory store, whether written directly or
+// committed at the end of a round.
+let write = (indexerState, ~entityConfig: Internal.entityConfig, ~scope, change) => {
+  switch change {
+  | Change.Set({entityId}) =>
+    PreloadLookahead.onSet(
+      ~entityName=entityConfig.name,
+      ~scope,
+      ~entityId=entityConfig.table.entityIdToKey(entityId),
+    )
+  | Delete(_) => ()
+  }
+  indexerState
+  ->InMemoryStore.getInMemTable(~entityConfig, ~scope)
+  ->InMemoryTable.Entity.set(
+    ~committedCheckpointId=indexerState->IndexerState.committedCheckpointIdFor(~scope),
+    change,
+  )
+}
+
+let forEachChange = (writes: t, fn) =>
+  writes->Utils.Dict.forEach(({entityConfig, scope, changes}) =>
+    changes->Map.forEach(change => fn(~entityConfig, ~scope, change))
+  )
+
+let commit = (writes: t, ~indexerState) =>
+  writes->Utils.Dict.forEach(({entityConfig, scope, changes}) =>
+    changes->Map.forEach(change => indexerState->write(~entityConfig, ~scope, change))
+  )

package/src/SyncWrites.res.mjs

@@ -0,0 +1,98 @@
+// Generated by ReScript, PLEASE EDIT WITH CARE
+
+import * as Utils from "./Utils.res.mjs";
+import * as EntityFilter from "./db/EntityFilter.res.mjs";
+import * as IndexerState from "./IndexerState.res.mjs";
+import * as InMemoryStore from "./InMemoryStore.res.mjs";
+import * as InMemoryTable from "./InMemoryTable.res.mjs";
+import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js";
+import * as PreloadLookahead from "./PreloadLookahead.res.mjs";
+
+function make() {
+  return {};
+}
+
+function record(writes, entityConfig, scope, change) {
+  let table = writes[entityConfig.name];
+  let table$1;
+  if (table !== undefined) {
+    table$1 = table;
+  } else {
+    let table_changes = new Map();
+    let table$2 = {
+      entityConfig: entityConfig,
+      scope: scope,
+      changes: table_changes
+    };
+    writes[entityConfig.name] = table$2;
+    table$1 = table$2;
+  }
+  table$1.changes.set(entityConfig.table.entityIdToKey(change.entityId), change);
+}
+
+function changesOf(writes, entityConfig) {
+  let match = writes[entityConfig.name];
+  if (match !== undefined) {
+    return match.changes;
+  }
+}
+
+function getById(writes, entityConfig, entityId) {
+  let changes = changesOf(writes, entityConfig);
+  if (changes !== undefined) {
+    return Stdlib_Option.map(changes.get(entityConfig.table.entityIdToKey(entityId)), InMemoryTable.Entity.mapChangeToEntity);
+  }
+}
+
+function overlayFilter(writes, entities, entityConfig, filter) {
+  let changes = changesOf(writes, entityConfig);
+  if (changes === undefined) {
+    return entities;
+  }
+  let matches = EntityFilter.makeMatcher(filter, entityConfig.table);
+  let untouched = entities.filter(entity => !changes.has(entityConfig.table.entityIdToKey(entity.id)));
+  let written = [];
+  changes.forEach(change => {
+    let entity = InMemoryTable.Entity.mapChangeToEntity(change);
+    if (entity !== undefined && matches(entity)) {
+      written.push(entity);
+      return;
+    }
+  });
+  return untouched.concat(written);
+}
+
+function write(indexerState, entityConfig, scope, change) {
+  if (change.type === "SET") {
+    PreloadLookahead.onSet(entityConfig.name, scope, entityConfig.table.entityIdToKey(change.entityId));
+  }
+  InMemoryTable.Entity.set(InMemoryStore.getInMemTable(indexerState, entityConfig, scope), IndexerState.committedCheckpointIdFor(indexerState, scope), change);
+}
+
+function forEachChange(writes, fn) {
+  Utils.Dict.forEach(writes, param => {
+    let scope = param.scope;
+    let entityConfig = param.entityConfig;
+    param.changes.forEach(change => fn(entityConfig, scope, change));
+  });
+}
+
+function commit(writes, indexerState) {
+  Utils.Dict.forEach(writes, param => {
+    let scope = param.scope;
+    let entityConfig = param.entityConfig;
+    param.changes.forEach(change => write(indexerState, entityConfig, scope, change));
+  });
+}
+
+export {
+  make,
+  record,
+  changesOf,
+  getById,
+  overlayFilter,
+  write,
+  forEachChange,
+  commit,
+}
+/* Utils Not a pure module */