envio 3.11.0 → 3.12.1

package/src/Metrics.res.mjs

@@ -9,6 +9,15 @@ import * as Stdlib_Array from "@rescript/runtime/lib/es6/Stdlib_Array.js";
 import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js";
 import * as Primitive_option from "@rescript/runtime/lib/es6/Primitive_option.js";
 
+function hasProcessedToEndblock(m) {
+  let endBlock = m.endBlock;
+  if (endBlock !== undefined) {
+    return m.progressBlockNumber >= endBlock;
+  } else {
+    return false;
+  }
+}
+
 function formatValue(value) {
   return (Math.round(value * 1000) / 1000).toString();
 }
@@ -96,16 +105,26 @@ function renderMetrics(b, metrics) {
     ]);
     let existing = byLabels[labels];
     if (existing !== undefined) {
+      let match = existing.responseBlocks;
+      let match$1 = s.responseBlocks;
       byLabels[labels] = {
         source: existing.source,
         chainId: existing.chainId,
         method: existing.method,
         count: existing.count + s.count | 0,
-        seconds: existing.seconds + s.seconds
+        seconds: existing.seconds + s.seconds,
+        responseBlocks: match !== undefined ? (
+            match$1 !== undefined ? match + match$1 | 0 : (
+                match$1 !== undefined ? match$1 : match
+              )
+          ) : (
+            match$1 !== undefined ? match$1 : match
+          ),
+        emptyResponseCount: existing.emptyResponseCount + s.emptyResponseCount | 0
       };
-    } else {
-      byLabels[labels] = s;
+      return;
     }
+    byLabels[labels] = s;
   });
   let sourceRequests = Object.entries(byLabels);
   let byLabels$1 = {};
@@ -176,6 +195,8 @@ function renderMetrics(b, metrics) {
       return s.seconds;
     }
   });
+  seriesOpt(b, "envio_source_response_blocks_total", "The number of blocks a data source returned, summed over its responses. Counted as the source sent them, before the indexer drops the blocks no event needs. Only sources whose responses are measured in blocks report it.", "counter", sourceRequests, s => Stdlib_Option.map(s.responseBlocks, prim => prim));
+  seriesOpt(b, "envio_source_response_empty_total", "The number of responses that came back with no blocks at all — a range the source scanned and matched nothing in. Compare against envio_source_request_total for the share of requests that returned nothing.", "counter", sourceRequests, s => Stdlib_Option.map(s.responseBlocks, param => s.emptyResponseCount));
   if (heightStreamConnects.length !== 0) {
     series(b, "envio_source_height_stream_connects_total", "The number of times a source's height subscription connected. Compare against the disconnects total, which is absent until the first disconnect and counts as zero while it is: one more connect than disconnects means the stream is up, and equal counts mean it is down and the indexer is polling instead. Zero connects means the stream has not come up, which is the normal reading for a chain that is still backfilling: subscriptions are only opened once a chain reaches the head.", "counter", heightStreamConnects, count => count);
   }
@@ -200,6 +221,7 @@ function renderMetrics(b, metrics) {
   single(b, "envio_processing_max_batch_size", "The maximum number of items to process in a single batch.", "gauge", metrics.maxBatchSize);
   series(b, "envio_progress_block", "The block number of the latest block processed and stored in the database.", "gauge", chains, m => m.progressBlockNumber);
   series(b, "envio_progress_events", "The number of events processed and reflected in the database.", "gauge", chains, m => m.numEventsProcessed);
+  seriesOpt(b, "envio_progress_block_time_seconds", "Unix timestamp of the block the chain has processed up to. Subtract it from the scrape time for how far behind chain time the indexer is, which stays honest when the data source itself is behind the chain. Best effort in realtime mode, absent during backfill.", "gauge", chains, m => Stdlib_Option.map(m.progressBlockTime, prim => prim));
   seriesOpt(b, "envio_progress_latency", "The latency in milliseconds between the latest processed event creation and the time it was written to storage.", "gauge", chains, m => Stdlib_Option.map(m.progressLatencyMs, prim => prim));
   let ifCalled = (s, value) => {
     if (s.callCount > 0 || s.activeCallsCount > 0) {
@@ -414,6 +436,7 @@ function collectRuntime() {
 let contentType = "text/plain; version=0.0.4; charset=utf-8";
 
 export {
+  hasProcessedToEndblock,
   formatValue,
   escapeLabelValue,
   block,

package/src/Persistence.res

@@ -24,6 +24,7 @@ type initialChainState = {
   endBlock: option<int>,
   maxReorgDepth: int,
   progressBlockNumber: int,
+  progressBlockTime: option<int>,
   numEventsProcessed: float,
   firstEventBlockNumber: option<int>,
   timestampCaughtUpToHeadOrEndblock: option<Date.t>,
@@ -43,7 +44,8 @@ type initialState = {
   envioInfo: option<JSON.t>,
   cache: dict<effectCacheRecord>,
   chains: array<initialChainState>,
-  checkpointId: Internal.checkpointId,
+  // Where each chain's checkpoint sequence stands in the database.
+  checkpointFrontier: Frontier.t,
   // Needed to keep reorg detection logic between restarts
   reorgCheckpoints: array<Internal.reorgCheckpoint>,
 }
@@ -59,7 +61,12 @@ type updatedEffectCache = {
 }
 
 type rollback = {
-  diffCheckpointId: Internal.checkpointId,
+  // The id each chain's diff rows are stamped with: the first after what that
+  // chain has committed.
+  diffFrontier: Frontier.t,
+  // The same ids as checkpoint rows, for a sink that resolves current state
+  // through them. Postgres writes the diff to the entity table and ignores these.
+  diffCheckpoints: array<InternalTable.Checkpoints.diffCheckpoint>,
   // How far back the deletes reach on each chain, travelling with the diff so
   // the write leaves an untouched sibling's rows alone.
   floors: RollbackFloors.t,
@@ -75,6 +82,14 @@ type rollback = {
   progressedChains: array<InternalTable.Chains.progressedChain>,
 }
 
+// Where a write leaves each chain's sequence. A rollback's diff rows sit on
+// chains the batch may not have progressed at all, so the write reaches them too.
+let writtenFrontier = (~batch: Batch.t, ~rollback: option<rollback>) =>
+  switch rollback {
+  | Some({diffFrontier}) => Frontier.mergeMax(batch->Batch.checkpointFrontier, diffFrontier)
+  | None => batch->Batch.checkpointFrontier
+  }
+
 // One flush group: the changes an entity accumulated within a single chain
 // scope. A per-chain entity contributes one group per chain, and the scope is
 // what stamps the chain id onto the rows — it's never re-derived downstream.
@@ -82,6 +97,7 @@ type updatedEntity = {
   entityConfig: Internal.entityConfig,
   scope: Internal.chainScope,
   changes: array<Change.t<Internal.entity>>,
+  shouldSaveHistory: bool,
 }
 
 // An id the rollback must delete, together with the scope its row lives in.
@@ -110,8 +126,13 @@ type storage = {
   // `throwIfIncompatible` gets what the storage holds before any sink is
   // resumed, so a config the stored one rules out is reported as such rather
   // than as the sink tripping over tables it never created.
+  //
+  // `chainIds` is what this run drives. An isolated run resumes a subset of
+  // the stored chains, and the sink's resume trims past each resumed chain's
+  // checkpoint, so chains a sibling process is still writing must stay out.
   resumeInitialState: (
     ~entities: array<Internal.entityConfig>,
+    ~chainIds: array<ChainId.t>,
     ~throwIfIncompatible: (
       ~storedEnvioInfo: option<JSON.t>,
       ~storedContractMapping: ContractMapping.t,
@@ -124,17 +145,24 @@ type storage = {
   // Creates whatever indexes the filters need and aren't there yet, resolving
   // once they're queryable. Best-effort: it resolves even when a build fails,
   // leaving the query to run unindexed rather than failing the handler.
-  ensureQueryIndexes: (~table: Table.table, ~filters: array<EntityFilter.t>) => promise<unit>,
-  // Creates every schema-defined index still missing, without touching
-  // `ready_at`. For a resumed indexer that is already ready and so never runs
-  // `finalizeBackfill`: an index dropped or invalidated while it was down would
-  // otherwise never be rebuilt. Best-effort, and safe to run with indexing live.
-  ensureSchemaIndexes: (~entities: array<Internal.entityConfig>) => promise<unit>,
-  // Creates every schema-defined index still missing, then stamps `ready_at` on
-  // the given chains. Called once, when backfill completes. The indexes are
-  // committed one at a time so a failure part way through doesn't undo the ones
-  // already built; `ready_at` is only written once they all verify, and all
-  // chains are stamped together.
+  ensureQueryIndexes: (
+    ~entityConfig: Internal.entityConfig,
+    ~scope: Internal.chainScope,
+    ~filters: array<EntityFilter.t>,
+  ) => promise<unit>,
+  // Creates every index the schema promises for the given chains, without
+  // touching `ready_at`. For a resumed indexer that is already ready and so
+  // never runs `finalizeBackfill`: an index dropped or invalidated while it was
+  // down would otherwise never be rebuilt. Best-effort, and safe to run with
+  // indexing live.
+  ensureSchemaIndexes: (
+    ~entities: array<Internal.entityConfig>,
+    ~chainIds: array<ChainId.t>,
+  ) => promise<unit>,
+  // Creates every index the schema promises for the given chains, then stamps
+  // `ready_at` on them. Called once those chains finish backfill. The indexes
+  // are committed one at a time so a failure part way through doesn't undo the
+  // ones already built; `ready_at` is only written once they all verify.
   finalizeBackfill: (
     ~entities: array<Internal.entityConfig>,
     ~chainIds: array<ChainId.t>,
@@ -146,13 +174,15 @@ type storage = {
   // Update chain metadata
   setChainMeta: dict<InternalTable.Chains.metaFields> => promise<unknown>,
   // Prune old checkpoints
-  pruneStaleCheckpoints: (~safeCheckpoints: CheckpointBounds.t) => promise<unit>,
+  pruneStaleCheckpoints: (
+    ~safeCheckpoints: CheckpointSequence.checkpointBoundsByChain,
+  ) => promise<unit>,
   // Prune stale entity history
   pruneStaleEntityHistory: (
     ~entityName: string,
     ~entityIndex: int,
     ~chainIdColumn: option<string>,
-    ~safeCheckpoints: CheckpointBounds.t,
+    ~safeCheckpoints: CheckpointSequence.checkpointBoundsByChain,
   ) => promise<unit>,
   // Get rollback target checkpoint
   getRollbackTargetCheckpoint: (
@@ -180,7 +210,6 @@ type storage = {
   writeBatch: (
     ~batch: Batch.t,
     ~rollback: option<rollback>,
-    ~isInReorgThreshold: bool,
     ~config: Config.t,
     ~allEntities: array<Internal.entityConfig>,
     ~updatedEffectsCache: array<updatedEffectCache>,
@@ -242,6 +271,10 @@ let init = {
     ~runCommand,
     ~reset=false,
     ~lowercaseAddresses=false,
+    // An isolated run needs the schema to exist already: initializing under it
+    // would create rows for this process's chains only, leaving the ones it
+    // skipped with no state for their own processes to resume.
+    ~requireInitialized=false,
     ~startBlockRetry=StartBlockResolver.UntilItAnswers,
   ) => {
     try {
@@ -260,6 +293,11 @@ let init = {
         })
         persistence.storageStatus = Initializing(promise)
         if reset || !(await persistence.storage.isInitialized()) {
+          if requireInitialized {
+            JsError.throwWithMessage(
+              "`envio start --chain` needs a database that already holds every chain. Run `envio local db-migrate up` once with the full config, then start a process per chain.",
+            )
+          }
           Logging.info(`Initializing the indexer storage...`)
           // Only runs once per schema (this branch is the "first deploy or
           // reset" gate), which is exactly when a `latest` start block must be
@@ -289,6 +327,7 @@ let init = {
           Logging.info(`Found existing indexer storage. Resuming indexing state...`)
           let initialState = await persistence.storage.resumeInitialState(
             ~entities=persistence.allEntities,
+            ~chainIds=chainConfigs->Array.map(chain => chain.id),
             ~throwIfIncompatible=(~storedEnvioInfo, ~storedContractMapping) =>
               Config.throwIfResumeIncompatible(
                 ~storedEnvioInfo,

package/src/Persistence.res.mjs

@@ -1,13 +1,23 @@
 // Generated by ReScript, PLEASE EDIT WITH CARE
 
+import * as Batch from "./Batch.res.mjs";
 import * as Config from "./Config.res.mjs";
 import * as Logging from "./Logging.res.mjs";
+import * as Frontier from "./db/Frontier.res.mjs";
 import * as EntityHistory from "./db/EntityHistory.res.mjs";
 import * as ErrorHandling from "./ErrorHandling.res.mjs";
 import * as Stdlib_JsError from "@rescript/runtime/lib/es6/Stdlib_JsError.js";
 import * as StartBlockResolver from "./sources/StartBlockResolver.res.mjs";
 import * as Primitive_exceptions from "@rescript/runtime/lib/es6/Primitive_exceptions.js";
 
+function writtenFrontier(batch, rollback) {
+  if (rollback !== undefined) {
+    return Frontier.mergeMax(Batch.checkpointFrontier(batch), rollback.diffFrontier);
+  } else {
+    return Batch.checkpointFrontier(batch);
+  }
+}
+
 let StorageError = /* @__PURE__ */Primitive_exceptions.create("Persistence.StorageError");
 
 function make(userEntities, allEnums, storage) {
@@ -21,9 +31,10 @@ function make(userEntities, allEnums, storage) {
   };
 }
 
-async function init(persistence, chainConfigs, contractMapping, envioInfo, resetCommand, runCommand, resetOpt, lowercaseAddressesOpt, startBlockRetryOpt) {
+async function init(persistence, chainConfigs, contractMapping, envioInfo, resetCommand, runCommand, resetOpt, lowercaseAddressesOpt, requireInitializedOpt, startBlockRetryOpt) {
   let reset = resetOpt !== undefined ? resetOpt : false;
   let lowercaseAddresses = lowercaseAddressesOpt !== undefined ? lowercaseAddressesOpt : false;
+  let requireInitialized = requireInitializedOpt !== undefined ? requireInitializedOpt : false;
   let startBlockRetry = startBlockRetryOpt !== undefined ? startBlockRetryOpt : "UntilItAnswers";
   try {
     let promise = persistence.storageStatus;
@@ -50,6 +61,9 @@ async function init(persistence, chainConfigs, contractMapping, envioInfo, reset
       _0: promise$1
     };
     if (reset || !await persistence.storage.isInitialized()) {
+      if (requireInitialized) {
+        Stdlib_JsError.throwWithMessage("`envio start --chain` needs a database that already holds every chain. Run `envio local db-migrate up` once with the full config, then start a process per chain.");
+      }
       Logging.info(`Initializing the indexer storage...`);
       let chainConfigs$1 = await StartBlockResolver.resolveAllOrThrow(chainConfigs, lowercaseAddresses, startBlockRetry, undefined, undefined);
       let initialState = await persistence.storage.initialize(chainConfigs$1, persistence.allEntities, persistence.allEnums, contractMapping, envioInfo);
@@ -64,7 +78,7 @@ async function init(persistence, chainConfigs, contractMapping, envioInfo, reset
       tmp = typeof match !== "object" ? false : match.TAG === "Initializing";
       if (tmp) {
         Logging.info(`Found existing indexer storage. Resuming indexing state...`);
-        let initialState$1 = await persistence.storage.resumeInitialState(persistence.allEntities, (storedEnvioInfo, storedContractMapping) => Config.throwIfResumeIncompatible(storedEnvioInfo, storedContractMapping, envioInfo, contractMapping, resetCommand, runCommand));
+        let initialState$1 = await persistence.storage.resumeInitialState(persistence.allEntities, chainConfigs.map(chain => chain.id), (storedEnvioInfo, storedContractMapping) => Config.throwIfResumeIncompatible(storedEnvioInfo, storedContractMapping, envioInfo, contractMapping, resetCommand, runCommand));
         persistence.storageStatus = {
           TAG: "Ready",
           _0: initialState$1
@@ -105,10 +119,11 @@ function getInitializedState(persistence) {
 }
 
 export {
+  writtenFrontier,
   StorageError,
   make,
   init,
   getInitializedStorageOrThrow,
   getInitializedState,
 }
-/* Config Not a pure module */
+/* Batch Not a pure module */