envio 3.10.0 → 3.11.0-svm-alpha.1

package/evm.schema.json

@@ -494,10 +494,8 @@
           "minimum": 0
         },
         "start_block": {
-          "description": "The block at which the indexer should start ingesting data",
-          "type": "integer",
-          "format": "uint64",
-          "minimum": 0
+          "description": "The block at which the indexer should start ingesting data, or \"latest\" to start from the chain's current head block when the indexer is first deployed. Once resolved, the concrete block is persisted and reused every time the indexer resumes normally (for example recovering from a crash), so downtime is backfilled instead of skipped. Running `envio start`/`dev` with -r (--restart) resets this like any other config change: \"latest\" resolves again, against the head at that time.",
+          "$ref": "#/$defs/StartBlock"
         },
         "end_block": {
           "description": "The block at which the indexer should terminate.",
@@ -685,6 +683,25 @@
         "url"
       ]
     },
+    "StartBlock": {
+      "description": "A chain's configured start block: either a concrete block number or the\nliteral \"latest\". Config parsing never touches the network, so `Latest`\nstays unresolved here — it's resolved once at runtime, right before the\nindexer's first-ever persisted state is written, and never re-resolved\non a normal resume (see packages/envio/src/sources/StartBlockResolver.res).\nNote: this repo's `-r`/`--restart` CLI flag wipes the DB and re-deploys\nfrom scratch, so it re-resolves \"latest\" too — \"resume\" here means the\nopposite: recovering from a crash or process restart without `-r`.",
+      "anyOf": [
+        {
+          "type": "integer",
+          "format": "uint64",
+          "minimum": 0
+        },
+        {
+          "$ref": "#/$defs/StartBlockTag"
+        }
+      ]
+    },
+    "StartBlockTag": {
+      "type": "string",
+      "enum": [
+        "latest"
+      ]
+    },
     "ChainContract": {
       "type": "object",
       "properties": {

package/fuel.schema.json

@@ -328,10 +328,8 @@
           ]
         },
         "start_block": {
-          "description": "The block at which the indexer should start ingesting data",
-          "type": "integer",
-          "format": "uint64",
-          "minimum": 0
+          "description": "The block at which the indexer should start ingesting data, or \"latest\" to start from the chain's current head block when the indexer is first deployed. Once resolved, the concrete block is persisted and reused every time the indexer resumes normally (for example recovering from a crash), so downtime is backfilled instead of skipped. Running `envio start`/`dev` with -r (--restart) resets this like any other config change: \"latest\" resolves again, against the head at that time.",
+          "$ref": "#/$defs/StartBlock"
         },
         "end_block": {
           "description": "The block at which the indexer should terminate.",
@@ -388,6 +386,25 @@
         "start_block"
       ]
     },
+    "StartBlock": {
+      "description": "A chain's configured start block: either a concrete block number or the\nliteral \"latest\". Config parsing never touches the network, so `Latest`\nstays unresolved here — it's resolved once at runtime, right before the\nindexer's first-ever persisted state is written, and never re-resolved\non a normal resume (see packages/envio/src/sources/StartBlockResolver.res).\nNote: this repo's `-r`/`--restart` CLI flag wipes the DB and re-deploys\nfrom scratch, so it re-resolves \"latest\" too — \"resume\" here means the\nopposite: recovering from a crash or process restart without `-r`.",
+      "anyOf": [
+        {
+          "type": "integer",
+          "format": "uint64",
+          "minimum": 0
+        },
+        {
+          "$ref": "#/$defs/StartBlockTag"
+        }
+      ]
+    },
+    "StartBlockTag": {
+      "type": "string",
+      "enum": [
+        "latest"
+      ]
+    },
     "HyperfuelConfig": {
       "type": "object",
       "properties": {

package/index.d.ts

@@ -1216,6 +1216,9 @@ export type SvmAllTransactionFields = {
   readonly err: string | undefined;
   readonly fee: bigint;
   readonly computeUnitsConsumed: bigint | undefined;
+  /** Every account the transaction resolves to: the static keys, then the
+   *  address lookup tables' writable then readonly addresses. A transaction's
+   *  account indexes address this list. */
   readonly accountKeys: readonly string[];
   readonly recentBlockhash: string;
   readonly version: string | undefined;
@@ -1377,7 +1380,7 @@ type SvmNamedAccounts<
   Acc extends Readonly<Record<string, unknown>>,
   Fields extends SvmFieldsSelection,
 > = {
-  readonly [K in keyof Acc & string]: SvmInstructionAccount<Fields, K>;
+  readonly [K in keyof Acc]: SvmInstructionAccount<Fields, K & string>;
 };
 
 /** The parent transaction of a {@link SvmInstruction}, narrowed to the
@@ -1524,8 +1527,8 @@ export type SvmOnInstructionOptions<
   /** Program name as declared under `chains[].programs[].name` in
    * `config.yaml`. */
   readonly program: P;
-  /** Instruction name as declared under
-   * `chains[].programs[].instructions[].name` in `config.yaml`. */
+  /** Instruction name from the program's IDL, or from
+   * `chains[].programs[].instructions[].name` when the layout is inline. */
   readonly instruction: I;
   readonly fields?: Fields & SvmFieldsLiteralCheck<Fields>;
   readonly where?: SvmOnInstructionWhere<SvmAccountsOf<P, I>>;
@@ -1768,9 +1771,10 @@ type SvmEcosystem<Config extends IndexerConfigTypes = GlobalConfig> =
           }
             ? {
                 /**
-                 * Register an instruction handler. Dispatch matches on
-                 * `(programId, discriminator)` from the YAML config.
-                 * Handler `fields` is the only source of payload selection.
+                 * Register an instruction handler. `program` and `instruction`
+                 * name an entry from the IDL or YAML. Dispatch uses that
+                 * instruction's discriminator. Handler `fields` is the only
+                 * source of payload selection.
                  */
                 readonly onInstruction: <
                   P extends keyof Programs & string,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "envio",
-  "version": "3.10.0",
+  "version": "3.11.0-svm-alpha.1",
   "type": "module",
   "description": "A latency and sync speed optimized, developer friendly blockchain data indexer.",
   "bin": "./bin.mjs",
@@ -68,10 +68,10 @@
     "tsx": "4.21.0"
   },
   "optionalDependencies": {
-    "envio-linux-x64": "3.10.0",
-    "envio-linux-x64-musl": "3.10.0",
-    "envio-linux-arm64": "3.10.0",
-    "envio-darwin-x64": "3.10.0",
-    "envio-darwin-arm64": "3.10.0"
+    "envio-linux-x64": "3.11.0-svm-alpha.1",
+    "envio-linux-x64-musl": "3.11.0-svm-alpha.1",
+    "envio-linux-arm64": "3.11.0-svm-alpha.1",
+    "envio-darwin-x64": "3.11.0-svm-alpha.1",
+    "envio-darwin-arm64": "3.11.0-svm-alpha.1"
   }
 }

package/src/Batch.res

@@ -34,24 +34,32 @@ type t = {
   totalBatchSize: int,
   items: array<Internal.item>,
   progressedChainsById: dict<chainAfterBatch>,
-  // Processed inside the reorg threshold. Drives whether history is saved, so
-  // writes never merge across a change in this value.
-  isInReorgThreshold: bool,
+  // Whether the batch's rows get history. Writes never merge across a change in
+  // it, so a single write can't mix the two.
+  history: HistoryPolicy.t,
   // Unnest-like checkpoint fields:
   checkpointIds: array<bigint>,
   checkpointChainIds: array<ChainId.t>,
   checkpointBlockNumbers: array<int>,
   checkpointBlockHashes: array<Null.t<string>>,
+  // Items the checkpoint carries, which is what indexes `items`.
+  checkpointItemsCount: array<int>,
+  // Logs the checkpoint carries: one log routed to several registrations is one
+  // event, however many items it made.
   checkpointEventsProcessed: array<int>,
   registeredAddresses: array<AddressRows.staged>,
 }
 
+// What a chain contributed to the batch: items taken off its buffer, and the
+// logs behind them.
+type chainBatchCounts = {size: int, eventsProcessed: int}
+
 let getProgressedChainsById = {
   let getChainAfterBatchIfProgressed = (
     ~chainBeforeBatch: chainBeforeBatch,
     ~progressBlockNumberAfterBatch,
     ~fetchStateAfterBatch,
-    ~batchSize,
+    ~counts: chainBatchCounts,
   ) => {
     // The check is sufficient, since we guarantee to include a full block in a batch
     // Also, this might be true even if batchSize is 0,
@@ -60,10 +68,11 @@ let getProgressedChainsById = {
       Some(
         (
           {
-            batchSize,
+            batchSize: counts.size,
             progressBlockNumber: progressBlockNumberAfterBatch,
             sourceBlockNumber: chainBeforeBatch.sourceBlockNumber,
-            totalEventsProcessed: chainBeforeBatch.totalEventsProcessed +. batchSize->Int.toFloat,
+            totalEventsProcessed: chainBeforeBatch.totalEventsProcessed +.
+            counts.eventsProcessed->Int.toFloat,
             fetchState: fetchStateAfterBatch,
             isProgressAtHeadWhenBatchCreated: progressBlockNumberAfterBatch >=
             chainBeforeBatch.sourceBlockNumber - chainBeforeBatch.chainConfig.blockLag,
@@ -77,7 +86,7 @@ let getProgressedChainsById = {
 
   (
     ~chainsBeforeBatch: dict<chainBeforeBatch>,
-    ~batchSizePerChain: dict<int>,
+    ~countsPerChain: dict<chainBatchCounts>,
     ~progressBlockNumberPerChain: dict<int>,
   ) => {
     let progressedChainsById = Dict.make()
@@ -98,14 +107,14 @@ let getProgressedChainsById = {
       | None => chainBeforeBatch.progressBlockNumber
       }
 
-      switch switch batchSizePerChain->Utils.Dict.dangerouslyGetNonOption(
+      switch switch countsPerChain->Utils.Dict.dangerouslyGetNonOption(
         fetchState.chainId->ChainId.toString,
       ) {
-      | Some(batchSize) =>
-        let leftItems = fetchState.buffer->Array.slice(~start=batchSize)
+      | Some(counts) =>
+        let leftItems = fetchState.buffer->Array.slice(~start=counts.size)
         getChainAfterBatchIfProgressed(
           ~chainBeforeBatch,
-          ~batchSize,
+          ~counts,
           ~fetchStateAfterBatch=fetchState->FetchState.updateInternal(~mutItems=leftItems),
           ~progressBlockNumberAfterBatch,
         )
@@ -113,7 +122,7 @@ let getProgressedChainsById = {
       | None =>
         getChainAfterBatchIfProgressed(
           ~chainBeforeBatch,
-          ~batchSize=0,
+          ~counts={size: 0, eventsProcessed: 0},
           ~fetchStateAfterBatch=chainBeforeBatch.fetchState,
           ~progressBlockNumberAfterBatch,
         )
@@ -146,7 +155,7 @@ let seekFirstAbove = (blockNumbers: array<int>, blockNumber) => {
 
 @inline
 let addReorgCheckpoints = (
-  ~prevCheckpointId,
+  ~cursor,
   ~scannedHashes: reorgHashSnapshot,
   ~shouldRollbackOnReorg,
   ~fromBlockExclusive,
@@ -156,10 +165,10 @@ let addReorgCheckpoints = (
   ~mutCheckpointChainIds,
   ~mutCheckpointBlockNumbers,
   ~mutCheckpointBlockHashes,
+  ~mutCheckpointItemsCount,
   ~mutCheckpointEventsProcessed,
 ) => {
   if shouldRollbackOnReorg {
-    let prevCheckpointId = ref(prevCheckpointId)
     // The snapshot already holds only in-threshold scanned hashes, ascending,
     // so seeking to the gap's lower bound gives the gap checkpoints without
     // rescanning the whole snapshot for every gap in the batch.
@@ -172,37 +181,30 @@ let addReorgCheckpoints = (
         scannedHashes.hashByBlockNumber
         ->Utils.Dict.dangerouslyGetByIntNonOption(blockNumber)
         ->Option.getUnsafe
-      let checkpointId = prevCheckpointId.contents->BigInt.add(1n)
-      prevCheckpointId := checkpointId
+      let checkpointId = cursor->CheckpointSequence.next(~chainId)
 
       mutCheckpointIds->Array.push(checkpointId)
       mutCheckpointChainIds->Array.push(chainId)
       mutCheckpointBlockNumbers->Array.push(blockNumber)
       mutCheckpointBlockHashes->Array.push(Null.Value(hash))
+      mutCheckpointItemsCount->Array.push(0)
       mutCheckpointEventsProcessed->Array.push(0)
 
       idx := idx.contents + 1
     }
-    prevCheckpointId.contents
-  } else {
-    prevCheckpointId
   }
 }
 
-// Checkpoint ids are handed out in ascending order as each chain's items are
-// walked in block order, so within a chain a higher id always means a later
-// block. Rollback preserves that: it deletes the chain's ids above its target
-// before any higher one is allocated, so the ids the re-indexed blocks get are
-// above everything the chain still holds. An isolated rollback leans on the
-// invariant — it deletes by `chain_id = c AND id > target`, which is only the
-// chain's stale suffix if ids and blocks agree on order within the chain. Ids
-// interleave freely *across* chains, which is exactly why that predicate can't
-// be the id bound alone.
-let prepareBatch = (
-  ~checkpointIdBeforeBatch,
+// Within a chain, ids ascend with block order, and rollback preserves that by
+// deleting the chain's ids above its target before any higher one is allocated.
+// Every rollback deletes by `id > target`, which is only the stale suffix while
+// ids and blocks agree on order within each chain.
+let make = (
+  ~sequence: CheckpointSequence.t,
+  ~frontier: Frontier.t,
   ~chainsBeforeBatch: dict<chainBeforeBatch>,
   ~batchSizeTarget,
-  ~isInReorgThreshold,
+  ~history,
 ) => {
   let preparedFetchStates =
     chainsBeforeBatch
@@ -214,8 +216,8 @@ let prepareBatch = (
   let preparedNumber = preparedFetchStates->Array.length
   let totalBatchSize = ref(0)
 
-  let prevCheckpointId = ref(checkpointIdBeforeBatch)
-  let mutBatchSizePerChain = Dict.make()
+  let cursor = sequence->CheckpointSequence.cursor(~frontier)
+  let mutCountsPerChain = Dict.make()
   let mutProgressBlockNumberPerChain = Dict.make()
 
   let items = []
@@ -223,6 +225,7 @@ let prepareBatch = (
   let checkpointChainIds = []
   let checkpointBlockNumbers = []
   let checkpointBlockHashes = []
+  let checkpointItemsCount = []
   let checkpointEventsProcessed = []
 
   // Accumulate items for all actively indexing chains
@@ -241,29 +244,38 @@ let prepareBatch = (
       ->Option.getUnsafe
 
     let prevBlockNumber = ref(chainBeforeBatch.progressBlockNumber)
+    let chainEventsProcessed = ref(0)
     if chainBatchSize > 0 {
       for idx in 0 to chainBatchSize - 1 {
         let item = fetchState.buffer->Array.getUnsafe(idx)
         let blockNumber = item->Internal.getItemBlockNumber
+        // The buffer is sorted, so a log's items are consecutive: the first of
+        // them is the only one that counts as an event.
+        let isNewEvent =
+          idx === 0 ||
+            !(fetchState.buffer->Array.getUnsafe(idx - 1)->FetchState.isSameLog(item))
+        if isNewEvent {
+          chainEventsProcessed := chainEventsProcessed.contents + 1
+        }
 
         // Every new block we should create a new checkpoint
         if blockNumber !== prevBlockNumber.contents {
-          prevCheckpointId :=
-            addReorgCheckpoints(
-              ~chainId=fetchState.chainId,
-              ~scannedHashes=chainBeforeBatch.scannedHashes,
-              ~shouldRollbackOnReorg=chainBeforeBatch.shouldRollbackOnReorg,
-              ~prevCheckpointId=prevCheckpointId.contents,
-              ~fromBlockExclusive=prevBlockNumber.contents,
-              ~toBlockExclusive=blockNumber,
-              ~mutCheckpointIds=checkpointIds,
-              ~mutCheckpointChainIds=checkpointChainIds,
-              ~mutCheckpointBlockNumbers=checkpointBlockNumbers,
-              ~mutCheckpointBlockHashes=checkpointBlockHashes,
-              ~mutCheckpointEventsProcessed=checkpointEventsProcessed,
-            )
-
-          let checkpointId = prevCheckpointId.contents->BigInt.add(1n)
+          addReorgCheckpoints(
+            ~chainId=fetchState.chainId,
+            ~scannedHashes=chainBeforeBatch.scannedHashes,
+            ~shouldRollbackOnReorg=chainBeforeBatch.shouldRollbackOnReorg,
+            ~cursor,
+            ~fromBlockExclusive=prevBlockNumber.contents,
+            ~toBlockExclusive=blockNumber,
+            ~mutCheckpointIds=checkpointIds,
+            ~mutCheckpointChainIds=checkpointChainIds,
+            ~mutCheckpointBlockNumbers=checkpointBlockNumbers,
+            ~mutCheckpointBlockHashes=checkpointBlockHashes,
+            ~mutCheckpointItemsCount=checkpointItemsCount,
+            ~mutCheckpointEventsProcessed=checkpointEventsProcessed,
+          )
+
+          let checkpointId = cursor->CheckpointSequence.next(~chainId=fetchState.chainId)
 
           checkpointIds->Array.push(checkpointId)->ignore
           checkpointChainIds->Array.push(fetchState.chainId)->ignore
@@ -278,41 +290,49 @@ let prepareBatch = (
             },
           )
           ->ignore
+          checkpointItemsCount->Array.push(1)->ignore
           checkpointEventsProcessed->Array.push(1)->ignore
 
           prevBlockNumber := blockNumber
-          prevCheckpointId := checkpointId
         } else {
-          let lastIndex = checkpointEventsProcessed->Array.length - 1
-          checkpointEventsProcessed
-          ->Array.setUnsafe(lastIndex, checkpointEventsProcessed->Array.getUnsafe(lastIndex) + 1)
+          let lastIndex = checkpointItemsCount->Array.length - 1
+          checkpointItemsCount
+          ->Array.setUnsafe(lastIndex, checkpointItemsCount->Array.getUnsafe(lastIndex) + 1)
           ->ignore
+          if isNewEvent {
+            checkpointEventsProcessed
+            ->Array.setUnsafe(lastIndex, checkpointEventsProcessed->Array.getUnsafe(lastIndex) + 1)
+            ->ignore
+          }
         }
 
         items->Array.push(item)->ignore
       }
 
       totalBatchSize := totalBatchSize.contents + chainBatchSize
-      mutBatchSizePerChain->ChainId.Dict.set(fetchState.chainId, chainBatchSize)
+      mutCountsPerChain->ChainId.Dict.set(
+        fetchState.chainId,
+        {size: chainBatchSize, eventsProcessed: chainEventsProcessed.contents},
+      )
     }
 
     let progressBlockNumberAfterBatch =
       fetchState->FetchState.getProgressBlockNumberAt(~index=chainBatchSize)
 
-    prevCheckpointId :=
-      addReorgCheckpoints(
-        ~chainId=fetchState.chainId,
-        ~scannedHashes=chainBeforeBatch.scannedHashes,
-        ~shouldRollbackOnReorg=chainBeforeBatch.shouldRollbackOnReorg,
-        ~prevCheckpointId=prevCheckpointId.contents,
-        ~fromBlockExclusive=prevBlockNumber.contents,
-        ~toBlockExclusive=progressBlockNumberAfterBatch + 1, // Make it inclusive
-        ~mutCheckpointIds=checkpointIds,
-        ~mutCheckpointChainIds=checkpointChainIds,
-        ~mutCheckpointBlockNumbers=checkpointBlockNumbers,
-        ~mutCheckpointBlockHashes=checkpointBlockHashes,
-        ~mutCheckpointEventsProcessed=checkpointEventsProcessed,
-      )
+    addReorgCheckpoints(
+      ~chainId=fetchState.chainId,
+      ~scannedHashes=chainBeforeBatch.scannedHashes,
+      ~shouldRollbackOnReorg=chainBeforeBatch.shouldRollbackOnReorg,
+      ~cursor,
+      ~fromBlockExclusive=prevBlockNumber.contents,
+      ~toBlockExclusive=progressBlockNumberAfterBatch + 1, // Make it inclusive
+      ~mutCheckpointIds=checkpointIds,
+      ~mutCheckpointChainIds=checkpointChainIds,
+      ~mutCheckpointBlockNumbers=checkpointBlockNumbers,
+      ~mutCheckpointBlockHashes=checkpointBlockHashes,
+      ~mutCheckpointItemsCount=checkpointItemsCount,
+      ~mutCheckpointEventsProcessed=checkpointEventsProcessed,
+    )
 
     mutProgressBlockNumberPerChain->ChainId.Dict.set(
       fetchState.chainId,
@@ -327,26 +347,28 @@ let prepareBatch = (
     items,
     progressedChainsById: getProgressedChainsById(
       ~chainsBeforeBatch,
-      ~batchSizePerChain=mutBatchSizePerChain,
+      ~countsPerChain=mutCountsPerChain,
       ~progressBlockNumberPerChain=mutProgressBlockNumberPerChain,
     ),
-    isInReorgThreshold,
+    history,
     checkpointIds,
     checkpointChainIds,
     checkpointBlockNumbers,
     checkpointBlockHashes,
+    checkpointItemsCount,
     checkpointEventsProcessed,
     registeredAddresses: [],
   }
 }
 
-let make = (
-  ~checkpointIdBeforeBatch,
-  ~chainsBeforeBatch: dict<chainBeforeBatch>,
-  ~batchSizeTarget,
-  ~isInReorgThreshold,
-) => {
-  prepareBatch(~checkpointIdBeforeBatch, ~chainsBeforeBatch, ~batchSizeTarget, ~isInReorgThreshold)
+// Where the batch leaves each chain it handed ids to. Ids ascend within a
+// chain, so the last one seen per chain is its highest.
+let checkpointFrontier = (batch: t): Frontier.t => {
+  let frontier = Frontier.empty()
+  batch.checkpointChainIds->Array.forEachWithIndex((chainId, index) =>
+    frontier->Frontier.set(chainId, batch.checkpointIds->Array.getUnsafe(index))
+  )
+  frontier
 }
 
 let findFirstEventBlockNumber = (batch: t, ~chainId) => {
@@ -357,7 +379,7 @@ let findFirstEventBlockNumber = (batch: t, ~chainId) => {
     let checkpointChainId = batch.checkpointChainIds->Array.getUnsafe(idx.contents)
     if (
       checkpointChainId === chainId &&
-        batch.checkpointEventsProcessed->Array.getUnsafe(idx.contents) > 0
+        batch.checkpointItemsCount->Array.getUnsafe(idx.contents) > 0
     ) {
       result := Some(batch.checkpointBlockNumbers->Array.getUnsafe(idx.contents))
     } else {