envio 3.0.2-svm-alpha.0 → 3.0.2-svm-alpha.1

package/index.d.ts

@@ -1000,6 +1000,15 @@ export type SvmInstruction = {
   readonly decoded?: SvmDecodedInstruction;
 };
 
+export type SvmTokenBalance = {
+  readonly account?: string;
+  readonly mint?: string;
+  readonly owner?: string;
+  /** u64 decimal string. Cast with BigInt(...) for arithmetic. */
+  readonly preAmount?: string;
+  readonly postAmount?: string;
+};
+
 /** Parent transaction surfaced when an instruction's
  * `include_transaction` flag is `true`. */
 export type SvmTransaction = {
@@ -1013,6 +1022,9 @@ export type SvmTransaction = {
   readonly accountKeys: readonly string[];
   readonly recentBlockhash?: string;
   readonly version?: string;
+  /** SPL Token / Token-2022 balance snapshots for this transaction.
+   * Present when `include_token_balances` is `true`. */
+  readonly tokenBalances?: readonly SvmTokenBalance[];
 };
 
 export type SvmLog = {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "envio",
-  "version": "3.0.2-svm-alpha.0",
+  "version": "3.0.2-svm-alpha.1",
   "type": "module",
   "description": "A latency and sync speed optimized, developer friendly blockchain data indexer.",
   "bin": "./bin.mjs",
@@ -70,10 +70,10 @@
     "tsx": "4.21.0"
   },
   "optionalDependencies": {
-    "envio-linux-x64": "3.0.2-svm-alpha.0",
-    "envio-linux-x64-musl": "3.0.2-svm-alpha.0",
-    "envio-linux-arm64": "3.0.2-svm-alpha.0",
-    "envio-darwin-x64": "3.0.2-svm-alpha.0",
-    "envio-darwin-arm64": "3.0.2-svm-alpha.0"
+    "envio-linux-x64": "3.0.2-svm-alpha.1",
+    "envio-linux-x64-musl": "3.0.2-svm-alpha.1",
+    "envio-linux-arm64": "3.0.2-svm-alpha.1",
+    "envio-darwin-x64": "3.0.2-svm-alpha.1",
+    "envio-darwin-arm64": "3.0.2-svm-alpha.1"
   }
 }

package/src/Config.res

@@ -694,6 +694,7 @@ let fromPublic = (publicConfigJson: JSON.t) => {
                   "discriminatorByteLen": int,
                   "includeTransaction": bool,
                   "includeLogs": bool,
+                  "includeTokenBalances": bool,
                   "accountFilters": option<
                     array<{"position": int, "values": array<string>}>,
                   >,
@@ -723,6 +724,7 @@ let fromPublic = (publicConfigJson: JSON.t) => {
             ~discriminatorByteLen=svm["discriminatorByteLen"],
             ~includeTransaction=svm["includeTransaction"],
             ~includeLogs=svm["includeLogs"],
+            ~includeTokenBalances=svm["includeTokenBalances"],
             ~accountFilters,
             ~isInner=svm["isInner"],
             ~isWildcard=false,

package/src/Config.res.mjs

@@ -515,7 +515,7 @@ function fromPublic(publicConfigJson) {
               position: af.position,
               values: af.values
             }));
-            return EventConfigBuilder.buildSvmInstructionEventConfig(contractName, eventName, programId, svm.discriminator, svm.discriminatorByteLen, svm.includeTransaction, svm.includeLogs, accountFilters, svm.isInner, false, undefined, undefined, Stdlib_Option.getOr(svm.accounts, []), Stdlib_Option.getOr(svm.args, null), svmDefinedTypes, startBlock);
+            return EventConfigBuilder.buildSvmInstructionEventConfig(contractName, eventName, programId, svm.discriminator, svm.discriminatorByteLen, svm.includeTransaction, svm.includeLogs, svm.includeTokenBalances, accountFilters, svm.isInner, false, undefined, undefined, Stdlib_Option.getOr(svm.accounts, []), Stdlib_Option.getOr(svm.args, null), svmDefinedTypes, startBlock);
         }
       });
     } else {

package/src/Envio.res

@@ -58,6 +58,14 @@ type svmInstruction = {
   decoded?: svmDecodedInstruction,
 }
 
+type svmTokenBalance = {
+  account?: SvmTypes.Pubkey.t,
+  mint?: SvmTypes.Pubkey.t,
+  owner?: SvmTypes.Pubkey.t,
+  preAmount?: string,
+  postAmount?: string,
+}
+
 type svmTransaction = {
   signatures: array<string>,
   feePayer?: SvmTypes.Pubkey.t,
@@ -68,6 +76,7 @@ type svmTransaction = {
   accountKeys: array<SvmTypes.Pubkey.t>,
   recentBlockhash?: string,
   version?: string,
+  tokenBalances?: array<svmTokenBalance>,
 }
 
 type svmLog = {

package/src/EventConfigBuilder.res

@@ -492,6 +492,7 @@ let buildSvmInstructionEventConfig = (
   ~discriminatorByteLen: int,
   ~includeTransaction: bool,
   ~includeLogs: bool,
+  ~includeTokenBalances: bool,
   ~accountFilters: array<Internal.svmAccountFilter>,
   ~isInner: option<bool>,
   ~isWildcard: bool,
@@ -526,6 +527,7 @@ let buildSvmInstructionEventConfig = (
     discriminatorByteLen,
     includeTransaction,
     includeLogs,
+    includeTokenBalances,
     accountFilters,
     isInner,
     accounts,

package/src/EventConfigBuilder.res.mjs

@@ -350,7 +350,7 @@ function buildEvmEventConfig(contractName, eventName, sighash, params, isWildcar
   };
 }
 
-function buildSvmInstructionEventConfig(contractName, instructionName, programId, discriminator, discriminatorByteLen, includeTransaction, includeLogs, accountFilters, isInner, isWildcard, handler, contractRegister, accountsOpt, argsOpt, definedTypesOpt, startBlock) {
+function buildSvmInstructionEventConfig(contractName, instructionName, programId, discriminator, discriminatorByteLen, includeTransaction, includeLogs, includeTokenBalances, accountFilters, isInner, isWildcard, handler, contractRegister, accountsOpt, argsOpt, definedTypesOpt, startBlock) {
   let accounts = accountsOpt !== undefined ? accountsOpt : [];
   let args = argsOpt !== undefined ? argsOpt : null;
   let definedTypes = definedTypesOpt !== undefined ? definedTypesOpt : null;
@@ -372,6 +372,7 @@ function buildSvmInstructionEventConfig(contractName, instructionName, programId
     discriminatorByteLen: discriminatorByteLen,
     includeTransaction: includeTransaction,
     includeLogs: includeLogs,
+    includeTokenBalances: includeTokenBalances,
     accountFilters: accountFilters,
     isInner: isInner,
     accounts: accounts,

package/src/Hasura.res

@@ -40,6 +40,19 @@ let clearMetadataRoute = Rest.route(() => {
   responses,
 })
 
+let reloadMetadataRoute = Rest.route(() => {
+  method: Post,
+  path: "",
+  input: s => {
+    let _ = s.field("type", S.literal("reload_metadata"))
+    {
+      "args": s.field("args", S.json(~validate=false)),
+      "auth": s->auth,
+    }
+  },
+  responses,
+})
+
 let trackTablesRoute = Rest.route(() => {
   method: Post,
   path: "",
@@ -110,6 +123,31 @@ let clearHasuraMetadata = async (~endpoint, ~auth) => {
   }
 }
 
+let reloadHasuraMetadata = async (~endpoint, ~auth) => {
+  try {
+    let result = await reloadMetadataRoute->Rest.fetch(
+      {
+        "auth": auth,
+        "args": {
+          "reload_sources": ["default"],
+        }->(Utils.magic: 'a => JSON.t),
+      },
+      ~client=Rest.client(endpoint),
+    )
+    let msg = switch result {
+    | QuerySucceeded => "Hasura metadata reloaded"
+    | AlreadyDone => "Hasura metadata reload acknowledged"
+    }
+    Logging.trace(msg)
+  } catch {
+  | exn =>
+    Logging.error({
+      "msg": `There was an issue reloading hasura metadata - table tracking may race with schema creation.`,
+      "err": exn->Utils.prettifyExn,
+    })
+  }
+}
+
 let trackTables = async (~endpoint, ~auth, ~pgSchema, ~tableNames: array<string>) => {
   try {
     let result = await trackTablesRoute->Rest.fetch(
@@ -242,6 +280,11 @@ let trackDatabase = async (
 
   let _ = await clearHasuraMetadata(~endpoint, ~auth)
 
+  // Force Hasura to re-introspect the source schema before tracking, otherwise
+  // freshly-created user tables may be invisible to pg_track_tables and the call
+  // returns `metadata-warnings` (HTTP 400), leaving tracking permanently broken.
+  await reloadHasuraMetadata(~endpoint, ~auth)
+
   await trackTables(~endpoint, ~auth, ~pgSchema, ~tableNames)
 
   for i in 0 to tableNames->Array.length - 1 {

package/src/Hasura.res.mjs

@@ -46,6 +46,21 @@ function clearMetadataRoute() {
   };
 }
 
+function reloadMetadataRoute() {
+  return {
+    method: "POST",
+    path: "",
+    input: s => {
+      s.field("type", S$RescriptSchema.literal("reload_metadata"));
+      return {
+        args: s.field("args", S$RescriptSchema.json(false)),
+        auth: auth(s)
+      };
+    },
+    responses: responses
+  };
+}
+
 function trackTablesRoute() {
   return {
     method: "POST",
@@ -112,6 +127,26 @@ async function clearHasuraMetadata(endpoint, auth) {
   }
 }
 
+async function reloadHasuraMetadata(endpoint, auth) {
+  try {
+    let result = await Rest.fetch(reloadMetadataRoute, {
+      auth: auth,
+      args: {
+        reload_sources: ["default"]
+      }
+    }, Rest.client(endpoint, undefined));
+    let tmp;
+    tmp = result === "QuerySucceeded" ? "Hasura metadata reloaded" : "Hasura metadata reload acknowledged";
+    return Logging.trace(tmp);
+  } catch (raw_exn) {
+    let exn = Primitive_exceptions.internalToException(raw_exn);
+    return Logging.error({
+      msg: `There was an issue reloading hasura metadata - table tracking may race with schema creation.`,
+      err: Utils.prettifyExn(exn)
+    });
+  }
+}
+
 async function trackTables(endpoint, auth, pgSchema, tableNames) {
   try {
     let result = await Rest.fetch(trackTablesRoute, {
@@ -202,6 +237,7 @@ async function trackDatabase(endpoint, auth, pgSchema, userEntities, aggregateEn
   ]);
   Logging.info("Tracking tables in Hasura");
   await clearHasuraMetadata(endpoint, auth);
+  await reloadHasuraMetadata(endpoint, auth);
   await trackTables(endpoint, auth, pgSchema, tableNames);
   for (let i = 0, i_finish = tableNames.length; i < i_finish; ++i) {
     let tableName = tableNames[i];
@@ -231,10 +267,12 @@ export {
   auth,
   responses,
   clearMetadataRoute,
+  reloadMetadataRoute,
   trackTablesRoute,
   rawBodyRoute,
   sendOperation,
   clearHasuraMetadata,
+  reloadHasuraMetadata,
   trackTables,
   createSelectPermission,
   createEntityRelationship,

package/src/Internal.res

@@ -439,6 +439,7 @@ type svmInstructionEventConfig = {
   discriminatorByteLen: int,
   includeTransaction: bool,
   includeLogs: bool,
+  includeTokenBalances: bool,
   accountFilters: array<svmAccountFilter>,
   /** `None` matches both outer and inner (CPI-invoked) instructions. */
   isInner: option<bool>,

package/src/sources/HyperSyncSolanaClient.res

@@ -65,11 +65,21 @@ module QueryTypes = {
     | @as("kind") Kind
     | @as("message") Message
 
+  type tokenBalanceField =
+    | @as("slot") Slot
+    | @as("transaction_index") TransactionIndex
+    | @as("account") Account
+    | @as("mint") Mint
+    | @as("owner") Owner
+    | @as("pre_amount") PreAmount
+    | @as("post_amount") PostAmount
+
   type fieldSelection = {
     block?: array<blockField>,
     transaction?: array<transactionField>,
     instruction?: array<instructionField>,
     log?: array<logField>,
+    tokenBalance?: array<tokenBalanceField>,
   }
 
   /** Filter for selecting instructions. All non-empty fields are AND-ed: an
@@ -96,6 +106,7 @@ module QueryTypes = {
     isInner?: bool,
     includeTransaction?: bool,
     includeLogs?: bool,
+    includeTokenBalances?: bool,
   }
 
   type transactionSelection = {
@@ -183,11 +194,22 @@ module ResponseTypes = {
     message?: string,
   }
 
+  type tokenBalance = {
+    slot: int,
+    transactionIndex?: int,
+    account?: string,
+    mint?: string,
+    owner?: string,
+    preAmount?: string,
+    postAmount?: string,
+  }
+
   type queryResponseData = {
     blocks: array<block>,
     transactions: array<transaction>,
     instructions: array<instruction>,
     logs: array<log>,
+    tokenBalances: array<tokenBalance>,
   }
 
   type queryResponse = {

package/src/sources/HyperSyncSolanaSource.res

@@ -68,6 +68,7 @@ let buildInstructionSelections = (
             isInner: ?cfg.isInner,
             includeTransaction: cfg.includeTransaction,
             includeLogs: cfg.includeLogs,
+            includeTokenBalances: cfg.includeTokenBalances,
           }: HyperSyncSolanaClient.QueryTypes.instructionSelection
         ),
       )
@@ -224,6 +225,16 @@ let toSvmTransaction = (
   version: ?tx.version,
 }
 
+let toSvmTokenBalance = (
+  tb: HyperSyncSolanaClient.ResponseTypes.tokenBalance,
+): Envio.svmTokenBalance => {
+  account: ?tb.account->Option.map(SvmTypes.Pubkey.fromStringUnsafe),
+  mint: ?tb.mint->Option.map(SvmTypes.Pubkey.fromStringUnsafe),
+  owner: ?tb.owner->Option.map(SvmTypes.Pubkey.fromStringUnsafe),
+  preAmount: ?tb.preAmount,
+  postAmount: ?tb.postAmount,
+}
+
 // Probe the discriminator byte-length ordering longest-first. Stops at the
 // first router hit. Falls back to the `_none` key (program-wide handler) when
 // no discriminator-keyed handler matches.
@@ -296,6 +307,8 @@ let make = ({chain, endpointUrl, apiToken, eventConfigs, clientMaxRetries, clien
   // startup; reused on every decoded instruction.
   let schemaHandlesByProgram = buildSchemaHandles(eventConfigs)
 
+  let needsTokenBalances = eventConfigs->Belt.Array.some(cfg => cfg.includeTokenBalances)
+
   let getItemsOrThrow = async (
     ~fromBlock,
     ~toBlock,
@@ -311,10 +324,22 @@ let make = ({chain, endpointUrl, apiToken, eventConfigs, clientMaxRetries, clien
     let pageFetchRef = Hrtime.makeTimer()
 
     let instructionSelections = buildInstructionSelections(eventConfigs)
+    let fields = if needsTokenBalances {
+      Some(
+        (
+          {
+            tokenBalance: [Slot, TransactionIndex, Account, Mint, Owner, PreAmount, PostAmount],
+          }: HyperSyncSolanaClient.QueryTypes.fieldSelection
+        ),
+      )
+    } else {
+      None
+    }
     let query: HyperSyncSolanaClient.query = {
       fromSlot: fromBlock,
       toSlot: ?toBlock,
       instructions: instructionSelections,
+      fields: ?fields,
     }
 
     Prometheus.SourceRequestCount.increment(
@@ -374,6 +399,21 @@ let make = ({chain, endpointUrl, apiToken, eventConfigs, clientMaxRetries, clien
       }
     })
 
+    let tokenBalancesByTx = Dict.make()
+    if needsTokenBalances {
+      resp.data.tokenBalances->Belt.Array.forEach(tb => {
+        switch tb.transactionIndex {
+        | Some(txIdx) =>
+          let key = tb.slot->Int.toString ++ ":" ++ txIdx->Int.toString
+          switch tokenBalancesByTx->Dict.get(key) {
+          | Some(existing) => existing->Array.push(tb)
+          | None => tokenBalancesByTx->Dict.set(key, [tb])
+          }
+        | None => ()
+        }
+      })
+    }
+
     let parsedQueueItems = []
     resp.data.instructions->Belt.Array.forEach(instr => {
       let programId = instr.programId->SvmTypes.Pubkey.fromStringUnsafe
@@ -399,6 +439,17 @@ let make = ({chain, endpointUrl, apiToken, eventConfigs, clientMaxRetries, clien
           instr.slot->Int.toString ++ ":" ++ instr.transactionIndex->Int.toString
         let maybeTx =
           txByKey->Utils.Dict.dangerouslyGetNonOption(txKey)->Option.map(toSvmTransaction)
+        let maybeTx = if eventConfig.includeTokenBalances {
+          maybeTx->Option.map(tx => {
+            let maybeBalances =
+              tokenBalancesByTx
+              ->Utils.Dict.dangerouslyGetNonOption(txKey)
+              ->Option.map(bals => bals->Array.map(toSvmTokenBalance))
+            {...tx, tokenBalances: ?maybeBalances}
+          })
+        } else {
+          maybeTx
+        }
         let logKey =
           instr.slot->Int.toString ++
           ":" ++

package/src/sources/HyperSyncSolanaSource.res.mjs

@@ -118,7 +118,8 @@ function buildInstructionSelections(eventConfigs) {
       a5: a5,
       isInner: cfg.isInner,
       includeTransaction: cfg.includeTransaction,
-      includeLogs: cfg.includeLogs
+      includeLogs: cfg.includeLogs,
+      includeTokenBalances: cfg.includeTokenBalances
     };
   });
 }
@@ -230,6 +231,16 @@ function toSvmTransaction(tx) {
   };
 }
 
+function toSvmTokenBalance(tb) {
+  return {
+    account: Stdlib_Option.map(tb.account, prim => prim),
+    mint: Stdlib_Option.map(tb.mint, prim => prim),
+    owner: Stdlib_Option.map(tb.owner, prim => prim),
+    preAmount: tb.preAmount,
+    postAmount: tb.postAmount
+  };
+}
+
 function probeRouter(router, programId, instr, byteLengthsDesc, contractAddress, indexingAddresses) {
   let probe = dN => {
     let tag = EventRouter.getSvmEventId(programId, dN);
@@ -279,15 +290,28 @@ function make(param) {
     orderingByProgram[o.programId] = o.byteLengthsDesc;
   });
   let schemaHandlesByProgram = buildSchemaHandles(eventConfigs);
+  let needsTokenBalances = Belt_Array.some(eventConfigs, cfg => cfg.includeTokenBalances);
   let getItemsOrThrow = async (fromBlock, toBlock, param, indexingAddresses, knownHeight, param$1, param$2, retry, logger) => {
     let totalTimeRef = Hrtime.makeTimer();
     let pageFetchRef = Hrtime.makeTimer();
     let instructionSelections = buildInstructionSelections(eventConfigs);
+    let fields = needsTokenBalances ? ({
+        tokenBalance: [
+          "slot",
+          "transaction_index",
+          "account",
+          "mint",
+          "owner",
+          "pre_amount",
+          "post_amount"
+        ]
+      }) : undefined;
     let query_instructions = instructionSelections;
     let query = {
       fromSlot: fromBlock,
       toSlot: toBlock,
-      instructions: query_instructions
+      instructions: query_instructions,
+      fields: fields
     };
     Prometheus.SourceRequestCount.increment(name, chain, "getInstructions");
     let resp;
@@ -335,6 +359,22 @@ function make(param) {
         logsByKey[key] = [log];
       }
     });
+    let tokenBalancesByTx = {};
+    if (needsTokenBalances) {
+      Belt_Array.forEach(resp.data.tokenBalances, tb => {
+        let txIdx = tb.transactionIndex;
+        if (txIdx === undefined) {
+          return;
+        }
+        let key = tb.slot.toString() + ":" + txIdx.toString();
+        let existing = tokenBalancesByTx[key];
+        if (existing !== undefined) {
+          existing.push(tb);
+        } else {
+          tokenBalancesByTx[key] = [tb];
+        }
+      });
+    }
     let parsedQueueItems = [];
     Belt_Array.forEach(resp.data.instructions, instr => {
       let programId = instr.programId;
@@ -344,6 +384,12 @@ function make(param) {
       if (maybeConfig !== undefined) {
         let txKey = instr.slot.toString() + ":" + instr.transactionIndex.toString();
         let maybeTx = Stdlib_Option.map(txByKey[txKey], toSvmTransaction);
+        let maybeTx$1 = maybeConfig.includeTokenBalances ? Stdlib_Option.map(maybeTx, tx => {
+            let maybeBalances = Stdlib_Option.map(tokenBalancesByTx[txKey], bals => bals.map(toSvmTokenBalance));
+            let newrecord = {...tx};
+            newrecord.tokenBalances = maybeBalances;
+            return newrecord;
+          }) : maybeTx;
         let logKey = instr.slot.toString() + ":" + instr.transactionIndex.toString() + ":" + serializeInstructionAddress(instr.instructionAddress);
         let maybeLogs = Stdlib_Option.map(logsByKey[logKey], logs => logs.map(log => ({
           kind: Stdlib_Option.getOr(log.kind, ""),
@@ -352,7 +398,7 @@ function make(param) {
         let payload_contractName = maybeConfig.contractName;
         let payload_eventName = maybeConfig.name;
         let payload_instruction = toSvmInstruction(instr, schemaHandlesByProgram);
-        let payload_transaction = maybeConfig.includeTransaction ? maybeTx : undefined;
+        let payload_transaction = maybeConfig.includeTransaction ? maybeTx$1 : undefined;
         let payload_logs = maybeConfig.includeLogs ? maybeLogs : undefined;
         let payload_slot = instr.slot;
         let payload_block = {
@@ -435,6 +481,7 @@ export {
   decodeIfPossible,
   toSvmInstruction,
   toSvmTransaction,
+  toSvmTokenBalance,
   probeRouter,
   make,
 }

package/svm.schema.json

@@ -287,6 +287,13 @@
             "null"
           ]
         },
+        "include_token_balances": {
+          "description": "When true, also fetch SPL Token / Token-2022 balance snapshots for the parent transaction. Implies include_transaction: true.",
+          "type": [
+            "boolean",
+            "null"
+          ]
+        },
         "accounts": {
           "description": "Optional positional account names. The Nth entry names account slot N on the dispatched instruction; surfaces as `event.instruction.decoded.accounts.<name>`. Accounts beyond the named list become `extra_accounts`.",
           "type": [