@sentio/sdk 4.0.0-rc.1 → 4.0.0-rc.10

package/src/eth/eth.ts

@@ -23,8 +23,9 @@ import {
 import type { AccessList } from 'ethers/transaction'
 import { ContractContext } from './context.js'
 import { getAddress } from 'ethers/address'
-import { getBigInt } from 'ethers/utils'
-import { EthCallContext, EthCallParam } from '@sentio/protos'
+import { getBigInt, getNumber } from 'ethers/utils'
+import { type EthCallContext, type EthCallParam, EthCallParamSchema } from '@sentio/protos'
+import { create } from '@bufbuild/protobuf'
 import { ALL_ADDRESS } from '../core/index.js'
 
 export interface IResult {
@@ -126,14 +127,14 @@ class FormattedTransactionReceipt implements TransactionReceiptParams {
     return this.data.transactionHash
   }
   get index(): number {
-    return this.data.transactionIndex
+    return getNumber(this.data.transactionIndex)
   }
 
   get blockHash(): string {
     return this.data.blockHash
   }
   get blockNumber(): number {
-    return this.data.blockNumber
+    return getNumber(this.data.blockNumber)
   }
   get logsBloom(): string {
     return this.data.logsBloom
@@ -178,10 +179,10 @@ class FormattedTransactionReceipt implements TransactionReceiptParams {
     return this._l1Fee !== undefined ? this._l1Fee : (this._l1Fee = allowNull(getBigInt)(this.data.l1Fee))
   }
   get type(): number {
-    return this.data.type
+    return allowNull(getNumber, 0)(this.data.type)
   }
   get status(): number | null {
-    return this.data.status
+    return allowNull(getNumber)(this.data.status)
   }
   get root(): string | null {
     return this.data.root
@@ -197,7 +198,7 @@ export class FormattedLog implements LogParams {
     return this.raw.blockHash
   }
   get blockNumber(): number {
-    return this.raw.blockNumber
+    return getNumber(this.raw.blockNumber)
   }
   get removed(): boolean {
     return this.raw.removed
@@ -213,11 +214,14 @@ export class FormattedLog implements LogParams {
   }
 
   get transactionIndex(): number {
-    return this.raw.transactionIndex
+    // Raw RPC logs carry these as hex strings (e.g. "0xf3"); coerce so the value honors the
+    // declared `number` type and is valid for the int32 RecordMetaData fields it feeds.
+    // (Restores the getNumber coercion that ethers' formatLog applied before #1411.)
+    return getNumber(this.raw.transactionIndex)
   }
 
   get index(): number {
-    return this.raw.logIndex
+    return getNumber(this.raw.logIndex)
   }
 
   get data(): string {
@@ -240,11 +244,12 @@ function formatLog(log: any): LogParams {
 }
 
 export function formatEthData(data: {
-  log?: any
-  block?: any
-  trace?: any
-  transaction?: any
-  transactionReceipt?: any
+  // raw_* JSON payloads sent by the driver
+  rawLog?: string
+  rawBlock?: string
+  rawTrace?: string
+  rawTransaction?: string
+  rawTransactionReceipt?: string
   __formattedEthData?: any
 }) {
   // Return cached result if it exists
@@ -253,22 +258,29 @@ export function formatEthData(data: {
   }
 
   try {
-    const log = data.log ? formatLog(data.log) : undefined
-    if (data.block && !data.block.transactions) {
-      data.block.transactions = []
+    // All Eth* data is delivered as raw_* JSON payloads; parse them here.
+    const logObj = data.rawLog ? JSON.parse(data.rawLog) : undefined
+    const blockObj = data.rawBlock ? JSON.parse(data.rawBlock) : undefined
+    const traceObj = data.rawTrace ? JSON.parse(data.rawTrace) : undefined
+    const transactionObj = data.rawTransaction ? JSON.parse(data.rawTransaction) : undefined
+    const transactionReceiptObj = data.rawTransactionReceipt ? JSON.parse(data.rawTransactionReceipt) : undefined
+
+    const log = logObj ? formatLog(logObj) : undefined
+    if (blockObj && !blockObj.transactions) {
+      blockObj.transactions = []
     }
-    const block = data.block ? formatRichBlock(data.block) : undefined
-    const trace = data.trace ? (data.trace as Trace) : undefined
+    const block = blockObj ? formatRichBlock(blockObj) : undefined
+    const trace = traceObj ? (traceObj as Trace) : undefined
     let transaction = undefined
-    if (data.transaction) {
-      if (!data.transaction.v) {
-        data.transaction.v = '0x1c'
-        data.transaction.r = '0x88ff6cf0fefd94db46111149ae4bfc179e9b94721fffd821d38d16464b3f71d0'
-        data.transaction.s = '0x45e0aff800961cfce805daef7016b9b675c137a6a41a548f7b60a3484c06a33a'
+    if (transactionObj) {
+      if (!transactionObj.v) {
+        transactionObj.v = '0x1c'
+        transactionObj.r = '0x88ff6cf0fefd94db46111149ae4bfc179e9b94721fffd821d38d16464b3f71d0'
+        transactionObj.s = '0x45e0aff800961cfce805daef7016b9b675c137a6a41a548f7b60a3484c06a33a'
       }
-      transaction = formatTransactionResponse(data.transaction)
+      transaction = formatTransactionResponse(transactionObj)
     }
-    const transactionReceipt = data.transactionReceipt ? formatTransactionReceipt(data.transactionReceipt) : undefined
+    const transactionReceipt = transactionReceiptObj ? formatTransactionReceipt(transactionReceiptObj) : undefined
 
     const result = {
       log,
@@ -291,7 +303,7 @@ class FormattedTransactionResponse implements TransactionResponseParams {
   constructor(readonly raw: any) {}
 
   get blockNumber(): number | null {
-    return this.raw.blockNumber
+    return allowNull(getNumber)(this.raw.blockNumber)
   }
   _blockHash: string | undefined | null
   get blockHash(): string | null {
@@ -309,13 +321,15 @@ class FormattedTransactionResponse implements TransactionResponseParams {
     return this.raw.hash
   }
   get index(): number {
-    return this.raw.transactionIndex
+    // transactionIndex is absent for pending/partial txs; default to 0 (matches the prior
+    // `this.transaction.index || 0` handling) while still coercing hex strings when present.
+    return allowNull(getNumber, 0)(this.raw.transactionIndex)
   }
   get type(): number {
     if (this.raw.type === '0x' || this.raw.type == null) {
       return 0
     }
-    return this.raw.type
+    return getNumber(this.raw.type)
   }
   _to: string | null
   get to(): string | null {
@@ -332,7 +346,7 @@ class FormattedTransactionResponse implements TransactionResponseParams {
     return this._from ?? (this._from = getAddress(this.raw.from))
   }
   get nonce(): number {
-    return this.raw.nonce
+    return getNumber(this.raw.nonce)
   }
   _gasLimit: bigint
   get gasLimit(): bigint {
@@ -528,10 +542,10 @@ export function encodeCallData(
   try {
     const iface = new Interface([funcABI])
     const calldata = iface.encodeFunctionData(name, values)
-    return {
+    return create(EthCallParamSchema, {
       context,
       calldata
-    }
+    })
   } catch (e) {
     const stack = new Error().stack
     throw transformEtherError(e, undefined, stack)

package/src/eth/generic-processor.ts

@@ -2,7 +2,7 @@ import { BaseContract } from 'ethers'
 import { BoundContractView, ContractView } from './context.js'
 import { BindOptions } from './bind-options.js'
 import { BaseProcessor } from './base-processor.js'
-import { getProvider } from '@sentio/runtime'
+import { getProvider } from './provider.js'
 import { addProcessor } from './binds.js'
 
 export class GenericProcessor extends BaseProcessor<

package/src/eth/index.ts

@@ -3,7 +3,7 @@ export { GenericProcessor } from './generic-processor.js'
 export { BaseProcessorTemplate } from './base-processor-template.js'
 export { AccountProcessor } from './account-processor.js'
 
-export { getProvider, DummyProvider } from '@sentio/runtime'
+export { getProvider, DummyProvider } from './provider.js'
 
 export { EthContext } from './context.js'
 export * from './eth.js'
@@ -13,7 +13,7 @@ export { AccountContext, ContractContext, GlobalContext, ContractView, BoundCont
 
 export { EthPlugin } from './eth-plugin.js'
 
-export { EthFetchConfig, PreprocessResult } from '@sentio/protos'
+export type { EthFetchConfig, PreprocessResult } from '@sentio/protos'
 
 export { EthChainId } from '@sentio/chain'
 

package/src/eth/provider.ts

@@ -0,0 +1,193 @@
+import { JsonRpcProvider, Network } from 'ethers'
+
+import PQueue from 'p-queue'
+import { EthChainId } from '@sentio/chain'
+import { LRUCache } from 'lru-cache'
+import { Endpoints, providerMetrics, processMetrics, metricsStorage, GLOBAL_CONFIG } from '@sentio/runtime'
+const { miss_count, hit_count, queue_size } = providerMetrics
+
+export const DummyProvider = new JsonRpcProvider('', Network.from(1))
+
+const providers = new Map<string, JsonRpcProvider>()
+
+// export function getEthChainId(networkish?: EthContext | EthChainId): EthChainId {
+//   if (!networkish) {
+//     networkish = EthChainId.ETHEREUM
+//   }
+//   if (networkish instanceof BaseContext) {
+//     networkish = networkish.getChainId()
+//   }
+//   return networkish
+// }
+
+export function getProvider(chainId?: EthChainId): JsonRpcProvider {
+  // const network = getNetworkFromCtxOrNetworkish(networkish)
+  if (!chainId) {
+    chainId = EthChainId.ETHEREUM
+  }
+  const network = Network.from(parseInt(chainId))
+  // TODO check if other key needed
+
+  const address = Endpoints.INSTANCE.getChainRpcUrl(chainId)
+  const key = network.chainId.toString() + '-' + address
+
+  // console.debug(`init provider for ${chainId}, address: ${address}`)
+  let provider = providers.get(key)
+
+  if (provider) {
+    return provider
+  }
+
+  if (address === undefined) {
+    throw Error(
+      'Provider not found for chain ' +
+        network.chainId +
+        ', configured chains: ' +
+        [...Endpoints.INSTANCE.chainRpc.keys()].join(' ')
+    )
+  }
+  // console.log(
+  //   `init provider for chain ${network.chainId}, concurrency: ${Endpoints.INSTANCE.concurrency}, batchCount: ${Endpoints.INSTANCE.batchCount}`
+  // )
+  provider = new QueuedStaticJsonRpcProvider(
+    address,
+    network,
+    Endpoints.INSTANCE.concurrency,
+    Endpoints.INSTANCE.batchCount
+  )
+  providers.set(key, provider)
+  return provider
+}
+
+function getTag(prefix: string, value: any): string {
+  return (
+    prefix +
+    ':' +
+    JSON.stringify(value, (k, v) => {
+      if (v == null) {
+        return 'null'
+      }
+      if (typeof v === 'bigint') {
+        return `bigint:${v.toString()}`
+      }
+      if (typeof v === 'string') {
+        return v.toLowerCase()
+      }
+
+      // Sort object keys
+      if (typeof v === 'object' && !Array.isArray(v)) {
+        const keys = Object.keys(v)
+        keys.sort()
+        return keys.reduce(
+          (accum, key) => {
+            accum[key] = v[key]
+            return accum
+          },
+          <any>{}
+        )
+      }
+
+      return v
+    })
+  )
+}
+
+export class QueuedStaticJsonRpcProvider extends JsonRpcProvider {
+  executor: PQueue
+  #performCache = new LRUCache<string, Promise<any>>({
+    max: 300000, // 300k items
+    maxSize: 500 * 1024 * 1024 // 500mb key size for cache
+    // ttl: 1000 * 60 * 60, // 1 hour   no ttl for better performance
+    // sizeCalculation: (value: any) => {
+    // assume each item is 1kb for simplicity
+    // return 1024
+    // }
+  })
+  #retryCache = new LRUCache<string, number>({
+    max: 300000 // 300k items
+  })
+
+  constructor(url: string, network: Network, concurrency: number, batchCount = 1) {
+    // TODO re-enable match when possible
+    super(url, network, { staticNetwork: network, batchMaxCount: batchCount })
+    this.executor = new PQueue({ concurrency: concurrency })
+  }
+
+  async send(method: string, params: Array<any>): Promise<any> {
+    if (method !== 'eth_call' || params.length > 2) {
+      return await this.executor.add(() => super.send(method, params))
+    }
+    const tag = getTag(method, params)
+    const block = params[params.length - 1]
+    let perform = this.#performCache.get(tag)
+    if (!perform) {
+      miss_count.add(1)
+      const handler = metricsStorage.getStore()
+      const queued: number = Date.now()
+      perform = this.executor.add(() => {
+        const started = Date.now()
+        processMetrics.processor_rpc_queue_duration.record(started - queued, {
+          chain_id: this._network.chainId.toString(),
+          handler
+        })
+
+        let success = true
+        return super
+          .send(method, params)
+          .catch((e) => {
+            success = false
+            throw e
+          })
+          .finally(() => {
+            processMetrics.processor_rpc_duration.record(Date.now() - started, {
+              chain_id: this._network.chainId.toString(),
+              handler,
+              success
+            })
+          })
+      })
+
+      queue_size.record(this.executor.size)
+
+      this.#performCache.set(tag, perform, {
+        size: tag.length
+      })
+      // For non latest block call, we cache permanently, otherwise we cache for one minute
+      if (block === 'latest') {
+        setTimeout(() => {
+          if (this.#performCache.get(tag) === perform) {
+            this.#performCache.delete(tag)
+          }
+        }, 60 * 1000)
+      }
+    } else {
+      hit_count.add(1)
+    }
+
+    let result
+    try {
+      result = await perform
+    } catch (e: any) {
+      this.#performCache.delete(tag)
+      if (e.code === 'TIMEOUT') {
+        let retryCount = this.#retryCache.get(tag)
+        if (GLOBAL_CONFIG.execution.rpcRetryTimes && retryCount === undefined) {
+          retryCount = GLOBAL_CONFIG.execution.rpcRetryTimes
+        }
+        if (retryCount) {
+          this.#retryCache.set(tag, retryCount - 1)
+          return this.send(method, params)
+        }
+      }
+      throw e
+    }
+    if (!result) {
+      throw Error('Unexpected null response')
+    }
+    return result
+  }
+
+  toString() {
+    return 'QueuedStaticJsonRpcProvider'
+  }
+}

package/src/fuel/asset-processor.ts

@@ -1,5 +1,11 @@
 import { CallHandler, FuelBaseProcessor, FuelProcessorState } from './types.js'
-import { Data_FuelReceipt, Data_FuelTransaction, FuelAssetHandlerConfig_AssetFilter } from '@sentio/protos'
+import {
+  Data_FuelTransaction,
+  FuelAssetHandlerConfig_AssetFilter,
+  FuelAssetHandlerConfig_AssetFilterSchema,
+  timestampDate
+} from '@sentio/protos'
+import { create } from '@bufbuild/protobuf'
 import { FuelNetwork, getProvider } from './network.js'
 import { FuelContext } from './context.js'
 import { decodeFuelTransaction } from './transaction.js'
@@ -8,7 +14,7 @@ import { getOptionsSignature } from './fuel-processor.js'
 import { getHandlerName, proxyProcessor } from '../utils/metrics.js'
 
 export class FuelAssetProcessor implements FuelBaseProcessor<FuelAssetProcessorConfig> {
-  txHandlers: CallHandler<Data_FuelTransaction | Data_FuelReceipt>[] = []
+  txHandlers: CallHandler<Data_FuelTransaction>[] = []
   blockHandlers = []
   private provider: Provider
 
@@ -42,11 +48,13 @@ export class FuelAssetProcessor implements FuelBaseProcessor<FuelAssetProcessorC
     for (const assetId of assetIds) {
       for (const from of froms) {
         for (const to of tos) {
-          filters.push({
-            assetId: assetId,
-            fromAddress: from,
-            toAddress: to
-          })
+          filters.push(
+            create(FuelAssetHandlerConfig_AssetFilterSchema, {
+              assetId: assetId,
+              fromAddress: from,
+              toAddress: to
+            })
+          )
         }
       }
     }
@@ -94,7 +102,7 @@ export class FuelAssetProcessor implements FuelBaseProcessor<FuelAssetProcessorC
           this.config.chainId,
           assetId,
           this.config.name ?? '',
-          call.timestamp || new Date(0),
+          call.timestamp ? timestampDate(call.timestamp) : new Date(0),
           tx,
           null
         )

package/src/fuel/context.ts

@@ -1,6 +1,7 @@
 import { BaseContext, Labels, normalizeLabels } from '../core/index.js'
 import { FuelChainId } from '@sentio/chain'
-import { RecordMetaData } from '@sentio/protos'
+import { RecordMetaData, RecordMetaDataSchema } from '@sentio/protos'
+import { create } from '@bufbuild/protobuf'
 import type { CallResult, Contract } from 'fuels'
 import { InvocationScopeLike } from 'fuels'
 import { FuelBlock, FuelLog, FuelTransaction } from './types.js'
@@ -39,7 +40,7 @@ export class FuelContext extends BaseContext {
   }
 
   protected getMetaDataInternal(name: string, labels: Labels): RecordMetaData {
-    return {
+    return create(RecordMetaDataSchema, {
       address: this.contractAddress,
       contractName: this.contractName,
       blockNumber: BigInt(this.block?.height?.toString() ?? (this.transaction?.blockNumber || 0)),
@@ -49,7 +50,7 @@ export class FuelContext extends BaseContext {
       name: name,
       logIndex: this.logIndex,
       labels: normalizeLabels(labels)
-    }
+    })
   }
 }