@sentio/sdk 2.39.7-rc.9 → 2.40.0-rc.1

package/src/eth/eth-plugin.ts

@@ -10,6 +10,8 @@ import {
   HandlerType,
   LogFilter,
   LogHandlerConfig,
+  PreparedData,
+  PreprocessResult,
   ProcessConfigResponse,
   ProcessResult,
   StartRequest
@@ -19,17 +21,24 @@ import { ServerError, Status } from 'nice-grpc'
 import { EthProcessorState } from './binds.js'
 import { AccountProcessorState } from './account-processor-state.js'
 import { ProcessorTemplateProcessorState } from './base-processor-template.js'
-import { GlobalProcessorState } from './base-processor.js'
+import { defaultPreprocessHandler, GlobalProcessorState } from './base-processor.js'
 import { validateAndNormalizeAddress } from './eth.js'
 import { EthChainId } from '@sentio/chain'
 import { EthContext } from './context.js'
 import { TemplateInstanceState } from '../core/template.js'
 
 interface Handlers {
-  eventHandlers: ((event: Data_EthLog) => Promise<ProcessResult>)[]
-  traceHandlers: ((trace: Data_EthTrace) => Promise<ProcessResult>)[]
-  blockHandlers: ((block: Data_EthBlock) => Promise<ProcessResult>)[]
-  transactionHandlers: ((trace: Data_EthTransaction) => Promise<ProcessResult>)[]
+  eventHandlers: ((event: Data_EthLog, preparedData?: PreparedData) => Promise<ProcessResult>)[]
+  traceHandlers: ((trace: Data_EthTrace, preparedData?: PreparedData) => Promise<ProcessResult>)[]
+  blockHandlers: ((block: Data_EthBlock, preparedData?: PreparedData) => Promise<ProcessResult>)[]
+  transactionHandlers: ((trace: Data_EthTransaction, preparedData?: PreparedData) => Promise<ProcessResult>)[]
+}
+
+interface PreprocessHandlers {
+  eventHandlers: ((event: Data_EthLog) => Promise<PreprocessResult>)[]
+  traceHandlers: ((trace: Data_EthTrace) => Promise<PreprocessResult>)[]
+  blockHandlers: ((block: Data_EthBlock) => Promise<PreprocessResult>)[]
+  transactionHandlers: ((txn: Data_EthTransaction) => Promise<PreprocessResult>)[]
 }
 
 export class EthPlugin extends Plugin {
@@ -40,6 +49,12 @@ export class EthPlugin extends Plugin {
     traceHandlers: [],
     transactionHandlers: []
   }
+  preprocessHandlers: PreprocessHandlers = {
+    blockHandlers: [],
+    eventHandlers: [],
+    traceHandlers: [],
+    transactionHandlers: []
+  }
 
   async configure(config: ProcessConfigResponse) {
     const handlers: Handlers = {
@@ -48,6 +63,12 @@ export class EthPlugin extends Plugin {
       traceHandlers: [],
       transactionHandlers: []
     }
+    const preprocessHandlers: PreprocessHandlers = {
+      blockHandlers: [],
+      eventHandlers: [],
+      traceHandlers: [],
+      transactionHandlers: []
+    }
 
     for (const processor of EthProcessorState.INSTANCE.getValues()) {
       // If server favor incremental update this need to change
@@ -69,6 +90,7 @@ export class EthPlugin extends Plugin {
 
       // Step 1. Prepare all the block handlers
       for (const blockHandler of processor.blockHandlers) {
+        preprocessHandlers.blockHandlers.push(blockHandler.preprocessHandler ?? defaultPreprocessHandler)
         const handlerId = handlers.blockHandlers.push(blockHandler.handler) - 1
         // TODO wrap the block handler into one
 
@@ -84,6 +106,7 @@ export class EthPlugin extends Plugin {
 
       // Step 2. Prepare all trace handlers
       for (const traceHandler of processor.traceHandlers) {
+        preprocessHandlers.traceHandlers.push(traceHandler.preprocessHandler ?? defaultPreprocessHandler)
         const handlerId = handlers.traceHandlers.push(traceHandler.handler) - 1
         for (const signature of traceHandler.signatures) {
           contractConfig.traceConfigs.push({
@@ -97,6 +120,7 @@ export class EthPlugin extends Plugin {
       // Step 3. Prepare all the event handlers
       for (const eventsHandler of processor.eventHandlers) {
         // associate id with filter
+        preprocessHandlers.eventHandlers.push(eventsHandler.preprocessHandler ?? defaultPreprocessHandler)
         const handlerId = handlers.eventHandlers.push(eventsHandler.handler) - 1
         const logConfig: LogHandlerConfig = {
           handlerId: handlerId,
@@ -192,6 +216,7 @@ export class EthPlugin extends Plugin {
       // TODO add interval
       for (const eventsHandler of processor.eventHandlers) {
         // associate id with filter
+        preprocessHandlers.eventHandlers.push(eventsHandler.preprocessHandler ?? defaultPreprocessHandler)
         const handlerId = handlers.eventHandlers.push(eventsHandler.handler) - 1
         const logConfig: LogHandlerConfig = {
           handlerId: handlerId,
@@ -232,21 +257,37 @@ export class EthPlugin extends Plugin {
     }
 
     this.handlers = handlers
+    this.preprocessHandlers = preprocessHandlers
   }
 
   supportedHandlers = [HandlerType.ETH_LOG, HandlerType.ETH_BLOCK, HandlerType.ETH_TRACE, HandlerType.ETH_TRANSACTION]
 
-  processBinding(request: DataBinding): Promise<ProcessResult> {
+  processBinding(request: DataBinding, preparedData: PreparedData | undefined): Promise<ProcessResult> {
     // return Promise.resolve(undefined);
     switch (request.handlerType) {
       case HandlerType.ETH_LOG:
-        return this.processLog(request)
+        return this.processLog(request, preparedData)
+      case HandlerType.ETH_TRACE:
+        return this.processTrace(request, preparedData)
+      case HandlerType.ETH_BLOCK:
+        return this.processBlock(request, preparedData)
+      case HandlerType.ETH_TRANSACTION:
+        return this.processTransaction(request, preparedData)
+      default:
+        throw new ServerError(Status.INVALID_ARGUMENT, 'No handle type registered ' + request.handlerType)
+    }
+  }
+
+  preprocessBinding(request: DataBinding): Promise<PreprocessResult> {
+    switch (request.handlerType) {
+      case HandlerType.ETH_LOG:
+        return this.preprocessLog(request)
       case HandlerType.ETH_TRACE:
-        return this.processTrace(request)
+        return this.preprocessTrace(request)
       case HandlerType.ETH_BLOCK:
-        return this.processBlock(request)
+        return this.preprocessBlock(request)
       case HandlerType.ETH_TRANSACTION:
-        return this.processTransaction(request)
+        return this.preprocessTransaction(request)
       default:
         throw new ServerError(Status.INVALID_ARGUMENT, 'No handle type registered ' + request.handlerType)
     }
@@ -285,7 +326,86 @@ export class EthPlugin extends Plugin {
     return TemplateInstanceState.INSTANCE.getValues().length !== config.templateInstances.length
   }
 
-  async processLog(request: DataBinding): Promise<ProcessResult> {
+  async preprocessLog(request: DataBinding): Promise<PreprocessResult> {
+    if (!request.data?.ethLog?.log) {
+      throw new ServerError(Status.INVALID_ARGUMENT, "Log can't be null")
+    }
+    const ethLog = request.data.ethLog
+
+    const promises: Promise<PreprocessResult>[] = []
+    for (const handlerId of request.handlerIds) {
+      const handler = this.preprocessHandlers.eventHandlers[handlerId]
+      const promise = handler(ethLog).catch((e) => {
+        throw new ServerError(
+          Status.INTERNAL,
+          'error processing log: ' + JSON.stringify(ethLog.log) + '\n' + errorString(e)
+        )
+      })
+      promises.push(promise)
+    }
+    return mergePreprocessResults(await Promise.all(promises))
+  }
+
+  async preprocessTrace(binding: DataBinding): Promise<PreprocessResult> {
+    if (!binding.data?.ethTrace?.trace) {
+      throw new ServerError(Status.INVALID_ARGUMENT, "Trace can't be null")
+    }
+    const ethTrace = binding.data.ethTrace
+
+    const promises: Promise<PreprocessResult>[] = []
+
+    for (const handlerId of binding.handlerIds) {
+      const promise = this.preprocessHandlers.traceHandlers[handlerId](ethTrace).catch((e) => {
+        throw new ServerError(
+          Status.INTERNAL,
+          'error processing trace: ' + JSON.stringify(ethTrace.trace) + '\n' + errorString(e)
+        )
+      })
+      promises.push(promise)
+    }
+    return mergePreprocessResults(await Promise.all(promises))
+  }
+
+  async preprocessBlock(binding: DataBinding): Promise<PreprocessResult> {
+    if (!binding.data?.ethBlock?.block) {
+      throw new ServerError(Status.INVALID_ARGUMENT, "Block can't be empty")
+    }
+    const ethBlock = binding.data.ethBlock
+
+    const promises: Promise<PreprocessResult>[] = []
+    for (const handlerId of binding.handlerIds) {
+      const promise = this.preprocessHandlers.blockHandlers[handlerId](ethBlock).catch((e) => {
+        throw new ServerError(
+          Status.INTERNAL,
+          'error processing block: ' + ethBlock.block?.number + '\n' + errorString(e)
+        )
+      })
+      promises.push(promise)
+    }
+    return mergePreprocessResults(await Promise.all(promises))
+  }
+
+  async preprocessTransaction(binding: DataBinding): Promise<PreprocessResult> {
+    if (!binding.data?.ethTransaction?.transaction) {
+      throw new ServerError(Status.INVALID_ARGUMENT, "transaction can't be null")
+    }
+    const ethTransaction = binding.data.ethTransaction
+
+    const promises: Promise<PreprocessResult>[] = []
+
+    for (const handlerId of binding.handlerIds) {
+      const promise = this.preprocessHandlers.transactionHandlers[handlerId](ethTransaction).catch((e) => {
+        throw new ServerError(
+          Status.INTERNAL,
+          'error processing transaction: ' + JSON.stringify(ethTransaction.transaction) + '\n' + errorString(e)
+        )
+      })
+      promises.push(promise)
+    }
+    return mergePreprocessResults(await Promise.all(promises))
+  }
+
+  async processLog(request: DataBinding, preparedData: PreparedData | undefined): Promise<ProcessResult> {
     if (!request.data?.ethLog?.log) {
       throw new ServerError(Status.INVALID_ARGUMENT, "Log can't be null")
     }
@@ -308,7 +428,7 @@ export class EthPlugin extends Plugin {
     return mergeProcessResults(await Promise.all(promises))
   }
 
-  async processTrace(binding: DataBinding): Promise<ProcessResult> {
+  async processTrace(binding: DataBinding, preparedData: PreparedData | undefined): Promise<ProcessResult> {
     if (!binding.data?.ethTrace?.trace) {
       throw new ServerError(Status.INVALID_ARGUMENT, "Trace can't be null")
     }
@@ -331,7 +451,7 @@ export class EthPlugin extends Plugin {
     return mergeProcessResults(await Promise.all(promises))
   }
 
-  async processBlock(binding: DataBinding): Promise<ProcessResult> {
+  async processBlock(binding: DataBinding, preparedData: PreparedData | undefined): Promise<ProcessResult> {
     if (!binding.data?.ethBlock?.block) {
       throw new ServerError(Status.INVALID_ARGUMENT, "Block can't be empty")
     }
@@ -353,7 +473,7 @@ export class EthPlugin extends Plugin {
     return mergeProcessResults(await Promise.all(promises))
   }
 
-  async processTransaction(binding: DataBinding): Promise<ProcessResult> {
+  async processTransaction(binding: DataBinding, preparedData: PreparedData | undefined): Promise<ProcessResult> {
     if (!binding.data?.ethTransaction?.transaction) {
       throw new ServerError(Status.INVALID_ARGUMENT, "transaction can't be null")
     }
@@ -388,3 +508,11 @@ class NoopContext extends EthContext {
     return ''
   }
 }
+
+function mergePreprocessResults(results: PreprocessResult[]): PreprocessResult {
+  const res: PreprocessResult = { ethCallParams: [] }
+  for (const r of results) {
+    res.ethCallParams = res.ethCallParams.concat(r.ethCallParams)
+  }
+  return res
+}

package/src/eth/eth.ts

@@ -71,7 +71,7 @@ export function transformEtherError(e: Error, ctx: ContractContext<any, any> | u
       }
       return new SimpleEthersError(msg, err, stack)
     } else {
-      return new SimpleEthersError(`eth call error, ${checkPage}:\n${err.message}`, err, stack)
+      return new SimpleEthersError(`eth call error, ${checkPage}:\n${JSON.stringify(err)}`, err, stack)
     }
   }
 

package/src/eth/index.ts

@@ -1,4 +1,4 @@
-export { BaseProcessor, GlobalProcessor } from './base-processor.js'
+export { BaseProcessor, GlobalProcessor, defaultPreprocessHandler } from './base-processor.js'
 export { GenericProcessor } from './generic-processor.js'
 export { BaseProcessorTemplate } from './base-processor-template.js'
 export { AccountProcessor } from './account-processor.js'
@@ -11,6 +11,6 @@ export { AccountContext, ContractContext, GlobalContext, ContractView, BoundCont
 
 export { EthPlugin } from './eth-plugin.js'
 
-export { EthFetchConfig } from '@sentio/protos'
+export { EthFetchConfig, PreprocessResult } from '@sentio/protos'
 
 export { EthChainId } from '@sentio/chain'

package/src/eth/provider.ts

@@ -4,9 +4,17 @@ import PQueue from 'p-queue'
 import { Endpoints } from '@sentio/runtime'
 import { EthChainId } from '@sentio/chain'
 import { LRUCache } from 'lru-cache'
+import { metrics } from '@opentelemetry/api'
 
 export const DummyProvider = new JsonRpcProvider('', Network.from(1))
 
+const meter = metrics.getMeter('processor_provider')
+const hit_count = meter.createCounter('provider_hit_count')
+const miss_count = meter.createCounter('provider_miss_count')
+const queue_size = meter.createGauge('provider_queue_size')
+const total_duration = meter.createCounter('provider_total_duration')
+const total_queued = meter.createCounter('provider_total_queued')
+
 const providers = new Map<string, JsonRpcProvider>()
 
 // export function getEthChainId(networkish?: EthContext | EthChainId): EthChainId {
@@ -97,10 +105,6 @@ class QueuedStaticJsonRpcProvider extends JsonRpcProvider {
     // return 1024
     // }
   })
-  hits = 0
-  misses = 0
-  totalDuration = 0
-  totalQueued = 0
 
   constructor(url: string, network: Network, concurrency: number, batchCount = 1) {
     // TODO re-enable match when possible
@@ -116,13 +120,14 @@ class QueuedStaticJsonRpcProvider extends JsonRpcProvider {
     const block = params[params.length - 1]
     let perform = this.#performCache.get(tag)
     if (!perform) {
-      this.misses++
+      miss_count.add(1)
       const queued: number = Date.now()
       perform = this.executor.add(() => {
         const started = Date.now()
-        this.totalQueued += started - queued
+        total_queued.add(started - queued)
+
         return super.send(method, params).finally(() => {
-          this.totalDuration += Date.now() - started
+          total_duration.add(Date.now() - started)
         })
       })
       perform.catch((e) => {
@@ -134,6 +139,8 @@ class QueuedStaticJsonRpcProvider extends JsonRpcProvider {
         }, 1000)
       })
 
+      queue_size.record(this.executor.size)
+
       this.#performCache.set(tag, perform)
       // For non latest block call, we cache permanently, otherwise we cache for one minute
       if (block === 'latest') {
@@ -144,7 +151,7 @@ class QueuedStaticJsonRpcProvider extends JsonRpcProvider {
         }, 60 * 1000)
       }
     } else {
-      this.hits++
+      hit_count.add(1)
     }
 
     const result = await perform
@@ -153,15 +160,4 @@ class QueuedStaticJsonRpcProvider extends JsonRpcProvider {
     }
     return result
   }
-
-  stats() {
-    const count = this.hits + this.misses
-    return {
-      queueSize: this.executor.size,
-      hits: this.hits,
-      misses: this.misses,
-      avgDuration: count == 0 ? 0 : this.totalDuration / count,
-      avgQueued: count == 0 ? 0 : this.totalQueued / count
-    }
-  }
 }

package/src/store/schema.ts

@@ -6,7 +6,7 @@ const customScalars = ['BigInt', 'BigDecimal', 'Timestamp', 'JSON', 'Bytes', 'ID
 
 const baseSchema = buildASTSchema(
   parse(`
-    directive @entity on OBJECT
+    directive @entity(immutable: Boolean! = false, sparse: Boolean! = false) on OBJECT
     directive @query on INTERFACE
     directive @derivedFrom(field: String!) on FIELD_DEFINITION
     directive @unique on FIELD_DEFINITION

package/src/store/store.ts

@@ -107,6 +107,12 @@ export class Store {
       for (const data of response.entityList?.entities || []) {
         yield this.newEntity(entity, data)
       }
+      console.debug(
+        'list returned ',
+        response.entityList?.entities?.length,
+        ' entities, next cursor',
+        response?.nextCursor
+      )
       if (!response.nextCursor) {
         break
       }
@@ -138,19 +144,22 @@ export class Store {
     cursor: string | undefined,
     pageSize?: number
   ): Promise<DBResponse> {
-    return (await this.context.sendRequest({
-      list: {
-        entity: getEntityName(entity),
-        cursor,
-        pageSize,
-        filters:
-          filters?.map((f) => ({
-            field: f.field as string,
-            op: ops[f.op],
-            value: { values: Array.isArray(f.value) ? f.value.map((v) => serialize(v)) : [serialize(f.value)] }
-          })) || []
-      }
-    })) as DBResponse
+    return (await this.context.sendRequest(
+      {
+        list: {
+          entity: getEntityName(entity),
+          cursor,
+          pageSize,
+          filters:
+            filters?.map((f) => ({
+              field: f.field as string,
+              op: ops[f.op],
+              value: { values: Array.isArray(f.value) ? f.value.map((v) => serialize(v)) : [serialize(f.value)] }
+            })) || []
+        }
+      },
+      60
+    )) as DBResponse
   }
 
   async list<T extends Entity, P extends keyof T, O extends Operators<T[P]>>(