@sentio/runtime 2.40.0-rc.2 → 2.40.0-rc.21

package/src/plugin.ts

@@ -28,7 +28,7 @@ export abstract class Plugin {
     return ProcessResult.create()
   }
 
-  async preprocessBinding(request: DataBinding): Promise<PreprocessResult> {
+  async preprocessBinding(request: DataBinding, preprocessStore: { [k: string]: any }): Promise<PreprocessResult> {
     return PreprocessResult.create()
   }
 }
@@ -84,13 +84,17 @@ export class PluginManager {
     })
   }
 
-  preprocessBinding(request: DataBinding, dbContext?: StoreContext): Promise<PreprocessResult> {
+  preprocessBinding(
+    request: DataBinding,
+    preprocessStore: { [k: string]: any },
+    dbContext?: StoreContext
+  ): Promise<PreprocessResult> {
     const plugin = this.typesToPlugin.get(request.handlerType)
     if (!plugin) {
       throw new Error(`No plugin for ${request.handlerType}`)
     }
     return this.dbContextLocalStorage.run(dbContext, () => {
-      return plugin.preprocessBinding(request)
+      return plugin.preprocessBinding(request, preprocessStore)
     })
   }
 }

package/src/processor-runner.ts

@@ -58,7 +58,9 @@ const optionDefinitions = [
 
 const options = commandLineArgs(optionDefinitions, { partial: true })
 
-setupLogger(options['log-format'] === 'json', options.debug)
+const logLevel = process.env['LOG_LEVEL']?.toUpperCase()
+
+setupLogger(options['log-format'] === 'json', logLevel === 'debug' ? true : options.debug)
 console.debug('Starting with', options.target)
 
 Error.stackTraceLimit = 20

package/src/provider.ts

@@ -4,17 +4,10 @@ import PQueue from 'p-queue'
 import { Endpoints } from './endpoints.js'
 import { EthChainId } from '@sentio/chain'
 import { LRUCache } from 'lru-cache'
-import { metrics } from '@opentelemetry/api'
-
+import { providerMetrics } from './metrics.js'
+const { miss_count, hit_count, total_duration, total_queued, queue_size } = providerMetrics
 export const DummyProvider = new JsonRpcProvider('', Network.from(1))
 
-const meter = metrics.getMeter('processor_provider')
-const hit_count = meter.createCounter('provider_hit_count')
-const miss_count = meter.createCounter('provider_miss_count')
-const queue_size = meter.createGauge('provider_queue_size')
-const total_duration = meter.createCounter('provider_total_duration')
-const total_queued = meter.createCounter('provider_total_queued')
-
 const providers = new Map<string, JsonRpcProvider>()
 
 // export function getEthChainId(networkish?: EthContext | EthChainId): EthChainId {
@@ -37,6 +30,8 @@ export function getProvider(chainId?: EthChainId): Provider {
 
   const address = Endpoints.INSTANCE.chainServer.get(chainId)
   const key = network.chainId.toString() + '-' + address
+
+  console.debug(`init provider for ${chainId}, address: ${address}`)
   let provider = providers.get(key)
 
   if (provider) {

package/src/service.ts

@@ -25,20 +25,16 @@ import {
 } from '@sentio/protos'
 
 import { PluginManager } from './plugin.js'
-import { errorString, mergeProcessResults } from './utils.js'
+import { errorString, makeEthCallKey, mergeProcessResults } from './utils.js'
 import { freezeGlobalConfig, GLOBAL_CONFIG } from './global-config.js'
 
 import { StoreContext } from './db-context.js'
 import { Subject } from 'rxjs'
-import { metrics } from '@opentelemetry/api'
 import { getProvider } from './provider.js'
 import { EthChainId } from '@sentio/chain'
-import { Provider, Interface } from 'ethers'
-
-const meter = metrics.getMeter('processor_service')
-const process_binding_count = meter.createCounter('process_binding_count')
-const process_binding_time = meter.createCounter('process_binding_time')
-const process_binding_error = meter.createCounter('process_binding_error')
+import { Provider } from 'ethers'
+import { processMetrics, providerMetrics, dbMetrics } from './metrics.js'
+const { process_binding_count, process_binding_time, process_binding_error } = processMetrics
 
 ;(BigInt.prototype as any).toJSON = function () {
   return this.toString()
@@ -54,6 +50,8 @@ export class ProcessorServiceImpl implements ProcessorServiceImplementation {
 
   private readonly shutdownHandler?: () => void
 
+  private preparedData: PreparedData | undefined
+
   constructor(loader: () => Promise<any>, shutdownHandler?: () => void) {
     this.loader = loader
     this.shutdownHandler = shutdownHandler
@@ -126,11 +124,11 @@ export class ProcessorServiceImpl implements ProcessorServiceImplementation {
   }
 
   async processBindings(request: ProcessBindingsRequest, options?: CallContext): Promise<ProcessBindingResponse> {
-    const ethCallResults = await this.preprocessBindings(request.bindings, undefined, options)
+    const preparedData = await this.preprocessBindings(request.bindings, {}, undefined, options)
 
     const promises = []
     for (const binding of request.bindings) {
-      const promise = this.processBinding(binding, { ethCallResults })
+      const promise = this.processBinding(binding, preparedData)
       if (GLOBAL_CONFIG.execution.sequential) {
         await promise
       }
@@ -157,13 +155,14 @@ export class ProcessorServiceImpl implements ProcessorServiceImplementation {
 
   async preprocessBindings(
     bindings: DataBinding[],
+    preprocessStore: { [k: string]: any },
     dbContext?: StoreContext,
     options?: CallContext
-  ): Promise<{ [calldata: string]: any[] }> {
-    console.log('preprocessBindings start')
+  ): Promise<PreparedData> {
+    console.debug(`preprocessBindings start, bindings: ${bindings.length}`)
     const promises = []
     for (const binding of bindings) {
-      promises.push(this.preprocessBinding(binding, dbContext, options))
+      promises.push(this.preprocessBinding(binding, preprocessStore, dbContext, options))
     }
     let preprocessResults: PreprocessResult[]
     try {
@@ -171,18 +170,15 @@ export class ProcessorServiceImpl implements ProcessorServiceImplementation {
     } catch (e) {
       throw e
     }
-    console.log(
-      'ethCallParams: ',
-      preprocessResults.map((r) => r.ethCallParams)
-    )
     const groupedRequests = new Map<string, EthCallParam[]>()
     const providers = new Map<string, Provider>()
     for (const result of preprocessResults) {
       for (const param of result.ethCallParams) {
-        if (!providers.has(param.chainId)) {
-          providers.set(param.chainId, getProvider(param.chainId as EthChainId))
+        const { chainId, address, blockTag } = param.context!
+        if (!providers.has(chainId)) {
+          providers.set(chainId, getProvider(chainId as EthChainId))
         }
-        const key = param.chainId + '|' + param.address
+        const key = [chainId, address, blockTag].join('|')
         if (!groupedRequests.has(key)) {
           groupedRequests.set(key, [])
         }
@@ -193,28 +189,36 @@ export class ProcessorServiceImpl implements ProcessorServiceImplementation {
     const start = Date.now()
     const callPromises = []
     for (const params of groupedRequests.values()) {
-      console.log(`chain: ${params[0].chainId}, address: ${params[0].address}, totalCalls: ${params.length}`)
+      const { chainId, address, blockTag } = params[0].context!
+      // TODO multicall
       for (const param of params) {
-        const frag = new Interface(param.signature)
-        const calldata = frag.encodeFunctionData(param.function, param.args)
         callPromises.push(
           providers
-            .get(param.chainId)!
+            .get(chainId)!
             .call({
-              to: param.address,
-              data: calldata
+              to: address,
+              data: param.calldata,
+              blockTag
             })
-            .then((ret) => [calldata, frag.decodeFunctionResult(param.function, ret).toArray()] as [string, any[]])
+            .then((result) => [makeEthCallKey(param), result])
         )
       }
     }
-    const results = Object.fromEntries(await Promise.all(callPromises))
+    let results: { [p: string]: string } = {}
+    try {
+      results = Object.fromEntries(await Promise.all(callPromises))
+    } catch (e) {
+      console.error(`eth call error: ${e}`)
+    }
     console.log(`${callPromises.length} calls finished, elapsed: ${Date.now() - start}ms`)
-    return results
+    return {
+      ethCallResults: results
+    }
   }
 
   async preprocessBinding(
     request: DataBinding,
+    preprocessStore: { [k: string]: any },
     dbContext?: StoreContext,
     options?: CallContext
   ): Promise<PreprocessResult> {
@@ -233,7 +237,7 @@ export class ProcessorServiceImpl implements ProcessorServiceImplementation {
         ]
       )
     }
-    return await PluginManager.INSTANCE.preprocessBinding(request, dbContext)
+    return await PluginManager.INSTANCE.preprocessBinding(request, preprocessStore, dbContext)
   }
 
   async processBinding(
@@ -269,6 +273,7 @@ export class ProcessorServiceImpl implements ProcessorServiceImplementation {
     const subject = new Subject<DeepPartial<ProcessStreamResponse>>()
     this.handleRequests(requests, subject)
       .then(() => {
+        this.preparedData = { ethCallResults: {} }
         subject.complete()
       })
       .catch((e) => {
@@ -283,16 +288,23 @@ export class ProcessorServiceImpl implements ProcessorServiceImplementation {
     subject: Subject<DeepPartial<PreprocessStreamResponse>>
   ) {
     const contexts = new Contexts()
+    const preprocessStore: { [k: string]: any } = {}
 
     for await (const request of requests) {
       try {
-        console.debug('received request:', request)
         if (request.bindings) {
           const bindings = request.bindings.bindings
           const dbContext = contexts.new(request.processId, subject)
           const start = Date.now()
-          this.preprocessBindings(bindings, dbContext)
-            .then(() => {
+          this.preprocessBindings(bindings, preprocessStore, dbContext, undefined)
+            .then((preparedData) => {
+              // TODO maybe not proper to pass data in this way
+              this.preparedData = {
+                ethCallResults: {
+                  ...this.preparedData?.ethCallResults,
+                  ...preparedData.ethCallResults
+                }
+              }
               subject.next({
                 processId: request.processId
               })
@@ -300,12 +312,10 @@ export class ProcessorServiceImpl implements ProcessorServiceImplementation {
             .catch((e) => {
               console.debug(e)
               dbContext.error(request.processId, e)
-              process_binding_error.add(1)
             })
             .finally(() => {
               const cost = Date.now() - start
               console.debug('preprocessBinding', request.processId, ' took', cost, 'ms')
-              process_binding_time.add(cost)
               contexts.delete(request.processId)
             })
         }
@@ -351,7 +361,7 @@ export class ProcessorServiceImpl implements ProcessorServiceImplementation {
           const binding = request.binding
           const dbContext = contexts.new(request.processId, subject)
           const start = Date.now()
-          PluginManager.INSTANCE.processBinding(binding, undefined, dbContext)
+          PluginManager.INSTANCE.processBinding(binding, this.preparedData, dbContext)
             .then((result) => {
               subject.next({
                 result,
@@ -369,6 +379,8 @@ export class ProcessorServiceImpl implements ProcessorServiceImplementation {
               console.debug('processBinding', request.processId, ' took', cost, 'ms')
               process_binding_time.add(cost)
               contexts.delete(request.processId)
+              console.debug('db stats', JSON.stringify(dbMetrics.stats()))
+              console.debug('provider stats', JSON.stringify(providerMetrics.stats()))
             })
         }
         if (request.dbResult) {

package/src/utils.ts

@@ -1,4 +1,4 @@
-import { ProcessResult } from '@sentio/protos'
+import { EthCallParam, ProcessResult } from '@sentio/protos'
 
 // TODO better handling this, because old proto doesn't have this
 import { StateResult, ProcessResult as ProcessResultFull } from './gen/processor/protos/processor.js'
@@ -8,7 +8,7 @@ import { Required } from 'utility-types'
 export function mergeProcessResults(results: ProcessResult[]): Required<ProcessResult, 'states'> {
   const res = {
     ...ProcessResultFull.create(),
-    states: StateResult.create(),
+    states: StateResult.create()
   }
 
   for (const r of results) {
@@ -17,7 +17,7 @@ export function mergeProcessResults(results: ProcessResult[]): Required<ProcessR
     res.events = res.events.concat(r.events)
     res.exports = res.exports.concat(r.exports)
     res.states = {
-      configUpdated: res.states?.configUpdated || r.states?.configUpdated || false,
+      configUpdated: res.states?.configUpdated || r.states?.configUpdated || false
     }
   }
   return res
@@ -28,3 +28,11 @@ export function errorString(e: Error): string {
 }
 
 export const USER_PROCESSOR = 'user_processor'
+
+export function makeEthCallKey(param: EthCallParam) {
+  if (!param.context) {
+    throw new Error('null context for eth call')
+  }
+  const { chainId, address, blockTag } = param.context
+  return `${chainId}|${address}|${blockTag}|${param.calldata}`.toLowerCase()
+}