@sentio/runtime 2.40.0-rc.1 → 2.40.0-rc.10

package/src/provider.ts

@@ -37,6 +37,8 @@ export function getProvider(chainId?: EthChainId): Provider {
 
   const address = Endpoints.INSTANCE.chainServer.get(chainId)
   const key = network.chainId.toString() + '-' + address
+
+  console.debug(`init provider for ${chainId}, address: ${address}`)
   let provider = providers.get(key)
 
   if (provider) {

package/src/service.ts

@@ -11,6 +11,8 @@ import {
   HandlerType,
   PreparedData,
   PreprocessResult,
+  PreprocessStreamRequest,
+  PreprocessStreamResponse,
   ProcessBindingResponse,
   ProcessBindingsRequest,
   ProcessConfigRequest,
@@ -23,7 +25,7 @@ import {
 } from '@sentio/protos'
 
 import { PluginManager } from './plugin.js'
-import { errorString, mergeProcessResults } from './utils.js'
+import { errorString, makeEthCallKey, mergeProcessResults } from './utils.js'
 import { freezeGlobalConfig, GLOBAL_CONFIG } from './global-config.js'
 
 import { StoreContext } from './db-context.js'
@@ -31,7 +33,7 @@ import { Subject } from 'rxjs'
 import { metrics } from '@opentelemetry/api'
 import { getProvider } from './provider.js'
 import { EthChainId } from '@sentio/chain'
-import { Provider, Interface } from 'ethers'
+import { Provider } from 'ethers'
 
 const meter = metrics.getMeter('processor_service')
 const process_binding_count = meter.createCounter('process_binding_count')
@@ -52,6 +54,8 @@ export class ProcessorServiceImpl implements ProcessorServiceImplementation {
 
   private readonly shutdownHandler?: () => void
 
+  private readonly preprocessedEthCalls: { [calldata: string]: any[] }
+
   constructor(loader: () => Promise<any>, shutdownHandler?: () => void) {
     this.loader = loader
     this.shutdownHandler = shutdownHandler
@@ -124,7 +128,7 @@ export class ProcessorServiceImpl implements ProcessorServiceImplementation {
   }
 
   async processBindings(request: ProcessBindingsRequest, options?: CallContext): Promise<ProcessBindingResponse> {
-    const ethCallResults = await this.preprocessBindings(request, options)
+    const ethCallResults = await this.preprocessBindings(request.bindings, undefined, options)
 
     const promises = []
     for (const binding of request.bindings) {
@@ -154,12 +158,14 @@ export class ProcessorServiceImpl implements ProcessorServiceImplementation {
   }
 
   async preprocessBindings(
-    request: ProcessBindingsRequest,
+    bindings: DataBinding[],
+    dbContext?: StoreContext,
     options?: CallContext
-  ): Promise<{ [calldata: string]: any[] }> {
+  ): Promise<{ [ethCallKey: string]: string }> {
+    console.debug(`preprocessBindings start, bindings: ${bindings.length}`)
     const promises = []
-    for (const binding of request.bindings) {
-      promises.push(this.preprocessBinding(binding, options))
+    for (const binding of bindings) {
+      promises.push(this.preprocessBinding(binding, dbContext, options))
     }
     let preprocessResults: PreprocessResult[]
     try {
@@ -167,14 +173,19 @@ export class ProcessorServiceImpl implements ProcessorServiceImplementation {
     } catch (e) {
       throw e
     }
+    console.debug(
+      'ethCallParams: ',
+      preprocessResults.map((r) => r.ethCallParams)
+    )
     const groupedRequests = new Map<string, EthCallParam[]>()
     const providers = new Map<string, Provider>()
     for (const result of preprocessResults) {
       for (const param of result.ethCallParams) {
-        if (!providers.has(param.chainId)) {
-          providers.set(param.chainId, getProvider(param.chainId as EthChainId))
+        const { chainId, address, blockTag } = param.context!
+        if (!providers.has(chainId)) {
+          providers.set(chainId, getProvider(chainId as EthChainId))
         }
-        const key = param.chainId + '|' + param.address
+        const key = [chainId, address, blockTag].join('|')
         if (!groupedRequests.has(key)) {
           groupedRequests.set(key, [])
         }
@@ -185,27 +196,37 @@ export class ProcessorServiceImpl implements ProcessorServiceImplementation {
     const start = Date.now()
     const callPromises = []
     for (const params of groupedRequests.values()) {
-      console.log(`chain: ${params[0].chainId}, address: ${params[0].address}, totalCalls: ${params.length}`)
+      const { chainId, address, blockTag } = params[0].context!
+      console.log(`chain: ${chainId}, address: ${address}, blockTag: ${blockTag}, totalCalls: ${params.length}`)
+      // TODO multicall
       for (const param of params) {
-        const frag = new Interface(param.signature)
-        const calldata = frag.encodeFunctionData(param.function, param.args)
         callPromises.push(
           providers
-            .get(param.chainId)!
+            .get(chainId)!
             .call({
-              to: param.address,
-              data: calldata
+              to: address,
+              data: param.calldata,
+              blockTag
             })
-            .then((ret) => [calldata, frag.decodeFunctionResult(param.function, ret).toArray()] as [string, any[]])
+            .then((result) => [makeEthCallKey(param), result])
         )
       }
     }
-    const results = Object.fromEntries(await Promise.all(callPromises))
+    let results = {}
+    try {
+      results = Object.fromEntries(await Promise.all(callPromises))
+    } catch (e) {
+      console.error(`eth call error: ${e}`)
+    }
     console.log(`${callPromises.length} calls finished, elapsed: ${Date.now() - start}ms`)
     return results
   }
 
-  async preprocessBinding(request: DataBinding, options?: CallContext): Promise<PreprocessResult> {
+  async preprocessBinding(
+    request: DataBinding,
+    dbContext?: StoreContext,
+    options?: CallContext
+  ): Promise<PreprocessResult> {
     if (!this.started) {
       throw new ServerError(Status.UNAVAILABLE, 'Service Not started.')
     }
@@ -221,7 +242,7 @@ export class ProcessorServiceImpl implements ProcessorServiceImplementation {
         ]
       )
     }
-    return await PluginManager.INSTANCE.preprocessBinding(request)
+    return await PluginManager.INSTANCE.preprocessBinding(request, dbContext)
   }
 
   async processBinding(
@@ -266,6 +287,63 @@ export class ProcessorServiceImpl implements ProcessorServiceImplementation {
     yield* from(subject).pipe(withAbort(context.signal))
   }
 
+  async handlePreprocessRequests(
+    requests: AsyncIterable<PreprocessStreamRequest>,
+    subject: Subject<DeepPartial<PreprocessStreamResponse>>
+  ) {
+    const contexts = new Contexts()
+
+    for await (const request of requests) {
+      try {
+        console.debug('received request:', request)
+        if (request.bindings) {
+          const bindings = request.bindings.bindings
+          const dbContext = contexts.new(request.processId, subject)
+          const start = Date.now()
+          this.preprocessBindings(bindings, dbContext)
+            .then(() => {
+              subject.next({
+                processId: request.processId
+              })
+            })
+            .catch((e) => {
+              console.debug(e)
+              dbContext.error(request.processId, e)
+            })
+            .finally(() => {
+              const cost = Date.now() - start
+              console.debug('preprocessBinding', request.processId, ' took', cost, 'ms')
+              contexts.delete(request.processId)
+            })
+        }
+        if (request.dbResult) {
+          const dbContext = contexts.get(request.processId)
+          dbContext?.result(request.dbResult)
+        }
+      } catch (e) {
+        // should not happen
+        console.error('unexpect error during handle loop', e)
+      }
+    }
+  }
+
+  async *preprocessBindingsStream(requests: AsyncIterable<PreprocessStreamRequest>, context: CallContext) {
+    if (!this.started) {
+      throw new ServerError(Status.UNAVAILABLE, 'Service Not started.')
+    }
+
+    const subject = new Subject<DeepPartial<PreprocessStreamResponse>>()
+    this.handlePreprocessRequests(requests, subject)
+      .then(() => {
+        subject.complete()
+      })
+      .catch((e) => {
+        console.error(e)
+        subject.error(e)
+      })
+    yield* from(subject).pipe(withAbort(context.signal))
+  }
+
   private async handleRequests(
     requests: AsyncIterable<ProcessStreamRequest>,
     subject: Subject<DeepPartial<ProcessStreamResponse>>

package/src/utils.ts

@@ -1,4 +1,4 @@
-import { ProcessResult } from '@sentio/protos'
+import { EthCallParam, ProcessResult } from '@sentio/protos'
 
 // TODO better handling this, because old proto doesn't have this
 import { StateResult, ProcessResult as ProcessResultFull } from './gen/processor/protos/processor.js'
@@ -8,7 +8,7 @@ import { Required } from 'utility-types'
 export function mergeProcessResults(results: ProcessResult[]): Required<ProcessResult, 'states'> {
   const res = {
     ...ProcessResultFull.create(),
-    states: StateResult.create(),
+    states: StateResult.create()
   }
 
   for (const r of results) {
@@ -17,7 +17,7 @@ export function mergeProcessResults(results: ProcessResult[]): Required<ProcessR
     res.events = res.events.concat(r.events)
     res.exports = res.exports.concat(r.exports)
     res.states = {
-      configUpdated: res.states?.configUpdated || r.states?.configUpdated || false,
+      configUpdated: res.states?.configUpdated || r.states?.configUpdated || false
     }
   }
   return res
@@ -28,3 +28,11 @@ export function errorString(e: Error): string {
 }
 
 export const USER_PROCESSOR = 'user_processor'
+
+export function makeEthCallKey(param: EthCallParam) {
+  if (!param.context) {
+    throw new Error('null context for eth call')
+  }
+  const { chainId, address, blockTag } = param.context
+  return `${chainId}|${address}|${blockTag}|${param.calldata}`
+}