@sentio/runtime 2.40.0-rc.9 → 2.40.1-rc.1

package/src/otlp.ts

@@ -0,0 +1,50 @@
+import { envDetector } from '@opentelemetry/resources'
+import { MeterProvider, PeriodicExportingMetricReader } from '@opentelemetry/sdk-metrics'
+import { OTLPMetricExporter } from '@opentelemetry/exporter-metrics-otlp-grpc'
+import { PrometheusExporter } from '@opentelemetry/exporter-prometheus'
+import { NodeTracerProvider } from '@opentelemetry/sdk-trace-node'
+import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-grpc'
+import { BatchSpanProcessor } from '@opentelemetry/sdk-trace-base'
+import { metrics, trace } from '@opentelemetry/api'
+
+export async function setupOTLP() {
+  const resource = await envDetector.detect()
+
+  const meterProvider = new MeterProvider({
+    resource,
+    readers: [
+      new PeriodicExportingMetricReader({
+        exporter: new OTLPMetricExporter()
+      }),
+      new PrometheusExporter({
+        // http://localhost:4041/metrics
+        port: 4041
+      })
+    ]
+  })
+
+  const traceProvider = new NodeTracerProvider({
+    resource: resource
+  })
+  const exporter = new OTLPTraceExporter() // new ConsoleSpanExporter();
+  const processor = new BatchSpanProcessor(exporter)
+  traceProvider.addSpanProcessor(processor)
+  traceProvider.register()
+
+  metrics.setGlobalMeterProvider(meterProvider)
+  trace.setGlobalTracerProvider(traceProvider)
+  ;['SIGINT', 'SIGTERM'].forEach((signal) => {
+    process.on(signal as any, () => shutdownProvider())
+  })
+}
+
+export async function shutdownProvider() {
+  const traceProvider = trace.getTracerProvider()
+  if (traceProvider instanceof NodeTracerProvider) {
+    traceProvider.shutdown().catch(console.error)
+  }
+  const meterProvider = metrics.getMeterProvider()
+  if (meterProvider instanceof MeterProvider) {
+    meterProvider.shutdown().catch(console.error)
+  }
+}

package/src/plugin.ts

@@ -28,7 +28,7 @@ export abstract class Plugin {
     return ProcessResult.create()
   }
 
-  async preprocessBinding(request: DataBinding): Promise<PreprocessResult> {
+  async preprocessBinding(request: DataBinding, preprocessStore: { [k: string]: any }): Promise<PreprocessResult> {
     return PreprocessResult.create()
   }
 }
@@ -84,13 +84,17 @@ export class PluginManager {
     })
   }
 
-  preprocessBinding(request: DataBinding, dbContext?: StoreContext): Promise<PreprocessResult> {
+  preprocessBinding(
+    request: DataBinding,
+    preprocessStore: { [k: string]: any },
+    dbContext?: StoreContext
+  ): Promise<PreprocessResult> {
     const plugin = this.typesToPlugin.get(request.handlerType)
     if (!plugin) {
       throw new Error(`No plugin for ${request.handlerType}`)
     }
     return this.dbContextLocalStorage.run(dbContext, () => {
-      return plugin.preprocessBinding(request)
+      return plugin.preprocessBinding(request, preprocessStore)
     })
   }
 }

package/src/processor-runner.ts

@@ -7,9 +7,9 @@ import { compressionAlgorithms } from '@grpc/grpc-js'
 import commandLineArgs from 'command-line-args'
 import { createServer } from 'nice-grpc'
 import { errorDetailsServerMiddleware } from 'nice-grpc-error-details'
-import { registry as niceGrpcRegistry, prometheusServerMiddleware } from 'nice-grpc-prometheus'
+// import { registry as niceGrpcRegistry } from 'nice-grpc-prometheus'
 import { openTelemetryServerMiddleware } from 'nice-grpc-opentelemetry'
-import { register as globalRegistry, Registry } from 'prom-client'
+// import { register as globalRegistry, Registry } from 'prom-client'
 import http from 'http'
 // @ts-ignore inspector promises is not included in @type/node
 import { Session } from 'node:inspector/promises'
@@ -21,22 +21,17 @@ import { FullProcessorServiceImpl } from './full-service.js'
 import { ChainConfig } from './chain-config.js'
 import { setupLogger } from './logger.js'
 
-import { NodeSDK } from '@opentelemetry/sdk-node'
+// import { NodeSDK } from '@opentelemetry/sdk-node'
+import { envDetector } from '@opentelemetry/resources'
 import { OTLPMetricExporter } from '@opentelemetry/exporter-metrics-otlp-grpc'
 import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-grpc'
-import { PeriodicExportingMetricReader } from '@opentelemetry/sdk-metrics'
+import { PeriodicExportingMetricReader, MeterProvider } from '@opentelemetry/sdk-metrics'
+import { BatchSpanProcessor } from '@opentelemetry/sdk-trace-base'
+import { NodeTracerProvider } from '@opentelemetry/sdk-trace-node'
+import { diag, DiagConsoleLogger, DiagLogLevel, metrics, trace } from '@opentelemetry/api'
+import { PrometheusExporter } from '@opentelemetry/exporter-prometheus'
 
-const sdk = new NodeSDK({
-  traceExporter: new OTLPTraceExporter(),
-  metricReader: new PeriodicExportingMetricReader({
-    exporter: new OTLPMetricExporter()
-  })
-  // instrumentations: [getNodeAutoInstrumentations()],
-})
-
-sdk.start()
-
-const mergedRegistry = Registry.merge([globalRegistry, niceGrpcRegistry])
+// const mergedRegistry = Registry.merge([globalRegistry, niceGrpcRegistry])
 
 const optionDefinitions = [
   { name: 'target', type: String, defaultOption: true },
@@ -58,9 +53,57 @@ const optionDefinitions = [
 
 const options = commandLineArgs(optionDefinitions, { partial: true })
 
-setupLogger(options['log-format'] === 'json', options.debug)
+const logLevel = process.env['LOG_LEVEL']?.toUpperCase()
+
+setupLogger(options['log-format'] === 'json', logLevel === 'debug' ? true : options.debug)
 console.debug('Starting with', options.target)
 
+if (options.debug) {
+  diag.setLogger(new DiagConsoleLogger(), DiagLogLevel.DEBUG)
+}
+
+const resource = await envDetector.detect()
+
+const meterProvider = new MeterProvider({
+  resource,
+  readers: [
+    new PeriodicExportingMetricReader({
+      exporter: new OTLPMetricExporter()
+    }),
+    new PrometheusExporter({
+      // http://localhost:4041/metrics
+      port: 4041
+    })
+  ]
+})
+
+const traceProvider = new NodeTracerProvider({
+  resource: resource
+})
+const exporter = new OTLPTraceExporter() // new ConsoleSpanExporter();
+const processor = new BatchSpanProcessor(exporter)
+traceProvider.addSpanProcessor(processor)
+traceProvider.register()
+
+metrics.setGlobalMeterProvider(meterProvider)
+trace.setGlobalTracerProvider(traceProvider)
+;['SIGINT', 'SIGTERM'].forEach((signal) => {
+  process.on(signal as any, () => shutdownProvider())
+})
+
+export async function shutdownProvider() {
+  const traceProvider = trace.getTracerProvider()
+  if (traceProvider instanceof NodeTracerProvider) {
+    traceProvider.shutdown().catch(console.error)
+  }
+  const meterProvider = metrics.getMeterProvider()
+  if (meterProvider instanceof MeterProvider) {
+    meterProvider.shutdown().catch(console.error)
+  }
+}
+
+metrics.getMeter('processor').createGauge('up').record(1)
+
 Error.stackTraceLimit = 20
 
 const fullPath = path.resolve(options['chains-config'])
@@ -99,7 +142,7 @@ const server = createServer({
   'grpc.max_receive_message_length': 384 * 1024 * 1024,
   'grpc.default_compression_algorithm': compressionAlgorithms.gzip
 })
-  .use(prometheusServerMiddleware())
+  // .use(prometheusServerMiddleware())
   .use(openTelemetryServerMiddleware())
   .use(errorDetailsServerMiddleware)
 const baseService = new ProcessorServiceImpl(async () => {
@@ -122,10 +165,10 @@ const httpServer = http
       const reqUrl = new URL(req.url, `http://${req.headers.host}`)
       const queries = reqUrl.searchParams
       switch (reqUrl.pathname) {
-        case '/metrics':
-          const metrics = await mergedRegistry.metrics()
-          res.write(metrics)
-          break
+        // case '/metrics':
+        //   const metrics = await mergedRegistry.metrics()
+        //   res.write(metrics)
+        //   break
         case '/profile': {
           try {
             const profileTime = parseInt(queries.get('t') || '1000', 10) || 1000

package/src/provider.ts

@@ -4,17 +4,11 @@ import PQueue from 'p-queue'
 import { Endpoints } from './endpoints.js'
 import { EthChainId } from '@sentio/chain'
 import { LRUCache } from 'lru-cache'
-import { metrics } from '@opentelemetry/api'
+import { providerMetrics } from './metrics.js'
+const { miss_count, hit_count, total_duration, total_queued, queue_size } = providerMetrics
 
 export const DummyProvider = new JsonRpcProvider('', Network.from(1))
 
-const meter = metrics.getMeter('processor_provider')
-const hit_count = meter.createCounter('provider_hit_count')
-const miss_count = meter.createCounter('provider_miss_count')
-const queue_size = meter.createGauge('provider_queue_size')
-const total_duration = meter.createCounter('provider_total_duration')
-const total_queued = meter.createCounter('provider_total_queued')
-
 const providers = new Map<string, JsonRpcProvider>()
 
 // export function getEthChainId(networkish?: EthContext | EthChainId): EthChainId {
@@ -38,7 +32,7 @@ export function getProvider(chainId?: EthChainId): Provider {
   const address = Endpoints.INSTANCE.chainServer.get(chainId)
   const key = network.chainId.toString() + '-' + address
 
-  console.debug(`init provider for ${chainId}, address: ${address}`)
+  // console.debug(`init provider for ${chainId}, address: ${address}`)
   let provider = providers.get(key)
 
   if (provider) {

package/src/service.ts

@@ -30,15 +30,13 @@ import { freezeGlobalConfig, GLOBAL_CONFIG } from './global-config.js'
 
 import { StoreContext } from './db-context.js'
 import { Subject } from 'rxjs'
-import { metrics } from '@opentelemetry/api'
 import { getProvider } from './provider.js'
 import { EthChainId } from '@sentio/chain'
 import { Provider } from 'ethers'
+import { decodeMulticallResult, encodeMulticallData, getMulticallAddress, Multicall3Call } from './multicall.js'
 
-const meter = metrics.getMeter('processor_service')
-const process_binding_count = meter.createCounter('process_binding_count')
-const process_binding_time = meter.createCounter('process_binding_time')
-const process_binding_error = meter.createCounter('process_binding_error')
+import { processMetrics, providerMetrics, dbMetrics } from './metrics.js'
+const { process_binding_count, process_binding_time, process_binding_error } = processMetrics
 
 ;(BigInt.prototype as any).toJSON = function () {
   return this.toString()
@@ -54,7 +52,7 @@ export class ProcessorServiceImpl implements ProcessorServiceImplementation {
 
   private readonly shutdownHandler?: () => void
 
-  private readonly preprocessedEthCalls: { [calldata: string]: any[] }
+  private preparedData: PreparedData | undefined
 
   constructor(loader: () => Promise<any>, shutdownHandler?: () => void) {
     this.loader = loader
@@ -128,11 +126,11 @@ export class ProcessorServiceImpl implements ProcessorServiceImplementation {
   }
 
   async processBindings(request: ProcessBindingsRequest, options?: CallContext): Promise<ProcessBindingResponse> {
-    const ethCallResults = await this.preprocessBindings(request.bindings, undefined, options)
+    const preparedData = await this.preprocessBindings(request.bindings, {}, undefined, options)
 
     const promises = []
     for (const binding of request.bindings) {
-      const promise = this.processBinding(binding, { ethCallResults })
+      const promise = this.processBinding(binding, preparedData)
       if (GLOBAL_CONFIG.execution.sequential) {
         await promise
       }
@@ -141,7 +139,9 @@ export class ProcessorServiceImpl implements ProcessorServiceImplementation {
     let promise
     try {
       promise = await Promise.all(promises)
+      processMetrics.process_binding_count.add(request.bindings.length)
     } catch (e) {
+      processMetrics.process_binding_error.add(request.bindings.length)
       throw e
     }
     const result = mergeProcessResults(promise)
@@ -159,13 +159,14 @@ export class ProcessorServiceImpl implements ProcessorServiceImplementation {
 
   async preprocessBindings(
     bindings: DataBinding[],
+    preprocessStore: { [k: string]: any },
     dbContext?: StoreContext,
     options?: CallContext
-  ): Promise<{ [ethCallKey: string]: string }> {
-    console.debug(`preprocessBindings start, bindings: ${bindings.length}`)
+  ): Promise<PreparedData> {
+    // console.debug(`preprocessBindings start, bindings: ${bindings.length}`)
     const promises = []
     for (const binding of bindings) {
-      promises.push(this.preprocessBinding(binding, dbContext, options))
+      promises.push(this.preprocessBinding(binding, preprocessStore, dbContext, options))
     }
     let preprocessResults: PreprocessResult[]
     try {
@@ -173,19 +174,15 @@ export class ProcessorServiceImpl implements ProcessorServiceImplementation {
     } catch (e) {
       throw e
     }
-    console.debug(
-      'ethCallParams: ',
-      preprocessResults.map((r) => r.ethCallParams)
-    )
     const groupedRequests = new Map<string, EthCallParam[]>()
     const providers = new Map<string, Provider>()
     for (const result of preprocessResults) {
       for (const param of result.ethCallParams) {
-        const { chainId, address, blockTag } = param.context!
+        const { chainId, blockTag } = param.context!
         if (!providers.has(chainId)) {
           providers.set(chainId, getProvider(chainId as EthChainId))
         }
-        const key = [chainId, address, blockTag].join('|')
+        const key = [chainId, blockTag].join('|')
         if (!groupedRequests.has(key)) {
           groupedRequests.set(key, [])
         }
@@ -194,36 +191,84 @@ export class ProcessorServiceImpl implements ProcessorServiceImplementation {
     }
 
     const start = Date.now()
-    const callPromises = []
+    const MULTICALL_THRESHOLD = 1
+    const callPromises: Promise<[string, string]>[] = []
+    const multicallPromises: Promise<[string, string][]>[] = []
+
     for (const params of groupedRequests.values()) {
-      const { chainId, address, blockTag } = params[0].context!
-      console.log(`chain: ${chainId}, address: ${address}, blockTag: ${blockTag}, totalCalls: ${params.length}`)
-      // TODO multicall
-      for (const param of params) {
-        callPromises.push(
+      const { chainId, blockTag } = params[0].context!
+      const multicallAddress = getMulticallAddress(chainId as EthChainId)
+      if (params.length <= MULTICALL_THRESHOLD || !multicallAddress) {
+        for (const param of params) {
+          callPromises.push(
+            providers
+              .get(chainId)!
+              .call({
+                to: param.context!.address,
+                data: param.calldata,
+                blockTag
+              })
+              .then((result) => [makeEthCallKey(param), result])
+          )
+        }
+        continue
+      }
+
+      // construct multicalls
+      const CHUNK_SIZE = 128
+      for (let i = 0; i < params.length; i += CHUNK_SIZE) {
+        const chunk = params.slice(i, i + CHUNK_SIZE)
+        const calls: Multicall3Call[] = chunk.map((param) => ({
+          target: param.context!.address,
+          callData: param.calldata
+        }))
+        const data = encodeMulticallData(calls)
+        multicallPromises.push(
           providers
             .get(chainId)!
             .call({
-              to: address,
-              data: param.calldata,
+              to: multicallAddress,
+              data: data,
               blockTag
             })
-            .then((result) => [makeEthCallKey(param), result])
+            .then((raw) => {
+              const result = decodeMulticallResult(raw).returnData
+              if (result.length != chunk.length) {
+                throw new Error(`multicall result length mismatch, params: ${chunk.length}, result: ${result.length}`)
+              }
+              const ret: [string, string][] = []
+              for (let i = 0; i < chunk.length; i++) {
+                ret.push([makeEthCallKey(chunk[i]), result[i]])
+              }
+              return ret
+            })
         )
       }
     }
-    let results = {}
+
+    let results: { [p: string]: string } = {}
     try {
       results = Object.fromEntries(await Promise.all(callPromises))
+      for (const multicallResult of await Promise.all(multicallPromises)) {
+        results = {
+          ...results,
+          ...Object.fromEntries(multicallResult)
+        }
+      }
     } catch (e) {
       console.error(`eth call error: ${e}`)
     }
-    console.log(`${callPromises.length} calls finished, elapsed: ${Date.now() - start}ms`)
-    return results
+    // console.debug(
+    //   `${Object.keys(results).length} calls finished, actual calls: ${callPromises.length + multicallPromises.length}, elapsed: ${Date.now() - start}ms`
+    // )
+    return {
+      ethCallResults: results
+    }
   }
 
   async preprocessBinding(
     request: DataBinding,
+    preprocessStore: { [k: string]: any },
     dbContext?: StoreContext,
     options?: CallContext
   ): Promise<PreprocessResult> {
@@ -242,7 +287,7 @@ export class ProcessorServiceImpl implements ProcessorServiceImplementation {
         ]
       )
     }
-    return await PluginManager.INSTANCE.preprocessBinding(request, dbContext)
+    return await PluginManager.INSTANCE.preprocessBinding(request, preprocessStore, dbContext)
   }
 
   async processBinding(
@@ -278,6 +323,7 @@ export class ProcessorServiceImpl implements ProcessorServiceImplementation {
     const subject = new Subject<DeepPartial<ProcessStreamResponse>>()
     this.handleRequests(requests, subject)
       .then(() => {
+        this.preparedData = { ethCallResults: {} }
         subject.complete()
       })
       .catch((e) => {
@@ -292,16 +338,23 @@ export class ProcessorServiceImpl implements ProcessorServiceImplementation {
     subject: Subject<DeepPartial<PreprocessStreamResponse>>
   ) {
     const contexts = new Contexts()
+    const preprocessStore: { [k: string]: any } = {}
 
     for await (const request of requests) {
       try {
-        console.debug('received request:', request)
         if (request.bindings) {
           const bindings = request.bindings.bindings
           const dbContext = contexts.new(request.processId, subject)
           const start = Date.now()
-          this.preprocessBindings(bindings, dbContext)
-            .then(() => {
+          this.preprocessBindings(bindings, preprocessStore, dbContext, undefined)
+            .then((preparedData) => {
+              // TODO maybe not proper to pass data in this way
+              this.preparedData = {
+                ethCallResults: {
+                  ...this.preparedData?.ethCallResults,
+                  ...preparedData.ethCallResults
+                }
+              }
               subject.next({
                 processId: request.processId
               })
@@ -352,14 +405,16 @@ export class ProcessorServiceImpl implements ProcessorServiceImplementation {
 
     for await (const request of requests) {
       try {
-        console.debug('received request:', request)
+        // console.debug('received request:', request)
         if (request.binding) {
           process_binding_count.add(1)
           const binding = request.binding
           const dbContext = contexts.new(request.processId, subject)
           const start = Date.now()
-          PluginManager.INSTANCE.processBinding(binding, undefined, dbContext)
-            .then((result) => {
+          PluginManager.INSTANCE.processBinding(binding, this.preparedData, dbContext)
+            .then(async (result) => {
+              // await all pending db requests
+              await dbContext.awaitPendings()
               subject.next({
                 result,
                 processId: request.processId
@@ -376,11 +431,17 @@ export class ProcessorServiceImpl implements ProcessorServiceImplementation {
               console.debug('processBinding', request.processId, ' took', cost, 'ms')
               process_binding_time.add(cost)
               contexts.delete(request.processId)
+              console.debug('db stats', JSON.stringify(dbMetrics.stats()))
+              console.debug('provider stats', JSON.stringify(providerMetrics.stats()))
             })
         }
         if (request.dbResult) {
           const dbContext = contexts.get(request.processId)
-          dbContext?.result(request.dbResult)
+          try {
+            dbContext?.result(request.dbResult)
+          } catch (e) {
+            subject.error(new Error('db result error, process should stop'))
+          }
         }
       } catch (e) {
         // should not happen

package/src/tsup.config.ts

@@ -8,6 +8,8 @@ export default defineConfig({
   },
   entry: ['src/index.ts', 'src/processor-runner.ts'],
   outDir: 'lib',
+  minify: true,
+  sourcemap: true,
   clean: true,
   dts: true,
   format: 'esm'

package/src/utils.ts

@@ -34,5 +34,5 @@ export function makeEthCallKey(param: EthCallParam) {
     throw new Error('null context for eth call')
   }
   const { chainId, address, blockTag } = param.context
-  return `${chainId}|${address}|${blockTag}|${param.calldata}`
+  return `${chainId}|${address}|${blockTag}|${param.calldata}`.toLowerCase()
 }