@sentio/runtime 0.0.0-rc.a

package/src/otlp.ts

@@ -0,0 +1,59 @@
+import { envDetector } from '@opentelemetry/resources'
+import { MeterProvider, PeriodicExportingMetricReader } from '@opentelemetry/sdk-metrics'
+import { OTLPMetricExporter } from '@opentelemetry/exporter-metrics-otlp-grpc'
+import { PrometheusExporter } from '@opentelemetry/exporter-prometheus'
+import { NodeTracerProvider } from '@opentelemetry/sdk-trace-node'
+import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-grpc'
+import { BatchSpanProcessor } from '@opentelemetry/sdk-trace-base'
+import { diag, DiagConsoleLogger, DiagLogLevel, metrics, trace, ProxyTracerProvider } from '@opentelemetry/api'
+
+export async function setupOTLP(debug?: boolean) {
+  if (debug) {
+    diag.setLogger(new DiagConsoleLogger(), DiagLogLevel.DEBUG)
+  }
+
+  const resource = await envDetector.detect()
+
+  const meterProvider = new MeterProvider({
+    resource,
+    readers: [
+      new PeriodicExportingMetricReader({
+        exporter: new OTLPMetricExporter()
+      }),
+      new PrometheusExporter({
+        host: '0.0.0.0',
+        port: 4041
+      })
+    ]
+  })
+
+  const traceProvider = new NodeTracerProvider({
+    resource: resource
+  })
+  const exporter = new OTLPTraceExporter() // new ConsoleSpanExporter();
+  const processor = new BatchSpanProcessor(exporter)
+  traceProvider.addSpanProcessor(processor)
+
+  metrics.setGlobalMeterProvider(meterProvider)
+  trace.setGlobalTracerProvider(traceProvider)
+  ;['SIGINT', 'SIGTERM'].forEach((signal) => {
+    process.on(signal as any, () => shutdownProvider())
+  })
+
+  metrics.getMeter('processor').createGauge('up').record(1)
+}
+
+export async function shutdownProvider() {
+  const traceProvider = trace.getTracerProvider()
+
+  if (traceProvider instanceof ProxyTracerProvider) {
+    const delegate = traceProvider.getDelegate()
+    if (delegate instanceof NodeTracerProvider) {
+      delegate.shutdown().catch(console.error)
+    }
+  }
+  const meterProvider = metrics.getMeterProvider()
+  if (meterProvider instanceof MeterProvider) {
+    meterProvider.shutdown().catch(console.error)
+  }
+}

package/src/plugin.ts

@@ -0,0 +1,120 @@
+import {
+  DataBinding,
+  HandlerType,
+  PreparedData,
+  PreprocessResult,
+  ProcessConfigResponse,
+  ProcessResult,
+  StartRequest
+} from '@sentio/protos'
+import { StoreContext } from './db-context.js'
+import { AsyncLocalStorage } from 'node:async_hooks'
+
+export abstract class Plugin {
+  name: string
+  supportedHandlers: HandlerType[] = []
+
+  async configure(config: ProcessConfigResponse): Promise<void> {}
+
+  async start(start: StartRequest): Promise<void> {}
+
+  /**
+   * @deprecated The method should not be used, use ctx.states instead
+   */
+  stateDiff(config: ProcessConfigResponse): boolean {
+    return false
+  }
+
+  async processBinding(request: DataBinding, preparedData: PreparedData | undefined): Promise<ProcessResult> {
+    return ProcessResult.create()
+  }
+
+  async preprocessBinding(request: DataBinding, preprocessStore: { [k: string]: any }): Promise<PreprocessResult> {
+    return PreprocessResult.create()
+  }
+
+  /**
+   * method used by action server only
+   * @param port
+   */
+  async startServer(port?: number): Promise<void> {}
+
+  /**
+   * method used by action server only
+   */
+  shutdownServer() {}
+}
+
+export class PluginManager {
+  static INSTANCE = new PluginManager()
+
+  dbContextLocalStorage = new AsyncLocalStorage<StoreContext | undefined>()
+  plugins: Plugin[] = []
+  typesToPlugin = new Map<HandlerType, Plugin>()
+
+  register(plugin: Plugin) {
+    if (this.plugins.find((p) => p.name === plugin.name)) {
+      return
+    }
+    this.plugins.push(plugin)
+
+    for (const handlerType of plugin.supportedHandlers) {
+      const exsited = this.typesToPlugin.get(handlerType)
+      if (exsited) {
+        throw new Error(`Duplicate plugin for ${handlerType}: ${exsited.name} and ${plugin.name}`)
+      }
+      this.typesToPlugin.set(handlerType, plugin)
+    }
+  }
+
+  configure(config: ProcessConfigResponse) {
+    return Promise.all(this.plugins.map((plugin) => plugin.configure(config)))
+  }
+
+  start(start: StartRequest, actionServerPort?: number) {
+    return Promise.all(this.plugins.map((plugin) => plugin.start(start)))
+  }
+
+  startServer(port?: number) {
+    return Promise.all(this.plugins.map((plugin) => plugin.startServer(port)))
+  }
+
+  shutdown() {
+    this.plugins.forEach((plugin) => plugin.shutdownServer())
+  }
+
+  /**
+   * @deprecated The method should not be used, use ctx.states instead
+   */
+  stateDiff(config: ProcessConfigResponse): boolean {
+    return this.plugins.some((plugin) => plugin.stateDiff(config))
+  }
+
+  processBinding(
+    request: DataBinding,
+    preparedData: PreparedData | undefined,
+    dbContext?: StoreContext
+  ): Promise<ProcessResult> {
+    const plugin = this.typesToPlugin.get(request.handlerType)
+    if (!plugin) {
+      throw new Error(`No plugin for ${request.handlerType}`)
+    }
+    return this.dbContextLocalStorage.run(dbContext, () => {
+      return plugin.processBinding(request, preparedData)
+    })
+  }
+
+  preprocessBinding(
+    request: DataBinding,
+    preprocessStore: { [k: string]: any },
+    dbContext?: StoreContext
+  ): Promise<PreprocessResult> {
+    const plugin = this.typesToPlugin.get(request.handlerType)
+    if (!plugin) {
+      throw new Error(`No plugin for ${request.handlerType}`)
+    }
+    return this.dbContextLocalStorage.run(dbContext, () => {
+      return plugin.preprocessBinding(request, preprocessStore)
+    })
+  }
+}

package/src/processor-runner.ts

@@ -0,0 +1,226 @@
+#!/usr/bin/env node
+
+import path from 'path'
+import fs from 'fs-extra'
+
+import { compressionAlgorithms } from '@grpc/grpc-js'
+import commandLineArgs from 'command-line-args'
+import { createServer } from 'nice-grpc'
+import { errorDetailsServerMiddleware } from 'nice-grpc-error-details'
+// import { registry as niceGrpcRegistry } from 'nice-grpc-prometheus'
+import { openTelemetryServerMiddleware } from 'nice-grpc-opentelemetry'
+import http from 'http'
+// @ts-ignore inspector promises is not included in @type/node
+import { Session } from 'node:inspector/promises'
+
+import { ProcessorDefinition } from './gen/processor/protos/processor.js'
+import { ProcessorServiceImpl } from './service.js'
+import { Endpoints } from './endpoints.js'
+import { FullProcessorServiceImpl } from './full-service.js'
+import { ChainConfig } from './chain-config.js'
+import { setupLogger } from './logger.js'
+
+import { setupOTLP } from './otlp.js'
+import { ActionServer } from './action-server.js'
+import { ServiceManager } from './service-manager.js'
+
+// const mergedRegistry = Registry.merge([globalRegistry, niceGrpcRegistry])
+
+export const optionDefinitions = [
+  { name: 'target', type: String, defaultOption: true },
+  { name: 'port', alias: 'p', type: String, defaultValue: '4000' },
+  { name: 'concurrency', type: Number, defaultValue: 4 },
+  { name: 'batch-count', type: Number, defaultValue: 1 },
+  // { name: 'use-chainserver', type: Boolean, defaultValue: false },
+  {
+    name: 'chains-config',
+    alias: 'c',
+    type: String,
+    defaultValue: 'chains-config.json'
+  },
+  { name: 'chainquery-server', type: String, defaultValue: '' },
+  { name: 'pricefeed-server', type: String, defaultValue: '' },
+  { name: 'log-format', type: String, defaultValue: 'console' },
+  { name: 'debug', type: Boolean, defaultValue: false },
+  { name: 'otlp-debug', type: Boolean, defaultValue: false },
+  { name: 'start-action-server', type: Boolean, defaultValue: false },
+  { name: 'worker', type: Number, defaultValue: 8 }
+]
+
+const options = commandLineArgs(optionDefinitions, { partial: true })
+
+const logLevel = process.env['LOG_LEVEL']?.toUpperCase()
+
+setupLogger(options['log-format'] === 'json', logLevel === 'debug' ? true : options.debug)
+console.debug('Starting with', options.target)
+
+await setupOTLP(options['otlp-debug'])
+
+Error.stackTraceLimit = 20
+
+const fullPath = path.resolve(options['chains-config'])
+const chainsConfig = fs.readJsonSync(fullPath)
+
+const concurrencyOverride = process.env['OVERRIDE_CONCURRENCY']
+  ? parseInt(process.env['OVERRIDE_CONCURRENCY'])
+  : undefined
+const batchCountOverride = process.env['OVERRIDE_BATCH_COUNT']
+  ? parseInt(process.env['OVERRIDE_BATCH_COUNT'])
+  : undefined
+
+Endpoints.INSTANCE.concurrency = concurrencyOverride ?? options.concurrency
+Endpoints.INSTANCE.batchCount = batchCountOverride ?? options['batch-count']
+Endpoints.INSTANCE.chainQueryAPI = options['chainquery-server']
+Endpoints.INSTANCE.priceFeedAPI = options['pricefeed-server']
+
+for (const [id, config] of Object.entries(chainsConfig)) {
+  const chainConfig = config as ChainConfig
+  if (chainConfig.ChainServer) {
+    Endpoints.INSTANCE.chainServer.set(id, chainConfig.ChainServer)
+  } else {
+    const http = chainConfig.Https?.[0]
+    if (http) {
+      Endpoints.INSTANCE.chainServer.set(id, http)
+    } else {
+      console.error('not valid config for chain', id)
+    }
+  }
+}
+
+console.debug('Starting Server', options)
+
+let server: any
+let baseService: ProcessorServiceImpl | ServiceManager
+const loader = async () => {
+  const m = await import(options.target)
+  console.debug('Module loaded', m)
+  return m
+}
+if (options['start-action-server']) {
+  server = new ActionServer(loader)
+  server.listen(options.port)
+} else {
+  server = createServer({
+    'grpc.max_send_message_length': 768 * 1024 * 1024,
+    'grpc.max_receive_message_length': 768 * 1024 * 1024,
+    'grpc.default_compression_algorithm': compressionAlgorithms.gzip
+  })
+    // .use(prometheusServerMiddleware())
+    .use(openTelemetryServerMiddleware())
+    .use(errorDetailsServerMiddleware)
+
+  if (options.worker > 1) {
+    baseService = new ServiceManager(options, loader, server.shutdown)
+  } else {
+    baseService = new ProcessorServiceImpl(loader, server.shutdown)
+  }
+
+  const service = new FullProcessorServiceImpl(baseService)
+
+  server.add(ProcessorDefinition, service)
+
+  server.listen('0.0.0.0:' + options.port)
+
+  console.log('Processor Server Started at:', options.port)
+}
+
+const metricsPort = 4040
+const httpServer = http
+  .createServer(async function (req, res) {
+    if (req.url) {
+      const reqUrl = new URL(req.url, `http://${req.headers.host}`)
+      const queries = reqUrl.searchParams
+      switch (reqUrl.pathname) {
+        // case '/metrics':
+        //   const metrics = await mergedRegistry.metrics()
+        //   res.write(metrics)
+        //   break
+        case '/heap': {
+          try {
+            const file = '/tmp/' + Date.now() + '.heapsnapshot'
+            const session = new Session()
+
+            const fd = fs.openSync(file, 'w')
+            session.connect()
+            session.on('HeapProfiler.addHeapSnapshotChunk', (m) => {
+              fs.writeSync(fd, m.params.chunk)
+            })
+
+            await session.post('HeapProfiler.takeHeapSnapshot')
+            session.disconnect()
+            fs.closeSync(fd)
+            // send the file
+            const readStream = fs.createReadStream(file)
+            res.writeHead(200, { 'Content-Type': 'application/json' })
+            readStream.pipe(res)
+            res.end()
+          } catch {
+            res.writeHead(500)
+            res.end()
+          }
+          break
+        }
+        case '/profile': {
+          try {
+            const profileTime = parseInt(queries.get('t') || '1000', 10) || 1000
+            const session = new Session()
+            session.connect()
+
+            await session.post('Profiler.enable')
+            await session.post('Profiler.start')
+
+            await new Promise((resolve) => setTimeout(resolve, profileTime))
+            const { profile } = await session.post('Profiler.stop')
+
+            res.writeHead(200, { 'Content-Type': 'application/json' })
+            res.write(JSON.stringify(profile))
+            session.disconnect()
+          } catch {
+            res.writeHead(500)
+          }
+          break
+        }
+        default:
+          res.writeHead(404)
+      }
+    } else {
+      res.writeHead(404)
+    }
+    res.end()
+  })
+  .listen(metricsPort)
+
+console.log('Metric Server Started at:', metricsPort)
+
+process
+  .on('SIGINT', function () {
+    shutdownServers(0)
+  })
+  .on('uncaughtException', (err) => {
+    console.error('Uncaught Exception, please checking if await is properly used', err)
+    if (baseService) {
+      baseService.unhandled = err
+    }
+    // shutdownServers(1)
+  })
+  .on('unhandledRejection', (reason, p) => {
+    // @ts-ignore ignore invalid ens error
+    if (reason?.message.startsWith('invalid ENS name (disallowed character: "*"')) {
+      return
+    }
+    console.error('Unhandled Rejection, please checking if await is properly', reason)
+    if (baseService) {
+      baseService.unhandled = reason as Error
+    }
+    // shutdownServers(1)
+  })
+
+function shutdownServers(exitCode: number) {
+  server?.forceShutdown()
+  console.log('RPC server shut down')
+
+  httpServer.close(function () {
+    console.log('Http server shut down')
+    process.exit(exitCode)
+  })
+}

package/src/provider.ts

@@ -0,0 +1,195 @@
+import { JsonRpcProvider, Network, Provider } from 'ethers'
+
+import PQueue from 'p-queue'
+import { Endpoints } from './endpoints.js'
+import { EthChainId } from '@sentio/chain'
+import { LRUCache } from 'lru-cache'
+import { providerMetrics, processMetrics, metricsStorage } from './metrics.js'
+import { GLOBAL_CONFIG } from './global-config.js'
+const { miss_count, hit_count, queue_size } = providerMetrics
+
+export const DummyProvider = new JsonRpcProvider('', Network.from(1))
+
+const providers = new Map<string, JsonRpcProvider>()
+
+// export function getEthChainId(networkish?: EthContext | EthChainId): EthChainId {
+//   if (!networkish) {
+//     networkish = EthChainId.ETHEREUM
+//   }
+//   if (networkish instanceof BaseContext) {
+//     networkish = networkish.getChainId()
+//   }
+//   return networkish
+// }
+
+export function getProvider(chainId?: EthChainId): Provider {
+  // const network = getNetworkFromCtxOrNetworkish(networkish)
+  if (!chainId) {
+    chainId = EthChainId.ETHEREUM
+  }
+  const network = Network.from(parseInt(chainId))
+  // TODO check if other key needed
+
+  const address = Endpoints.INSTANCE.chainServer.get(chainId)
+  const key = network.chainId.toString() + '-' + address
+
+  // console.debug(`init provider for ${chainId}, address: ${address}`)
+  let provider = providers.get(key)
+
+  if (provider) {
+    return provider
+  }
+
+  if (address === undefined) {
+    throw Error(
+      'Provider not found for chain ' +
+        network.chainId +
+        ', configured chains: ' +
+        [...Endpoints.INSTANCE.chainServer.keys()].join(' ')
+    )
+  }
+  // console.log(
+  //   `init provider for chain ${network.chainId}, concurrency: ${Endpoints.INSTANCE.concurrency}, batchCount: ${Endpoints.INSTANCE.batchCount}`
+  // )
+  provider = new QueuedStaticJsonRpcProvider(
+    address,
+    network,
+    Endpoints.INSTANCE.concurrency,
+    Endpoints.INSTANCE.batchCount
+  )
+  providers.set(key, provider)
+  return provider
+}
+
+function getTag(prefix: string, value: any): string {
+  return (
+    prefix +
+    ':' +
+    JSON.stringify(value, (k, v) => {
+      if (v == null) {
+        return 'null'
+      }
+      if (typeof v === 'bigint') {
+        return `bigint:${v.toString()}`
+      }
+      if (typeof v === 'string') {
+        return v.toLowerCase()
+      }
+
+      // Sort object keys
+      if (typeof v === 'object' && !Array.isArray(v)) {
+        const keys = Object.keys(v)
+        keys.sort()
+        return keys.reduce(
+          (accum, key) => {
+            accum[key] = v[key]
+            return accum
+          },
+          <any>{}
+        )
+      }
+
+      return v
+    })
+  )
+}
+
+export class QueuedStaticJsonRpcProvider extends JsonRpcProvider {
+  executor: PQueue
+  #performCache = new LRUCache<string, Promise<any>>({
+    max: 300000, // 300k items
+    maxSize: 500 * 1024 * 1024 // 500mb key size for cache
+    // ttl: 1000 * 60 * 60, // 1 hour   no ttl for better performance
+    // sizeCalculation: (value: any) => {
+    // assume each item is 1kb for simplicity
+    // return 1024
+    // }
+  })
+  #retryCache = new LRUCache<string, number>({
+    max: 300000 // 300k items
+  })
+
+  constructor(url: string, network: Network, concurrency: number, batchCount = 1) {
+    // TODO re-enable match when possible
+    super(url, network, { staticNetwork: network, batchMaxCount: batchCount })
+    this.executor = new PQueue({ concurrency: concurrency })
+  }
+
+  async send(method: string, params: Array<any>): Promise<any> {
+    if (method !== 'eth_call') {
+      return await this.executor.add(() => super.send(method, params))
+    }
+    const tag = getTag(method, params)
+    const block = params[params.length - 1]
+    let perform = this.#performCache.get(tag)
+    if (!perform) {
+      miss_count.add(1)
+      const handler = metricsStorage.getStore()
+      const queued: number = Date.now()
+      perform = this.executor.add(() => {
+        const started = Date.now()
+        processMetrics.processor_rpc_queue_duration.record(started - queued, {
+          chain_id: this._network.chainId.toString(),
+          handler
+        })
+
+        let success = true
+        return super
+          .send(method, params)
+          .catch((e) => {
+            success = false
+            throw e
+          })
+          .finally(() => {
+            processMetrics.processor_rpc_duration.record(Date.now() - started, {
+              chain_id: this._network.chainId.toString(),
+              handler,
+              success
+            })
+          })
+      })
+
+      queue_size.record(this.executor.size)
+
+      this.#performCache.set(tag, perform, {
+        size: tag.length
+      })
+      // For non latest block call, we cache permanently, otherwise we cache for one minute
+      if (block === 'latest') {
+        setTimeout(() => {
+          if (this.#performCache.get(tag) === perform) {
+            this.#performCache.delete(tag)
+          }
+        }, 60 * 1000)
+      }
+    } else {
+      hit_count.add(1)
+    }
+
+    let result
+    try {
+      result = await perform
+    } catch (e) {
+      this.#performCache.delete(tag)
+      if (e.code === 'TIMEOUT') {
+        let retryCount = this.#retryCache.get(tag)
+        if (GLOBAL_CONFIG.execution.rpcRetryTimes && retryCount === undefined) {
+          retryCount = GLOBAL_CONFIG.execution.rpcRetryTimes
+        }
+        if (retryCount) {
+          this.#retryCache.set(tag, retryCount - 1)
+          return this.send(method, params)
+        }
+      }
+      throw e
+    }
+    if (!result) {
+      throw Error('Unexpected null response')
+    }
+    return result
+  }
+
+  toString() {
+    return 'QueuedStaticJsonRpcProvider'
+  }
+}