@sentio/runtime 1.37.0-rc.6 → 1.37.0-rc.7

package/src/full-service.ts

@@ -0,0 +1,61 @@
+import { CallContext } from 'nice-grpc'
+
+// Different than the simple one which
+import {
+  DataBinding,
+  HandlerType,
+  ProcessBindingsRequest,
+  ProcessConfigRequest,
+  ProcessorServiceImplementation,
+  StartRequest,
+} from './gen/processor/protos/processor'
+
+import { Empty } from '@sentio/protos/lib/google/protobuf/empty'
+
+export class FullProcessorServiceImpl implements ProcessorServiceImplementation {
+  constructor(instance: ProcessorServiceImplementation) {
+    this.instance = instance
+  }
+
+  instance: ProcessorServiceImplementation
+
+  async getConfig(request: ProcessConfigRequest, context: CallContext) {
+    return this.instance.getConfig(request, context)
+  }
+
+  async start(request: StartRequest, context: CallContext) {
+    return this.instance.start(request, context)
+  }
+
+  async stop(request: Empty, context: CallContext) {
+    return this.instance.stop(request, context)
+  }
+
+  async processBindings(request: ProcessBindingsRequest, options: CallContext) {
+    for (const binding of request.bindings) {
+      this.adjustDataBinding(binding)
+    }
+    return this.instance.processBindings(request, options)
+  }
+
+  async *processBindingsStream(requests: AsyncIterable<DataBinding>, context: CallContext) {
+    throw new Error('Not Implemented for streaming')
+    // y this.instance.processBindingsStream(requests, context)
+  }
+
+  protected adjustDataBinding(dataBinding: DataBinding): void {
+    switch (dataBinding.handlerType) {
+      case HandlerType.UNKNOWN:
+        if (dataBinding.data?.ethBlock) {
+          if (dataBinding.data.raw.length === 0) {
+            // This is actually not needed in current system, just as initla test propose, move to test only
+            // when this is stable
+            dataBinding.data.raw = new TextEncoder().encode(JSON.stringify(dataBinding.data.ethBlock.block))
+          }
+        }
+        break
+      default:
+        break
+    }
+  }
+}

package/src/gen/{src/google → google}/protobuf/timestamp.ts

@@ -1,86 +1,91 @@
 /* eslint-disable */
-import Long from "long";
-import _m0 from "protobufjs/minimal";
+import Long from 'long'
+import _m0 from 'protobufjs/minimal'
 
 export interface Timestamp {
-  seconds: bigint;
-  nanos: number;
+  seconds: bigint
+  nanos: number
 }
 
 function createBaseTimestamp(): Timestamp {
-  return { seconds: BigInt("0"), nanos: 0 };
+  return { seconds: BigInt('0'), nanos: 0 }
 }
 
 export const Timestamp = {
   encode(message: Timestamp, writer: _m0.Writer = _m0.Writer.create()): _m0.Writer {
-    if (message.seconds !== BigInt("0")) {
-      writer.uint32(8).int64(message.seconds.toString());
+    if (message.seconds !== BigInt('0')) {
+      writer.uint32(8).int64(message.seconds.toString())
     }
     if (message.nanos !== 0) {
-      writer.uint32(16).int32(message.nanos);
+      writer.uint32(16).int32(message.nanos)
     }
-    return writer;
+    return writer
   },
 
   decode(input: _m0.Reader | Uint8Array, length?: number): Timestamp {
-    const reader = input instanceof _m0.Reader ? input : new _m0.Reader(input);
-    let end = length === undefined ? reader.len : reader.pos + length;
-    const message = createBaseTimestamp();
+    const reader = input instanceof _m0.Reader ? input : new _m0.Reader(input)
+    let end = length === undefined ? reader.len : reader.pos + length
+    const message = createBaseTimestamp()
     while (reader.pos < end) {
-      const tag = reader.uint32();
+      const tag = reader.uint32()
       switch (tag >>> 3) {
         case 1:
-          message.seconds = longToBigint(reader.int64() as Long);
-          break;
+          message.seconds = longToBigint(reader.int64() as Long)
+          break
         case 2:
-          message.nanos = reader.int32();
-          break;
+          message.nanos = reader.int32()
+          break
         default:
-          reader.skipType(tag & 7);
-          break;
+          reader.skipType(tag & 7)
+          break
       }
     }
-    return message;
+    return message
   },
 
   fromJSON(object: any): Timestamp {
     return {
-      seconds: isSet(object.seconds) ? BigInt(object.seconds) : BigInt("0"),
+      seconds: isSet(object.seconds) ? BigInt(object.seconds) : BigInt('0'),
       nanos: isSet(object.nanos) ? Number(object.nanos) : 0,
-    };
+    }
   },
 
   toJSON(message: Timestamp): unknown {
-    const obj: any = {};
-    message.seconds !== undefined && (obj.seconds = message.seconds.toString());
-    message.nanos !== undefined && (obj.nanos = Math.round(message.nanos));
-    return obj;
+    const obj: any = {}
+    message.seconds !== undefined && (obj.seconds = message.seconds.toString())
+    message.nanos !== undefined && (obj.nanos = Math.round(message.nanos))
+    return obj
   },
 
   fromPartial(object: DeepPartial<Timestamp>): Timestamp {
-    const message = createBaseTimestamp();
-    message.seconds = object.seconds ?? BigInt("0");
-    message.nanos = object.nanos ?? 0;
-    return message;
+    const message = createBaseTimestamp()
+    message.seconds = object.seconds ?? BigInt('0')
+    message.nanos = object.nanos ?? 0
+    return message
   },
-};
+}
 
-type Builtin = Date | Function | Uint8Array | string | number | boolean | undefined;
+type Builtin = Date | Function | Uint8Array | string | number | boolean | undefined
 
-type DeepPartial<T> = T extends Builtin ? T
-  : T extends Array<infer U> ? Array<DeepPartial<U>> : T extends ReadonlyArray<infer U> ? ReadonlyArray<DeepPartial<U>>
-  : T extends {} ? { [K in keyof T]?: DeepPartial<T[K]> }
-  : Partial<T>;
+type DeepPartial<T> = T extends Builtin
+  ? T
+  : T extends Array<infer U>
+  ? Array<DeepPartial<U>>
+  : T extends ReadonlyArray<infer U>
+  ? ReadonlyArray<DeepPartial<U>>
+  : T extends {}
+  ? { [K in keyof T]?: DeepPartial<T[K]> }
+  : Partial<T>
 
 function longToBigint(long: Long) {
-  return BigInt(long.toString());
+  return BigInt(long.toString())
 }
 
 if (_m0.util.Long !== Long) {
-  _m0.util.Long = Long as any;
-  _m0.configure();
+  _m0.util.Long = Long as any
+  _m0.configure()
 }
 
 function isSet(value: any): boolean {
-  return value !== null && value !== undefined;
+  return value !== null && value !== undefined
 }

package/src/index.ts

@@ -0,0 +1,7 @@
+export * from './provider'
+export * from './plugin'
+export * from './state'
+export * from './utils'
+export * from './endpoints'
+export * from './chain-config'
+export * from './service'

package/src/loader.ts

@@ -0,0 +1,24 @@
+export function load(name: string): { module: any; name: string; path: string } | undefined {
+  const req = eval('require')
+
+  try {
+    let path: string
+    try {
+      path = req.resolve(name, { paths: [process.cwd()] })
+    } catch {
+      path = req.resolve(name)
+    }
+
+    const module = { module: req(path), name, path }
+    console.log('Processor Load successfully')
+    return module
+  } catch (err) {
+    if (err instanceof Error && err.message.startsWith(`Cannot find module '${name}'`)) {
+      // this error is expected
+      console.log("Couldn't load (expected): ", name)
+      return undefined
+    } else {
+      throw err
+    }
+  }
+}

package/src/plugin.ts

@@ -0,0 +1,53 @@
+import { DataBinding, HandlerType, ProcessConfigResponse, ProcessResult, StartRequest } from '@sentio/protos'
+
+export abstract class Plugin {
+  name: string
+  supportedHandlers: HandlerType[] = []
+
+  configure(config: ProcessConfigResponse) {}
+  start(start: StartRequest) {}
+  stateDiff(config: ProcessConfigResponse): boolean {
+    return false
+  }
+  async processBinding(request: DataBinding): Promise<ProcessResult> {
+    return ProcessResult.fromPartial({})
+  }
+}
+
+export class PluginManager {
+  static INSTANCE = new PluginManager()
+
+  plugins: Plugin[] = []
+  typesToPlugin = new Map<HandlerType, Plugin>()
+
+  register(plugin: Plugin) {
+    this.plugins.push(plugin)
+    for (const handlerType of plugin.supportedHandlers) {
+      const exsited = this.typesToPlugin.get(handlerType)
+      if (exsited) {
+        throw new Error(`Duplicate plugin for ${handlerType}: ${exsited.name} and ${plugin.name}`)
+      }
+      this.typesToPlugin.set(handlerType, plugin)
+    }
+  }
+
+  configure(config: ProcessConfigResponse) {
+    this.plugins.forEach((plugin) => plugin.configure(config))
+  }
+
+  start(start: StartRequest) {
+    this.plugins.forEach((plugin) => plugin.start(start))
+  }
+
+  stateDiff(config: ProcessConfigResponse): boolean {
+    return this.plugins.some((plugin) => plugin.stateDiff(config))
+  }
+
+  processBinding(request: DataBinding): Promise<ProcessResult> {
+    const plugin = this.typesToPlugin.get(request.handlerType)
+    if (!plugin) {
+      throw new Error(`No plugin for ${request.handlerType}`)
+    }
+    return plugin.processBinding(request)
+  }
+}

package/src/processor-runner.ts

@@ -0,0 +1,97 @@
+#!/usr/bin/env node
+
+import path from 'path'
+import fs from 'fs-extra'
+import * as util from 'util'
+
+import commandLineArgs from 'command-line-args'
+import { createServer } from 'nice-grpc'
+import { createLogger, transports, format } from 'winston'
+import { CompressionAlgorithms } from '@grpc/grpc-js/build/src/compression-algorithms'
+
+import { ProcessorDefinition } from '@sentio/protos'
+import { ProcessorServiceImpl } from './service'
+import { setProvider } from './provider'
+import { State } from './state'
+import { Endpoints } from './endpoints'
+
+import { load } from './loader'
+import { FullProcessorServiceImpl } from './full-service'
+
+State.reset()
+Endpoints.reset()
+
+const optionDefinitions = [
+  { name: 'target', type: String, defaultOption: true },
+  { name: 'port', alias: 'p', type: String, defaultValue: '4000' },
+  { name: 'concurrency', type: Number, defaultValue: 4 },
+  { name: 'use-chainserver', type: Boolean, defaultValue: false },
+  {
+    name: 'chains-config',
+    alias: 'c',
+    type: String,
+    defaultValue: 'chains-config.json',
+  },
+  { name: 'chainquery-server', type: String, defaultValue: '' },
+  { name: 'pricefeed-server', type: String, defaultValue: '' },
+  { name: 'log-format', type: String, defaultValue: 'console' },
+  { name: 'debug', type: Boolean, defaultValue: false },
+]
+
+const options = commandLineArgs(optionDefinitions, { partial: true })
+
+if (options['log-format'] === 'json') {
+  const utilFormatter = {
+    transform: (info: any) => {
+      const args = info[Symbol.for('splat')]
+      if (args) {
+        info.message = util.format(info.message, ...args)
+      }
+      return info
+    },
+  }
+  const logger = createLogger({
+    format: format.combine(
+      format.timestamp({ format: 'YYYY-MM-DDTHH:mm:ssZ' }),
+      utilFormatter,
+      format.errors({ stack: true }),
+      format.json()
+    ),
+    transports: [new transports.Console()],
+  })
+
+  console.log = (...args) => logger.info.call(logger, ...args)
+  console.info = (...args) => logger.info.call(logger, ...args)
+  console.warn = (...args) => logger.warn.call(logger, ...args)
+  console.error = (...args) => logger.error.call(logger, ...args)
+  console.debug = (...args) => logger.debug.call(logger, ...args)
+}
+if (options.debug) {
+  console.log('Starting with', options.target)
+}
+
+const fullPath = path.resolve(options['chains-config'])
+const chainsConfig = fs.readJsonSync(fullPath)
+
+setProvider(chainsConfig, options.concurrency, options['use-chainserver'])
+Endpoints.INSTANCE.chainQueryAPI = options['chainquery-server']
+Endpoints.INSTANCE.priceFeedAPI = options['pricefeed-server']
+
+if (options.debug) {
+  console.log('Starting Server', options)
+}
+
+const server = createServer({
+  'grpc.max_send_message_length': 128 * 1024 * 1024,
+  'grpc.max_receive_message_length': 128 * 1024 * 1024,
+  'grpc.default_compression_algorithm': CompressionAlgorithms.gzip,
+})
+
+const baseService = new ProcessorServiceImpl(() => load(options.target), server.shutdown)
+const service = new FullProcessorServiceImpl(baseService)
+
+server.add(ProcessorDefinition, service)
+
+server.listen('0.0.0.0:' + options.port)
+
+console.log('Processor Server Started')

package/src/provider.ts

@@ -0,0 +1,75 @@
+import { getNetwork, Provider, StaticJsonRpcProvider } from '@ethersproject/providers'
+import { Networkish } from '@ethersproject/networks'
+import PQueue from 'p-queue'
+import { ConnectionInfo } from '@ethersproject/web'
+import { ChainConfig } from './chain-config'
+import { Endpoints } from './endpoints'
+
+export const DummyProvider = new StaticJsonRpcProvider(undefined, 1)
+
+export function getProvider(networkish?: Networkish): Provider {
+  if (!networkish) {
+    networkish = 1
+  }
+  const network = getNetwork(networkish)
+
+  if (!Endpoints.INSTANCE.providers) {
+    throw Error('Provider not set')
+  }
+  const value = Endpoints.INSTANCE.providers.get(network.chainId)
+  if (value === undefined) {
+    throw Error(
+      'Provider not found for chain ' +
+        network.chainId +
+        ', configured chains: ' +
+        [...Endpoints.INSTANCE.providers.keys()].join(' ')
+    )
+  }
+  return value
+}
+
+export function setProvider(config: Record<string, ChainConfig>, concurrency = 4, useChainServer = false) {
+  Endpoints.INSTANCE.providers = new Map<number, Provider>()
+
+  for (const chainIdStr in config) {
+    if (isNaN(Number.parseInt(chainIdStr))) {
+      continue
+    }
+
+    const chainConfig = config[chainIdStr]
+    const chainId = Number(chainIdStr)
+
+    // let providers: StaticJsonRpcProvider[] = []
+    // for (const http of chainConfig.Https) {
+    //   providers.push(new StaticJsonRpcProvider(http, chainId))
+    // }
+    // random shuffle
+    // providers = providers.sort(() => Math.random() - 0.5)
+
+    // const provider = new FallbackProvider(providers)
+
+    let rpcAddress = ''
+    if (useChainServer && chainConfig.ChainServer) {
+      rpcAddress = chainConfig.ChainServer
+    } else {
+      const idx = Math.floor(Math.random() * chainConfig.Https.length)
+      rpcAddress = chainConfig.Https[idx]
+    }
+
+    const provider = new QueuedStaticJsonRpcProvider(rpcAddress, chainId, concurrency)
+    Endpoints.INSTANCE.providers.set(chainId, provider)
+  }
+}
+
+class QueuedStaticJsonRpcProvider extends StaticJsonRpcProvider {
+  executor: PQueue
+
+  constructor(url: ConnectionInfo | string, network: Networkish, concurrency: number) {
+    super(url, network)
+    this.executor = new PQueue({ concurrency: concurrency })
+  }
+
+  send(method: string, params: Array<any>): Promise<any> {
+    return this.executor.add(() => super.send(method, params))
+  }
+}

package/src/service.ts

@@ -0,0 +1,145 @@
+import { CallContext, ServerError, Status } from 'nice-grpc'
+
+import {
+  DataBinding,
+  HandlerType,
+  ProcessBindingResponse,
+  ProcessBindingsRequest,
+  ProcessConfigRequest,
+  ProcessConfigResponse,
+  ProcessorServiceImplementation,
+  ProcessResult,
+  StartRequest,
+} from '@sentio/protos'
+
+import { Empty } from '@sentio/protos/lib/google/protobuf/empty'
+
+import { PluginManager } from './plugin'
+import { errorString, mergeProcessResults } from './utils'
+;(BigInt.prototype as any).toJSON = function () {
+  return this.toString()
+}
+
+export class ProcessorServiceImpl implements ProcessorServiceImplementation {
+  private started = false
+  private processorConfig: ProcessConfigResponse
+
+  private readonly loader: () => void
+
+  private readonly shutdownHandler?: () => void
+
+  constructor(loader: () => void, shutdownHandler?: () => void) {
+    this.loader = loader
+    this.shutdownHandler = shutdownHandler
+  }
+
+  async getConfig(request: ProcessConfigRequest, context: CallContext): Promise<ProcessConfigResponse> {
+    if (!this.started) {
+      throw new ServerError(Status.UNAVAILABLE, 'Service Not started.')
+    }
+    if (!this.processorConfig) {
+      throw new ServerError(Status.INTERNAL, 'Process config empty.')
+    }
+    return this.processorConfig
+  }
+
+  async configure() {
+    this.processorConfig = ProcessConfigResponse.fromPartial({})
+    PluginManager.INSTANCE.configure(this.processorConfig)
+  }
+
+  async start(request: StartRequest, context: CallContext): Promise<Empty> {
+    if (this.started) {
+      return {}
+    }
+
+    try {
+      for (const plugin of [
+        '@sentio/sdk/lib/core/core-plugin',
+        '@sentio/sdk/lib/core/eth-plugin',
+        '@sentio/sdk/lib/core/sui-plugin',
+        '@sentio/sdk-aptos/lib/aptos-plugin',
+        '@sentio/sdk-solana/lib/solana-plugin',
+      ]) {
+        try {
+          require(plugin)
+        } catch (e) {
+          console.error('Failed to load plugin: ', plugin)
+        }
+      }
+
+      this.loader()
+    } catch (e) {
+      throw new ServerError(Status.INVALID_ARGUMENT, 'Failed to load processor: ' + errorString(e))
+    }
+
+    PluginManager.INSTANCE.start(request)
+
+    try {
+      await this.configure()
+    } catch (e) {
+      throw new ServerError(Status.INTERNAL, 'Failed to start processor : ' + errorString(e))
+    }
+    this.started = true
+    return {}
+  }
+
+  async stop(request: Empty, context: CallContext): Promise<Empty> {
+    console.log('Server Shutting down in 5 seconds')
+    if (this.shutdownHandler) {
+      setTimeout(this.shutdownHandler, 5000)
+    }
+    return {}
+  }
+
+  async processBindings(request: ProcessBindingsRequest, options?: CallContext): Promise<ProcessBindingResponse> {
+    if (!this.started) {
+      throw new ServerError(Status.UNAVAILABLE, 'Service Not started.')
+    }
+
+    const promises = request.bindings.map((binding) => this.processBinding(binding))
+    const result = mergeProcessResults(await Promise.all(promises))
+
+    let updated = false
+    if (PluginManager.INSTANCE.stateDiff(this.processorConfig)) {
+      await this.configure()
+      updated = true
+    }
+
+    return {
+      result,
+      configUpdated: updated,
+    }
+  }
+
+  async processBinding(request: DataBinding, options?: CallContext): Promise<ProcessResult> {
+    const result = await PluginManager.INSTANCE.processBinding(request)
+    recordRuntimeInfo(result, request.handlerType)
+    return result
+  }
+
+  async *processBindingsStream(requests: AsyncIterable<DataBinding>, context: CallContext) {
+    for await (const request of requests) {
+      const result = await this.processBinding(request)
+      let updated = false
+      if (PluginManager.INSTANCE.stateDiff(this.processorConfig)) {
+        await this.configure()
+        updated = true
+      }
+      yield {
+        result,
+        configUpdated: updated,
+      }
+    }
+  }
+}
+
+function recordRuntimeInfo(results: ProcessResult, handlerType: HandlerType) {
+  for (const list of [results.gauges, results.counters, results.logs, results.events, results.exports]) {
+    list.forEach((e) => {
+      e.runtimeInfo = {
+        from: handlerType,
+      }
+    })
+  }
+}

package/src/state.ts

@@ -0,0 +1,75 @@
+export class State {
+  stateMap = new Map<string, any>()
+
+  static INSTANCE = new State()
+
+  static reset() {
+    State.INSTANCE = new State()
+  }
+}
+
+export abstract class StateStorage<T> {
+  // TODO learn how to define single instance for all subclasses
+
+  protected constructor() {
+    //
+  }
+
+  abstract initValue(): T
+
+  key(): string {
+    return this.constructor.name
+  }
+
+  getOrRegister(): T {
+    let metricState: T = State.INSTANCE.stateMap.get(this.key())
+    if (!metricState) {
+      metricState = this.initValue()
+      State.INSTANCE.stateMap.set(this.key(), metricState)
+    }
+    return metricState
+  }
+}
+
+export abstract class MapStateStorage<T> extends StateStorage<Map<string, T>> {
+  initValue() {
+    return new Map<string, T>()
+  }
+
+  getValue(key: string): T | undefined {
+    const m = this.getOrRegister()
+    return m.get(key)
+  }
+
+  getValues(): T[] {
+    const m = this.getOrRegister()
+    return Array.from(m.values())
+  }
+
+  getOrSetValue(key: string, value: T): T {
+    const m = this.getOrRegister()
+    const oldValue = m.get(key)
+    if (oldValue) {
+      console.warn(key, 'has been registered twice, use the previous one')
+      return oldValue
+    }
+    m.set(key, value)
+    return value
+  }
+}
+
+export abstract class ListStateStorage<T> extends StateStorage<T[]> {
+  initValue() {
+    return []
+  }
+
+  getValues(): T[] {
+    return this.getOrRegister()
+  }
+
+  addValue(value: T): T {
+    const m = this.getOrRegister()
+    m.push(value)
+    return value
+  }
+}

package/src/utils.ts

@@ -0,0 +1,20 @@
+import { ProcessResult } from '@sentio/protos'
+
+export function mergeProcessResults(results: ProcessResult[]): ProcessResult {
+  const res = ProcessResult.fromPartial({})
+
+  for (const r of results) {
+    res.counters = res.counters.concat(r.counters)
+    res.gauges = res.gauges.concat(r.gauges)
+    res.logs = res.logs.concat(r.logs)
+    res.events = res.events.concat(r.events)
+    res.exports = res.exports.concat(r.exports)
+  }
+  return res
+}
+
+export function errorString(e: Error): string {
+  return e.stack || e.message
+}
+
+export const USER_PROCESSOR = 'user_processor'