@sentio/runtime 2.57.11 → 2.57.12-rc.a

package/package.json

@@ -1,10 +1,11 @@
 {
   "name": "@sentio/runtime",
-  "version": "2.57.11",
+  "version": "2.57.12-rc.a",
   "license": "Apache-2.0",
   "type": "module",
   "exports": {
-    ".": "./lib/index.js"
+    ".": "./lib/index.js",
+    "./service-worker": "./lib/service-worker.js"
   },
   "bin": {
     "processor-runner": "./lib/processor-runner.js"
@@ -14,12 +15,36 @@
     "!**/*.test.{js,ts}",
     "!{lib,src}/tests"
   ],
-  "dependencies": {},
+  "dependencies": {
+    "@grpc/grpc-js": "^1.9.14",
+    "@opentelemetry/exporter-metrics-otlp-grpc": "^0.57.0",
+    "@opentelemetry/exporter-prometheus": "^0.57.0",
+    "@opentelemetry/sdk-node": "^0.57.0",
+    "command-line-args": "^6.0.0",
+    "command-line-usage": "^7.0.1",
+    "ethers": "npm:@sentio/ethers@6.13.1-patch.5",
+    "fs-extra": "^11.2.0",
+    "google-protobuf": "^3.21.2",
+    "ix": "^7.0.0",
+    "long": "^5.2.3",
+    "nice-grpc": "^2.1.10",
+    "nice-grpc-client-middleware-retry": "^3.1.6",
+    "nice-grpc-common": "^2.0.2",
+    "nice-grpc-error-details": "^0.2.4",
+    "nice-grpc-opentelemetry": "^0.1.15",
+    "nice-grpc-prometheus": "^0.2.2",
+    "protobufjs": "^7.2.6",
+    "rxjs": "^7.8.1",
+    "utility-types": "^3.11.0",
+    "winston": "^3.11.0",
+    "@sentio/protos": "2.57.12-rc.a"
+  },
   "devDependencies": {
     "@types/command-line-args": "^5.2.3",
     "@types/command-line-usage": "^5.0.4",
     "@types/fs-extra": "^11.0.4",
-    "@types/google-protobuf": "^3.15.12"
+    "@types/google-protobuf": "^3.15.12",
+    "piscina": "5.0.0-alpha.0"
   },
   "engines": {
     "node": ">=20"

package/src/processor-runner.ts

@@ -22,10 +22,11 @@ import { setupLogger } from './logger.js'
 
 import { setupOTLP } from './otlp.js'
 import { ActionServer } from './action-server.js'
+import { ServiceManager } from './service-manager.js'
 
 // const mergedRegistry = Registry.merge([globalRegistry, niceGrpcRegistry])
 
-const optionDefinitions = [
+export const optionDefinitions = [
   { name: 'target', type: String, defaultOption: true },
   { name: 'port', alias: 'p', type: String, defaultValue: '4000' },
   { name: 'concurrency', type: Number, defaultValue: 4 },
@@ -42,7 +43,8 @@ const optionDefinitions = [
   { name: 'log-format', type: String, defaultValue: 'console' },
   { name: 'debug', type: Boolean, defaultValue: false },
   { name: 'otlp-debug', type: Boolean, defaultValue: false },
-  { name: 'start-action-server', type: Boolean, defaultValue: false }
+  { name: 'start-action-server', type: Boolean, defaultValue: false },
+  { name: 'worker', type: Number, defaultValue: 8 }
 ]
 
 const options = commandLineArgs(optionDefinitions, { partial: true })
@@ -88,7 +90,7 @@ for (const [id, config] of Object.entries(chainsConfig)) {
 console.debug('Starting Server', options)
 
 let server: any
-let baseService: ProcessorServiceImpl
+let baseService: ProcessorServiceImpl | ServiceManager
 const loader = async () => {
   const m = await import(options.target)
   console.debug('Module loaded', m)
@@ -106,7 +108,13 @@ if (options['start-action-server']) {
     // .use(prometheusServerMiddleware())
     .use(openTelemetryServerMiddleware())
     .use(errorDetailsServerMiddleware)
-  baseService = new ProcessorServiceImpl(loader, server.shutdown)
+
+  if (options.worker > 1) {
+    baseService = new ServiceManager(options, loader, server.shutdown)
+  } else {
+    baseService = new ProcessorServiceImpl(loader, server.shutdown)
+  }
+
   const service = new FullProcessorServiceImpl(baseService)
 
   server.add(ProcessorDefinition, service)

package/src/service-manager.ts

@@ -0,0 +1,263 @@
+import { CallContext, ServerError, Status } from 'nice-grpc'
+import { DebugInfo, RichServerError } from 'nice-grpc-error-details'
+import { from } from 'ix/Ix.asynciterable'
+import { withAbort } from 'ix/Ix.asynciterable.operators'
+import { Piscina } from 'piscina'
+
+import {
+  DataBinding,
+  DeepPartial,
+  Empty,
+  HandlerType,
+  PreparedData,
+  PreprocessStreamRequest,
+  ProcessBindingResponse,
+  ProcessBindingsRequest,
+  ProcessConfigRequest,
+  ProcessConfigResponse,
+  ProcessorServiceImplementation,
+  ProcessResult,
+  ProcessStreamRequest,
+  ProcessStreamResponse,
+  StartRequest
+} from '@sentio/protos'
+
+import { PluginManager } from './plugin.js'
+import { errorString, mergeProcessResults } from './utils.js'
+import { GLOBAL_CONFIG } from './global-config.js'
+
+import { StoreContext } from './db-context.js'
+import { Subject } from 'rxjs'
+
+import { processMetrics } from './metrics.js'
+
+const { process_binding_count, process_binding_time, process_binding_error } = processMetrics
+
+;(BigInt.prototype as any).toJSON = function () {
+  return this.toString()
+}
+
+export class ServiceManager implements ProcessorServiceImplementation {
+  private started = false
+  // When there is unhandled error, stop process and return unavailable error
+  unhandled: Error
+  // private processorConfig: ProcessConfigResponse
+  private readonly pool: Piscina<any, any>
+
+  constructor(
+    readonly options: any,
+    readonly loader: () => Promise<any>,
+    readonly shutdownHandler?: () => void
+  ) {
+    this.pool = new Piscina({
+      maxThreads: options.worker,
+      minThreads: options.worker,
+      filename: new URL('./service-worker.js', import.meta.url).href
+    })
+  }
+
+  async *preprocessBindingsStream(requests: AsyncIterable<PreprocessStreamRequest>, context: CallContext) {
+    throw new Error('not supported')
+  }
+
+  async getConfig(request: ProcessConfigRequest, context: CallContext): Promise<ProcessConfigResponse> {
+    if (!this.started) {
+      throw new ServerError(Status.UNAVAILABLE, 'Service Not started.')
+    }
+
+    // broadcast to all worker
+    const bc = new BroadcastChannel('configure_channel')
+    bc.postMessage(request)
+
+    // also configure the main thread
+    const newConfig = ProcessConfigResponse.fromPartial({})
+    await PluginManager.INSTANCE.configure(newConfig)
+    return newConfig
+  }
+
+  //
+  // async configure() {
+  //   this.processorConfig = ProcessConfigResponse.fromPartial({})
+  //   await PluginManager.INSTANCE.configure(this.processorConfig)
+  // }
+
+  async start(request: StartRequest, context: CallContext): Promise<Empty> {
+    if (this.started) {
+      return {}
+    }
+
+    try {
+      await this.loader()
+    } catch (e) {
+      throw new ServerError(Status.INVALID_ARGUMENT, 'Failed to load processor: ' + errorString(e))
+    }
+
+    // broadcast to all worker
+    const bc = new BroadcastChannel('start_channel')
+    bc.postMessage(request)
+
+    // also start the processor in main thread
+    await PluginManager.INSTANCE.start(request)
+    this.started = true
+    return {}
+  }
+
+  async stop(request: Empty, context: CallContext): Promise<Empty> {
+    console.log('Server Shutting down in 5 seconds')
+    if (this.shutdownHandler) {
+      setTimeout(this.shutdownHandler, 5000)
+    }
+    return {}
+  }
+
+  async processBindings(request: ProcessBindingsRequest, options?: CallContext): Promise<ProcessBindingResponse> {
+    const promises = []
+    for (const binding of request.bindings) {
+      const promise = this.processBinding(binding, undefined)
+      if (GLOBAL_CONFIG.execution.sequential) {
+        await promise
+      }
+      promises.push(promise)
+    }
+    let promise
+    try {
+      promise = await Promise.all(promises)
+      processMetrics.process_binding_count.add(request.bindings.length)
+    } catch (e) {
+      processMetrics.process_binding_error.add(request.bindings.length)
+      throw e
+    }
+    const result = mergeProcessResults(promise)
+
+    // let updated = false
+    // if (PluginManager.INSTANCE.stateDiff(this.processorConfig)) {
+    //   await this.configure()
+    //   updated = true
+    // }
+
+    return {
+      result
+    }
+  }
+
+  async processBinding(
+    request: DataBinding,
+    preparedData: PreparedData | undefined,
+    options?: CallContext
+  ): Promise<ProcessResult> {
+    if (!this.started) {
+      throw new ServerError(Status.UNAVAILABLE, 'Service Not started.')
+    }
+    if (this.unhandled) {
+      throw new RichServerError(
+        Status.UNAVAILABLE,
+        'Unhandled exception/rejection in previous request: ' + errorString(this.unhandled),
+        [
+          DebugInfo.fromPartial({
+            detail: this.unhandled.message,
+            stackEntries: this.unhandled.stack?.split('\n')
+          })
+        ]
+      )
+    }
+
+    return await this.pool.run({ request, preparedData }, { name: 'processBinding' })
+  }
+
+  async *processBindingsStream(requests: AsyncIterable<ProcessStreamRequest>, context: CallContext) {
+    if (!this.started) {
+      throw new ServerError(Status.UNAVAILABLE, 'Service Not started.')
+    }
+
+    const subject = new Subject<DeepPartial<ProcessStreamResponse>>()
+    this.handleRequests(requests, subject)
+      .then(() => {
+        subject.complete()
+      })
+      .catch((e) => {
+        console.error(e)
+        subject.error(e)
+      })
+    yield* from(subject).pipe(withAbort(context.signal))
+  }
+
+  private async handleRequests(
+    requests: AsyncIterable<ProcessStreamRequest>,
+    subject: Subject<DeepPartial<ProcessStreamResponse>>
+  ) {
+    const contexts = new Contexts()
+    for await (const request of requests) {
+      try {
+        // console.debug('received request:', request)
+        if (request.binding) {
+          process_binding_count.add(1)
+
+          // Adjust binding will make some request become invalid by setting UNKNOWN HandlerType
+          // for older SDK version, so we just return empty result for them here
+          if (request.binding.handlerType === HandlerType.UNKNOWN) {
+            subject.next({
+              processId: request.processId,
+              result: ProcessResult.create()
+            })
+            continue
+          }
+
+          const binding = request.binding
+          // todo support db request
+          // const dbContext = contexts.new(request.processId, subject)
+          const start = Date.now()
+          await this.processBinding(binding, undefined)
+            .then(async (result) => {
+              // await all pending db requests
+              // await dbContext.awaitPendings()
+              subject.next({
+                result,
+                processId: request.processId
+              })
+            })
+            .catch((e) => {
+              console.debug(e)
+              // dbContext.error(request.processId, e)
+              process_binding_error.add(1)
+            })
+            .finally(() => {
+              const cost = Date.now() - start
+              process_binding_time.add(cost)
+              contexts.delete(request.processId)
+            })
+        }
+        if (request.dbResult) {
+          const dbContext = contexts.get(request.processId)
+          try {
+            dbContext?.result(request.dbResult)
+          } catch (e) {
+            subject.error(new Error('db result error, process should stop'))
+          }
+        }
+      } catch (e) {
+        // should not happen
+        console.error('unexpect error during handle loop', e)
+      }
+    }
+  }
+}
+
+class Contexts {
+  private contexts: Map<number, StoreContext> = new Map()
+
+  get(processId: number) {
+    return this.contexts.get(processId)
+  }
+
+  new(processId: number, subject: Subject<DeepPartial<ProcessStreamResponse>>) {
+    const context = new StoreContext(subject, processId)
+    this.contexts.set(processId, context)
+    return context
+  }
+
+  delete(processId: number) {
+    const context = this.get(processId)
+    context?.close()
+    this.contexts.delete(processId)
+  }
+}

package/src/service-worker.ts

@@ -0,0 +1,116 @@
+import {
+  DataBinding,
+  Empty,
+  HandlerType,
+  PreparedData,
+  ProcessConfigRequest,
+  ProcessConfigResponse,
+  ProcessResult,
+  StartRequest
+} from '@sentio/protos'
+import { CallContext, ServerError, Status } from 'nice-grpc'
+import { PluginManager } from './plugin.js'
+import commandLineArgs from 'command-line-args'
+import { optionDefinitions } from './processor-runner.js'
+import { errorString } from './utils.js'
+import { freezeGlobalConfig } from './global-config.js'
+import { DebugInfo, RichServerError } from 'nice-grpc-error-details'
+import { recordRuntimeInfo } from './service.js'
+import { BroadcastChannel } from 'worker_threads'
+
+let started = false
+const options = commandLineArgs(optionDefinitions, { partial: true })
+
+let unhandled: Error | undefined
+
+process
+  .on('uncaughtException', (err) => {
+    console.error('Uncaught Exception, please checking if await is properly used', err)
+    unhandled = err
+  })
+  .on('unhandledRejection', (reason, p) => {
+    // @ts-ignore ignore invalid ens error
+    if (reason?.message.startsWith('invalid ENS name (disallowed character: "*"')) {
+      return
+    }
+    console.error('Unhandled Rejection, please checking if await is properly', reason)
+    unhandled = reason as Error
+    // shutdownServers(1)
+  })
+
+async function getConfig(request: ProcessConfigRequest, context?: CallContext): Promise<ProcessConfigResponse> {
+  if (!started) {
+    throw new ServerError(Status.UNAVAILABLE, 'Service Not started.')
+  }
+
+  const newConfig = ProcessConfigResponse.fromPartial({})
+  await PluginManager.INSTANCE.configure(newConfig)
+  return newConfig
+}
+
+const loader = async () => {
+  const m = await import(options.target)
+  console.debug('Module loaded', m)
+  return m
+}
+
+const configureChannel = new BroadcastChannel('configure_channel')
+configureChannel.onmessage = (request: ProcessConfigRequest) => {
+  getConfig(request)
+}
+
+async function start(request: StartRequest): Promise<Empty> {
+  if (started) {
+    return {}
+  }
+  freezeGlobalConfig()
+
+  try {
+    await loader()
+  } catch (e) {
+    throw new ServerError(Status.INVALID_ARGUMENT, 'Failed to load processor: ' + errorString(e))
+  }
+
+  await PluginManager.INSTANCE.start(request)
+  started = true
+  return {}
+}
+
+const startChannel = new BroadcastChannel('start_channel')
+startChannel.onmessage = (request: StartRequest) => {
+  start(request)
+}
+
+async function stop(request: Empty, context: CallContext): Promise<Empty> {
+  return {}
+}
+
+async function processBinding(
+  request: DataBinding,
+  preparedData: PreparedData | undefined,
+  options?: CallContext
+): Promise<ProcessResult> {
+  if (!started) {
+    throw new ServerError(Status.UNAVAILABLE, 'Service Not started.')
+  }
+  if (unhandled) {
+    throw new RichServerError(
+      Status.UNAVAILABLE,
+      'Unhandled exception/rejection in previous request: ' + errorString(unhandled),
+      [
+        DebugInfo.fromPartial({
+          detail: unhandled.message,
+          stackEntries: unhandled.stack?.split('\n')
+        })
+      ]
+    )
+  }
+
+  const result = await PluginManager.INSTANCE.processBinding(
+    request,
+    preparedData
+    // PluginManager.INSTANCE.dbContextLocalStorage.getStore()
+  )
+  recordRuntimeInfo(result, request.handlerType)
+  return result
+}

package/src/service.ts

@@ -474,7 +474,7 @@ export class ProcessorServiceImpl implements ProcessorServiceImplementation {
   }
 }
 
-function recordRuntimeInfo(results: ProcessResult, handlerType: HandlerType) {
+export function recordRuntimeInfo(results: ProcessResult, handlerType: HandlerType) {
   for (const list of [results.gauges, results.counters, results.events, results.exports]) {
     list.forEach((e) => {
       e.runtimeInfo = {