@sentio/runtime 2.61.1-rc.1 → 2.62.0-rc.10

package/src/processor-runner.ts

@@ -3,7 +3,7 @@
 import fs from 'fs-extra'
 
 import { compressionAlgorithms } from '@grpc/grpc-js'
-import commandLineArgs from 'command-line-args'
+
 import { createServer } from 'nice-grpc'
 import { errorDetailsServerMiddleware } from 'nice-grpc-error-details'
 // import { registry as niceGrpcRegistry } from 'nice-grpc-prometheus'
@@ -21,54 +21,24 @@ import { setupLogger } from './logger.js'
 import { setupOTLP } from './otlp.js'
 import { ActionServer } from './action-server.js'
 import { ServiceManager } from './service-manager.js'
-import path from 'path'
 import { ProcessorV3Definition } from '@sentio/protos'
 import { ProcessorServiceImplV3 } from './service-v3.js'
+import { dirname, join } from 'path'
+import { program, ProcessorRuntimeOptions } from 'processor-runner-program.js'
 
-// const mergedRegistry = Registry.merge([globalRegistry, niceGrpcRegistry])
+program.parse()
 
-let workerNum = 1
-try {
-  workerNum = parseInt(process.env['PROCESSOR_WORKER']?.trim() ?? '1')
-} catch (e) {
-  console.error('Failed to parse worker number', e)
+const options: ProcessorRuntimeOptions = {
+  ...program.opts(),
+  target: program.args[program.args.length - 1]
 }
-export const optionDefinitions = [
-  { name: 'target', type: String, defaultOption: true },
-  { name: 'port', alias: 'p', type: String, defaultValue: '4000' },
-  { name: 'concurrency', type: Number, defaultValue: 4 },
-  { name: 'batch-count', type: Number, defaultValue: 1 },
-  // { name: 'use-chainserver', type: Boolean, defaultValue: false },
-  {
-    name: 'chains-config',
-    alias: 'c',
-    type: String,
-    defaultValue: 'chains-config.json'
-  },
-  { name: 'chainquery-server', type: String, defaultValue: '' },
-  { name: 'pricefeed-server', type: String, defaultValue: '' },
-  { name: 'log-format', type: String, defaultValue: 'console' },
-  { name: 'debug', type: Boolean, defaultValue: false },
-  { name: 'otlp-debug', type: Boolean, defaultValue: false },
-  { name: 'start-action-server', type: Boolean, defaultValue: false },
-  { name: 'worker', type: Number, defaultValue: workerNum },
-  { name: 'process-timeout', type: Number, defaultValue: 60 },
-  { name: 'worker-timeout', type: Number, defaultValue: parseInt(process.env['WORKER_TIMEOUT_SECONDS'] || '60') },
-  {
-    name: 'enable-partition',
-    type: Boolean,
-    defaultValue: process.env['SENTIO_ENABLE_BINDING_DATA_PARTITION'] === 'true'
-  }
-]
-
-const options = commandLineArgs(optionDefinitions, { partial: true })
 
 const logLevel = process.env['LOG_LEVEL']?.toLowerCase()
 
-setupLogger(options['log-format'] === 'json', logLevel === 'debug' ? true : options.debug)
+setupLogger(options.logFormat === 'json', logLevel === 'debug' ? true : options.debug!)
 console.debug('Starting with', options.target)
 
-await setupOTLP(options['otlp-debug'])
+await setupOTLP(options.otlpDebug)
 
 Error.stackTraceLimit = 20
 
@@ -83,7 +53,7 @@ const loader = async () => {
   console.debug('Module loaded', m)
   return m
 }
-if (options['start-action-server']) {
+if (options.startActionServer) {
   server = new ActionServer(loader)
   server.listen(options.port)
 } else {
@@ -109,11 +79,10 @@ if (options['start-action-server']) {
     ProcessorV3Definition,
     new FullProcessorServiceV3Impl(new ProcessorServiceImplV3(loader, options, server.shutdown))
   )
-  server.listen('0.0.0.0:' + options.port)
 
+  server.listen('0.0.0.0:' + options.port)
   console.log('Processor Server Started at:', options.port)
 }
-
 const metricsPort = 4040
 
 const httpServer = http
@@ -198,7 +167,7 @@ process
 
 if (process.env['OOM_DUMP_MEMORY_SIZE_GB']) {
   let dumping = false
-  const memorySize = parseFloat(process.env['OOM_DUMP_MEMORY_SIZE_GB'])
+  const memorySize = parseFloat(process.env['OOM_DUMP_MEMORY_SIZE_GB']!)
   console.log('heap dumping is enabled, limit set to ', memorySize, 'gb')
   const dir = process.env['OOM_DUMP_DIR'] || '/tmp'
   setInterval(async () => {
@@ -206,7 +175,7 @@ if (process.env['OOM_DUMP_MEMORY_SIZE_GB']) {
     console.log('Current Memory Usage', mem)
     // if memory usage is greater this size, dump heap and exit
     if (mem.heapTotal > memorySize * 1024 * 1024 * 1024 && !dumping) {
-      const file = path.join(dir, `${Date.now()}.heapsnapshot`)
+      const file = join(dir, `${Date.now()}.heapsnapshot`)
       dumping = true
       await dumpHeap(file)
       // force exit and keep pod running
@@ -214,15 +183,16 @@ if (process.env['OOM_DUMP_MEMORY_SIZE_GB']) {
     }
   }, 1000 * 60)
 }
+// }
 
-async function dumpHeap(file: string) {
+async function dumpHeap(file: string): Promise<void> {
   console.log('Heap dumping to', file)
   const session = new Session()
-  fs.mkdirSync(path.dirname(file), { recursive: true })
+  fs.mkdirSync(dirname(file), { recursive: true })
   const fd = fs.openSync(file, 'w')
   try {
     session.connect()
-    session.on('HeapProfiler.addHeapSnapshotChunk', (m) => {
+    session.on('HeapProfiler.addHeapSnapshotChunk', (m: any) => {
       fs.writeSync(fd, m.params.chunk)
     })
 
@@ -234,8 +204,8 @@ async function dumpHeap(file: string) {
   }
 }
 
-function shutdownServers(exitCode: number) {
-  server?.forceShutdown()
+function shutdownServers(exitCode: number): void {
+  server.forceShutdown()
   console.log('RPC server shut down')
 
   httpServer.close(function () {

package/src/service-manager.ts

@@ -16,6 +16,7 @@ import { Subject } from 'rxjs'
 import { MessageChannel } from 'node:worker_threads'
 import { ProcessorServiceImpl } from './service.js'
 import { TemplateInstanceState } from './state.js'
+import { ProcessorRuntimeOptions } from 'processor-runner-program.js'
 ;(BigInt.prototype as any).toJSON = function () {
   return this.toString()
 }
@@ -26,7 +27,7 @@ export class ServiceManager extends ProcessorServiceImpl {
 
   constructor(
     loader: () => Promise<any>,
-    readonly options: any,
+    readonly options: ProcessorRuntimeOptions,
     shutdownHandler?: () => void
   ) {
     super(loader, options, shutdownHandler)
@@ -123,7 +124,7 @@ export class ServiceManager extends ProcessorServiceImpl {
 
     if (this.enablePartition) {
       const concurrent = parseInt(process.env['PROCESS_CONCURRENCY'] || '0')
-      if (this.options.worker < concurrent) {
+      if (this.options.worker! < concurrent) {
         console.warn(
           `When partition is enabled, the worker count must >= 'PROCESS_CONCURRENCY', will set worker count to ${concurrent})`
         )
@@ -139,7 +140,8 @@ export class ServiceManager extends ProcessorServiceImpl {
       argv: process.argv,
       workerData: this.workerData
     })
-    this.pool.on('message', (msg) => {
+    // @ts-ignore - Piscina message handling for template instance sync
+    this.pool.on('message', (msg: any) => {
       if (msg.event == 'add_template_instance') {
         // sync the template state from worker to the main thread
         TemplateInstanceState.INSTANCE.addValue(msg.value)

package/src/service-v3.ts

@@ -26,6 +26,7 @@ import { recordRuntimeInfo } from './service.js'
 import { DataBindingContext } from './db-context.js'
 import { TemplateInstanceState } from './state.js'
 import { freezeGlobalConfig } from './global-config.js'
+import { ProcessorRuntimeOptions } from 'processor-runner-program.js'
 
 const { process_binding_count, process_binding_time, process_binding_error } = processMetrics
 
@@ -35,11 +36,11 @@ export class ProcessorServiceImplV3 implements ProcessorV3ServiceImplementation
   private readonly shutdownHandler?: () => void
   private started = false
 
-  constructor(loader: () => Promise<any>, options?: any, shutdownHandler?: () => void) {
+  constructor(loader: () => Promise<any>, options?: ProcessorRuntimeOptions, shutdownHandler?: () => void) {
     this.loader = loader
     this.shutdownHandler = shutdownHandler
 
-    this.enablePartition = options?.['enable-partition'] == true
+    this.enablePartition = options?.enablePartition == true
   }
 
   async start(request: StartRequest, context: CallContext): Promise<Empty> {
@@ -170,9 +171,9 @@ export class ProcessorServiceImplV3 implements ProcessorV3ServiceImplementation
       .then(async (result) => {
         console.debug(`process binding ${processId} done`)
         await context.awaitPendings()
-
+        const { timeseriesResult, ...otherResults } = result
         console.debug('sending ts data length:', result.timeseriesResult.length)
-        for (const ts of result.timeseriesResult) {
+        for (const ts of timeseriesResult) {
           subject.next({
             processId,
             tsRequest: {
@@ -193,10 +194,7 @@ export class ProcessorServiceImplV3 implements ProcessorV3ServiceImplementation
 
         console.debug('sending binding result', processId)
         subject.next({
-          result: {
-            states: result.states,
-            exports: result.exports
-          },
+          result: otherResults,
           processId: processId
         })
         recordRuntimeInfo(result, binding.handlerType)

package/src/service-worker.ts

@@ -9,6 +9,7 @@ import { Piscina } from 'piscina'
 import { configureEndpoints } from './endpoints.js'
 import { setupLogger } from './logger.js'
 import { Subject } from 'rxjs'
+import { ProcessorRuntimeOptions } from 'processor-runner-program.js'
 
 let started = false
 
@@ -34,7 +35,7 @@ process
 
 let service: ProcessorServiceImpl | undefined
 
-const loader = async (options: any) => {
+const loader = async (options: ProcessorRuntimeOptions) => {
   if (options.target) {
     const m = await import(options.target)
     console.debug('Module loaded, path:', options.target, 'module:', m)
@@ -44,7 +45,7 @@ const loader = async (options: any) => {
 
 const emptyCallContext = <CallContext>{}
 
-async function start(request: StartRequest, options: any): Promise<Empty> {
+async function start(request: StartRequest, options: ProcessorRuntimeOptions): Promise<Empty> {
   if (started) {
     return {}
   }
@@ -73,7 +74,7 @@ export default async function ({
   const { startRequest, configRequest, options } = Piscina.workerData
   if (!started) {
     const logLevel = process.env['LOG_LEVEL']?.toUpperCase()
-    setupLogger(options['log-format'] === 'json', logLevel === 'debug' ? true : options.debug, threadId)
+    setupLogger(options.logFormat === 'json', logLevel === 'debug' ? true : options.debug, threadId)
 
     configureEndpoints(options)
 
@@ -103,8 +104,8 @@ export default async function ({
       ]
     )
   }
-  const timeout = (options['worker-timeout'] || 0) * 1000 // convert to milliseconds
-  const enablePartition = options['enable-partition'] || false
+  const timeout = (options.workerTimeout || 0) * 1000 // convert to milliseconds
+  const enablePartition = options.enablePartition || false
   await new Promise<void>((resolve, reject) => {
     const subject = new Subject<DeepPartial<ProcessStreamResponse>>()
     let timeoutId: NodeJS.Timeout | undefined = undefined

package/src/service.ts

@@ -36,6 +36,7 @@ import { Provider } from 'ethers'
 import { decodeMulticallResult, encodeMulticallData, getMulticallAddress, Multicall3Call } from './multicall.js'
 
 import { processMetrics } from './metrics.js'
+import { ProcessorRuntimeOptions } from 'processor-runner-program.js'
 
 const { process_binding_count, process_binding_time, process_binding_error } = processMetrics
 
@@ -58,7 +59,7 @@ export class ProcessorServiceImpl implements ProcessorServiceImplementation {
   private preparedData: PreparedData | undefined
   readonly enablePartition: boolean
 
-  constructor(loader: () => Promise<any>, options?: any, shutdownHandler?: () => void) {
+  constructor(loader: () => Promise<any>, options?: ProcessorRuntimeOptions, shutdownHandler?: () => void) {
     this.loader = loader
     this.shutdownHandler = shutdownHandler
 
@@ -66,7 +67,7 @@ export class ProcessorServiceImpl implements ProcessorServiceImplementation {
       ? process.env['ENABLE_PREPROCESS'].toLowerCase() == 'true'
       : false
 
-    this.enablePartition = options?.['enable-partition'] == true
+    this.enablePartition = options?.enablePartition == true
   }
 
   async getConfig(request: ProcessConfigRequest, context: CallContext): Promise<ProcessConfigResponse> {