@sentio/runtime 4.0.0-rc.1 → 4.0.0-rc.3

package/src/global-config.ts

@@ -1,4 +1,5 @@
-import { ExecutionConfig } from './gen/processor/protos/processor.js'
+import { ExecutionConfigSchema } from '@sentio/protos'
+import { type MessageInitShape } from '@bufbuild/protobuf'
 
 /**
  * Configuration for the in-memory cache feature.
@@ -33,7 +34,7 @@ export interface GlobalConfig {
    * Execution configuration controlling how handlers are processed.
    * Includes settings for sequential vs parallel execution, block time handling, etc.
    */
-  execution: Partial<ExecutionConfig>
+  execution: MessageInitShape<typeof ExecutionConfigSchema>
 
   /**
    * Optional cache configuration for enabling in-memory key-value storage.

package/src/index.ts

@@ -3,7 +3,7 @@ export * from './state.js'
 export * from './utils.js'
 export * from './endpoints.js'
 export * from './chain-config.js'
-export * from './service.js'
+export * from './service-v3.js'
 export { GLOBAL_CONFIG, type GlobalConfig } from './global-config.js'
 export * from './db-context.js'
 export * from './provider.js'

package/src/plugin.ts

@@ -1,15 +1,20 @@
 import {
-  DataBinding,
+  type DataBinding,
   HandlerType,
-  PreparedData,
-  PreprocessResult,
-  ProcessConfigResponse,
-  ProcessResult,
-  ProcessStreamResponse_Partitions,
+  type PreparedData,
+  type PreprocessResult,
+  PreprocessResultSchema,
+  type ProcessConfigResponse,
+  type ProcessResult,
+  ProcessResultSchema,
+  type StartRequest,
+  StartRequestSchema,
+  type ProcessStreamResponse_Partitions,
+  ProcessStreamResponse_PartitionsSchema,
   ProcessStreamResponse_Partitions_Partition_SysValue,
-  StartRequest,
-  UpdateTemplatesRequest
+  type UpdateTemplatesRequest
 } from '@sentio/protos'
+import { create } from '@bufbuild/protobuf'
 import { IDataBindingContext, IStoreContext } from './db-context.js'
 import { AsyncLocalStorage } from 'node:async_hooks'
 
@@ -29,25 +34,28 @@ export abstract class Plugin {
   }
 
   async processBinding(request: DataBinding, preparedData: PreparedData | undefined): Promise<ProcessResult> {
-    return ProcessResult.create()
+    return create(ProcessResultSchema)
   }
 
   async preprocessBinding(request: DataBinding, preprocessStore: { [k: string]: any }): Promise<PreprocessResult> {
-    return PreprocessResult.create()
+    return create(PreprocessResultSchema)
   }
 
   async partition(request: DataBinding): Promise<ProcessStreamResponse_Partitions> {
-    return {
+    return create(ProcessStreamResponse_PartitionsSchema, {
       partitions: request.handlerIds.reduce(
         (acc, id) => ({
           ...acc,
           [id]: {
-            sysValue: ProcessStreamResponse_Partitions_Partition_SysValue.UNRECOGNIZED
+            value: {
+              case: 'sysValue',
+              value: ProcessStreamResponse_Partitions_Partition_SysValue.BLOCK_NUMBER
+            }
           }
         }),
         {}
       )
-    }
+    })
   }
 
   /**
@@ -147,9 +155,11 @@ export class PluginManager {
 
   async updateTemplates(request: UpdateTemplatesRequest) {
     for (const plugin of this.plugins) {
-      await plugin.start({
-        templateInstances: request.templateInstances
-      })
+      await plugin.start(
+        create(StartRequestSchema, {
+          templateInstances: request.templateInstances
+        })
+      )
     }
   }
 }

package/src/processor-runner.ts

@@ -1,26 +1,22 @@
 #!/usr/bin/env node
 
 import fs from 'fs-extra'
-// import { compressionAlgorithms } from '@grpc/grpc-js'
-import { createServer } from 'nice-grpc'
-import { errorDetailsServerMiddleware } from 'nice-grpc-error-details'
-// import { registry as niceGrpcRegistry } from 'nice-grpc-prometheus'
-// import { openTelemetryServerMiddleware } from 'nice-grpc-opentelemetry'
+import { type ConnectRouter } from '@connectrpc/connect'
+import { connectNodeAdapter } from '@connectrpc/connect-node'
 import http from 'http'
+import http2 from 'node:http2'
 // @ts-ignore inspector promises is not included in @type/node
 import { Session } from 'node:inspector/promises'
 import { fork, ChildProcess } from 'child_process'
 import { fileURLToPath } from 'url'
 
-import { ProcessorDefinition } from './gen/processor/protos/processor.js'
-import { ProcessorServiceImpl } from './service.js'
 import { configureEndpoints } from './endpoints.js'
-import { FullProcessorServiceImpl, FullProcessorServiceV3Impl } from './full-service.js'
+import { FullProcessorServiceV3Impl } from './full-service.js'
 import { setupLogger } from './logger.js'
 
 import { setupOTLP } from './otlp.js'
 import { ActionServer } from './action-server.js'
-import { ProcessorV3Definition } from '@sentio/protos'
+import { ProcessorV3 } from '@sentio/protos'
 import { ProcessorServiceImplV3 } from './service-v3.js'
 import { dirname, join } from 'path'
 import { program, ProcessorRuntimeOptions } from './processor-runner-program.js'
@@ -69,7 +65,7 @@ if (!isChildProcess) {
 }
 
 let server: any
-let baseService: ProcessorServiceImpl
+let processorHttp2Server: http2.Http2Server | undefined
 let httpServer: http.Server | undefined
 
 const loader = async () => {
@@ -82,28 +78,24 @@ if (options.startActionServer) {
   server = new ActionServer(loader)
   server.listen(options.port)
 } else {
-  server = createServer({
-    'grpc.max_send_message_length': 768 * 1024 * 1024,
-    'grpc.max_receive_message_length': 768 * 1024 * 1024
-    // 'grpc.default_compression_algorithm': compressionAlgorithms.gzip
-  })
-    // .use(prometheusServerMiddleware())
-    // .use(openTelemetryServerMiddleware())
-    .use(errorDetailsServerMiddleware)
+  const shutdown = () => processorHttp2Server?.close()
 
-  // for V2
-  baseService = new ProcessorServiceImpl(loader, options, server.shutdown)
-  const service = new FullProcessorServiceImpl(baseService)
+  const serviceV3 = new FullProcessorServiceV3Impl(new ProcessorServiceImplV3(loader, options, shutdown))
 
-  server.add(ProcessorDefinition, service)
+  const routes = (router: ConnectRouter) => {
+    router.service(ProcessorV3, serviceV3)
+  }
 
-  // for V3
-  server.add(
-    ProcessorV3Definition,
-    new FullProcessorServiceV3Impl(new ProcessorServiceImplV3(loader, options, server.shutdown))
-  )
+  const adapter = connectNodeAdapter({
+    routes,
+    readMaxBytes: 768 * 1024 * 1024,
+    writeMaxBytes: 768 * 1024 * 1024
+  })
 
-  server.listen('0.0.0.0:' + options.port)
+  // h2c (cleartext HTTP/2) gRPC server — connect-node serves grpc + grpc-web + connect
+  // on the same port, so the Go launcher's gRPC client is unaffected.
+  processorHttp2Server = http2.createServer(adapter)
+  processorHttp2Server.listen(Number(options.port))
   console.log('Processor Server Started at:', options.port)
 }
 
@@ -208,9 +200,6 @@ process
   })
   .on('uncaughtException', (err) => {
     console.error('Uncaught Exception, please checking if await is properly used', err)
-    if (baseService) {
-      baseService.unhandled = err
-    }
     // shutdownServers(1)
   })
   .on('unhandledRejection', (reason, _p) => {
@@ -219,9 +208,6 @@ process
       return
     }
     console.error('Unhandled Rejection, please checking if await is properly', reason)
-    if (baseService) {
-      baseService.unhandled = reason as Error
-    }
     // shutdownServers(1)
   })
 
@@ -245,7 +231,11 @@ async function dumpHeap(file: string): Promise<void> {
 }
 
 function shutdownServers(exitCode: number): void {
-  server.forceShutdown()
+  if (processorHttp2Server) {
+    processorHttp2Server.close()
+  } else if (server?.forceShutdown) {
+    server.forceShutdown()
+  }
   console.log('RPC server shut down')
 
   if (httpServer) {

package/src/service-v3.ts

@@ -1,38 +1,37 @@
 import {
-  DataBinding,
-  DeepPartial,
-  Empty,
+  type DataBinding,
+  EmptySchema,
   HandlerType,
-  ProcessConfigRequest,
-  ProcessConfigResponse,
-  ProcessorV3ServiceImplementation,
-  ProcessResult,
-  ProcessStreamRequest,
-  ProcessStreamResponse,
-  ProcessStreamResponseV3,
-  StartRequest,
-  UpdateTemplatesRequest
+  type ProcessConfigRequest,
+  ProcessConfigResponseSchema,
+  ProcessorV3,
+  ProcessResultSchema,
+  type ProcessStreamRequest,
+  ProcessStreamResponseV3Schema,
+  type StartRequest,
+  type UpdateTemplatesRequest
 } from '@sentio/protos'
-import { CallContext, ServerError, Status } from 'nice-grpc'
-import { AsyncIterable } from 'ix'
+import { clone, create, type MessageInitShape } from '@bufbuild/protobuf'
+import { ConnectError, Code, type HandlerContext, type ServiceImpl } from '@connectrpc/connect'
 import { PluginManager } from './plugin.js'
 import { Subject } from 'rxjs'
 import { from } from 'ix/asynciterable'
 import { withAbort } from 'ix/asynciterable/operators'
-import { errorString } from './utils.js'
+import { errorString, recordRuntimeInfo } from './utils.js'
 
 import { processMetrics } from './metrics.js'
-import { recordRuntimeInfo } from './service.js'
 import { DataBindingContext } from './db-context.js'
 import { freezeGlobalConfig } from './global-config.js'
 import { ProcessorRuntimeOptions } from './processor-runner-program.js'
 
+type ProcessStreamResponseV3Init = MessageInitShape<typeof ProcessStreamResponseV3Schema>
+
 const { process_binding_count, process_binding_time, process_binding_error } = processMetrics
 
 const WRITE_V2_EVENT_LOGS = process.env.WRITE_V2_EVENT_LOGS !== 'false'
 const TIME_SERIES_RESULT_BATCH_SIZE = 1000
 
-export class ProcessorServiceImplV3 implements ProcessorV3ServiceImplementation {
+export class ProcessorServiceImplV3 implements ServiceImpl<typeof ProcessorV3> {
   readonly enablePartition: boolean
   private readonly loader: () => Promise<any>
   private readonly shutdownHandler?: () => void
@@ -45,9 +44,9 @@ export class ProcessorServiceImplV3 implements ProcessorV3ServiceImplementation
     this.enablePartition = options?.enablePartition == true
   }
 
-  async start(request: StartRequest, context: CallContext): Promise<Empty> {
+  async start(request: StartRequest, context: HandlerContext) {
     if (this.started) {
-      return {}
+      return create(EmptySchema)
     }
 
     freezeGlobalConfig()
@@ -55,27 +54,27 @@ export class ProcessorServiceImplV3 implements ProcessorV3ServiceImplementation
     try {
       await this.loader()
     } catch (e) {
-      throw new ServerError(Status.INVALID_ARGUMENT, 'Failed to load processor: ' + errorString(e))
+      throw new ConnectError('Failed to load processor: ' + errorString(e), Code.InvalidArgument)
     }
 
     await PluginManager.INSTANCE.start(request)
 
     this.started = true
-    return {}
+    return create(EmptySchema)
   }
 
-  async getConfig(request: ProcessConfigRequest, context: CallContext): Promise<ProcessConfigResponse> {
+  async getConfig(request: ProcessConfigRequest, context: HandlerContext) {
     if (!this.started) {
-      throw new ServerError(Status.UNAVAILABLE, 'Service Not started.')
+      throw new ConnectError('Service Not started.', Code.Unavailable)
     }
 
-    const newConfig = ProcessConfigResponse.fromPartial({})
+    const newConfig = create(ProcessConfigResponseSchema, {})
     await PluginManager.INSTANCE.configure(newConfig)
     return newConfig
   }
 
-  async *processBindingsStream(requests: AsyncIterable<ProcessStreamRequest>, context: CallContext) {
-    const subject = new Subject<DeepPartial<ProcessStreamResponseV3>>()
+  async *processBindingsStream(requests: AsyncIterable<ProcessStreamRequest>, context: HandlerContext) {
+    const subject = new Subject<ProcessStreamResponseV3Init>()
     this.handleRequests(requests, subject)
       .then(() => {
         subject.complete()
@@ -89,14 +88,14 @@ export class ProcessorServiceImplV3 implements ProcessorV3ServiceImplementation
 
   protected async handleRequests(
     requests: AsyncIterable<ProcessStreamRequest>,
-    subject: Subject<DeepPartial<ProcessStreamResponse>>
+    subject: Subject<ProcessStreamResponseV3Init>
   ) {
     let lastBinding: DataBinding | undefined = undefined
     for await (const request of requests) {
       try {
         // console.log('received request:', request, 'lastBinding:', lastBinding)
-        if (request.binding) {
-          lastBinding = request.binding
+        if (request.value.case === 'binding') {
+          lastBinding = request.value.value
         }
         this.handleRequest(request, lastBinding, subject)
       } catch (e) {
@@ -111,26 +110,27 @@ export class ProcessorServiceImplV3 implements ProcessorV3ServiceImplementation
   async handleRequest(
     request: ProcessStreamRequest,
     lastBinding: DataBinding | undefined,
-    subject: Subject<DeepPartial<ProcessStreamResponseV3>>
+    subject: Subject<ProcessStreamResponseV3Init>
   ) {
-    if (request.binding) {
+    const binding = request.value.case === 'binding' ? request.value.value : undefined
+    if (binding) {
       process_binding_count.add(1)
 
-      if (request.binding.handlerType === HandlerType.UNKNOWN) {
+      if (binding.handlerType === HandlerType.UNKNOWN) {
         subject.next({
           processId: request.processId,
-          result: ProcessResult.create()
+          value: { case: 'result', value: create(ProcessResultSchema) }
         })
         return
       }
 
       if (this.enablePartition) {
         try {
-          console.debug('sending partition request', request.binding)
-          const partitions = await PluginManager.INSTANCE.partition(request.binding)
+          console.debug('sending partition request', binding)
+          const partitions = await PluginManager.INSTANCE.partition(binding)
           subject.next({
             processId: request.processId,
-            partitions
+            value: { case: 'partitions', value: partitions }
           })
         } catch (e) {
           console.error('Partition error:', e)
@@ -138,11 +138,11 @@ export class ProcessorServiceImplV3 implements ProcessorV3ServiceImplementation
           return
         }
       } else {
-        this.startProcess(request.processId, request.binding, subject)
+        this.startProcess(request.processId, binding, subject)
       }
     }
 
-    if (request.start) {
+    if (request.value.case === 'start') {
       if (!lastBinding) {
         console.error('start request received without binding')
         subject.error(new Error('start request received without binding'))
@@ -151,46 +151,44 @@ export class ProcessorServiceImplV3 implements ProcessorV3ServiceImplementation
       this.startProcess(request.processId, lastBinding, subject)
     }
 
-    if (request.dbResult) {
+    if (request.value.case === 'dbResult') {
       const context = this.contexts.get(request.processId)
       try {
-        context?.result(request.dbResult)
+        context?.result(request.value.value)
       } catch (e) {
         subject.error(new Error('db result error, process should stop'))
       }
     }
   }
 
-  private startProcess(
-    processId: number,
-    binding: DataBinding,
-    subject: Subject<DeepPartial<ProcessStreamResponseV3>>
-  ) {
+  private startProcess(processId: number, binding: DataBinding, subject: Subject<ProcessStreamResponseV3Init>) {
     const context = this.contexts.new(processId, subject)
     const start = Date.now()
     PluginManager.INSTANCE.processBinding(binding, undefined, context)
       .then(async (result) => {
         await context.awaitPendings()
-        const { timeseriesResult, ...otherResults } = result
+        recordRuntimeInfo(result, binding.handlerType)
+
+        const timeseriesResult = result.timeseriesResult
         for (let i = 0; i < timeseriesResult.length; i += TIME_SERIES_RESULT_BATCH_SIZE) {
           const batch = timeseriesResult.slice(i, i + TIME_SERIES_RESULT_BATCH_SIZE)
           subject.next({
             processId,
-            tsRequest: {
-              data: batch
-            }
+            value: { case: 'tsRequest', value: { data: batch } }
           })
         }
 
+        // Send everything except the (already-batched) timeseries result back.
+        const otherResults = clone(ProcessResultSchema, result)
+        otherResults.timeseriesResult = []
+
         subject.next({
-          result: WRITE_V2_EVENT_LOGS
-            ? otherResults
-            : {
-                states: otherResults.states
-              },
-          processId: processId
+          processId,
+          value: {
+            case: 'result',
+            value: WRITE_V2_EVENT_LOGS ? otherResults : create(ProcessResultSchema, { states: result.states })
+          }
         })
-        recordRuntimeInfo(result, binding.handlerType)
       })
       .catch((e) => {
         console.error(e, e.stack)
@@ -204,9 +202,9 @@ export class ProcessorServiceImplV3 implements ProcessorV3ServiceImplementation
       })
   }
 
-  async updateTemplates(request: UpdateTemplatesRequest, context: CallContext): Promise<DeepPartial<Empty>> {
+  async updateTemplates(request: UpdateTemplatesRequest, context: HandlerContext) {
     await PluginManager.INSTANCE.updateTemplates(request)
-    return {}
+    return create(EmptySchema)
   }
 }
 
@@ -217,7 +215,7 @@ class Contexts {
     return this.contexts.get(processId)
   }
 
-  new(processId: number, subject: Subject<DeepPartial<ProcessStreamResponseV3>>) {
+  new(processId: number, subject: Subject<ProcessStreamResponseV3Init>) {
     const context = new DataBindingContext(processId, subject)
     this.contexts.set(processId, context)
     return context

package/src/utils.ts

@@ -1,18 +1,25 @@
-import { EthCallParam, ProcessResult } from '@sentio/protos'
+import {
+  type EthCallParam,
+  HandlerType,
+  type ProcessResult,
+  ProcessResultSchema,
+  RuntimeInfoSchema,
+  StateResultSchema
+} from '@sentio/protos'
+import { create } from '@bufbuild/protobuf'
 import { createRequire } from 'module'
 
-// TODO better handling this, because old proto doesn't have this
-import { StateResult, ProcessResult as ProcessResultFull } from './gen/processor/protos/processor.js'
-
 import { Required } from 'utility-types'
 import path from 'path'
 import fs from 'fs'
+;(BigInt.prototype as any).toJSON = function () {
+  return this.toString()
+}
 
 export function mergeProcessResults(results: ProcessResult[]): Required<ProcessResult, 'states'> {
-  const res = {
-    ...ProcessResultFull.create(),
-    states: StateResult.create()
-  }
+  const res = create(ProcessResultSchema, {
+    states: create(StateResultSchema)
+  })
   return mergeProcessResultsInPlace(res, results)
 }
 
@@ -20,7 +27,7 @@ export function mergeProcessResultsInPlace(
   res: ProcessResult,
   results: ProcessResult[]
 ): Required<ProcessResult, 'states'> {
-  res.states = res.states || StateResult.create()
+  res.states = res.states || create(StateResultSchema)
   for (const r of results) {
     // not using spread operator since it puts all element on the stack
     // cause maximum call stack size exceeded error if it's a large array
@@ -30,11 +37,21 @@ export function mergeProcessResultsInPlace(
     res.events = mergeArrayInPlace(res.events, r.events)
     res.exports = mergeArrayInPlace(res.exports, r.exports)
     res.timeseriesResult = mergeArrayInPlace(res.timeseriesResult, r.timeseriesResult)
-    res.states = {
+    res.states = create(StateResultSchema, {
       configUpdated: res.states?.configUpdated || r.states?.configUpdated || false
-    }
+    })
+  }
+  return res as Required<ProcessResult, 'states'>
+}
+
+export function recordRuntimeInfo(results: ProcessResult, handlerType: HandlerType) {
+  for (const list of [results.gauges, results.counters, results.events, results.exports]) {
+    list.forEach((e) => {
+      e.runtimeInfo = create(RuntimeInfoSchema, {
+        from: handlerType
+      })
+    })
   }
-  return res
 }
 
 function mergeArrayInPlace<T>(dst: T[], src: T[]): T[] {