@sentio/runtime 4.0.0-rc.1 → 4.0.0-rc.2

package/src/global-config.ts

@@ -1,4 +1,5 @@
-import { ExecutionConfig } from './gen/processor/protos/processor.js'
+import { ExecutionConfigSchema } from '@sentio/protos'
+import { type MessageInitShape } from '@bufbuild/protobuf'
 
 /**
  * Configuration for the in-memory cache feature.
@@ -33,7 +34,7 @@ export interface GlobalConfig {
    * Execution configuration controlling how handlers are processed.
    * Includes settings for sequential vs parallel execution, block time handling, etc.
    */
-  execution: Partial<ExecutionConfig>
+  execution: MessageInitShape<typeof ExecutionConfigSchema>
 
   /**
    * Optional cache configuration for enabling in-memory key-value storage.

package/src/plugin.ts

@@ -1,15 +1,20 @@
 import {
-  DataBinding,
+  type DataBinding,
   HandlerType,
-  PreparedData,
-  PreprocessResult,
-  ProcessConfigResponse,
-  ProcessResult,
-  ProcessStreamResponse_Partitions,
+  type PreparedData,
+  type PreprocessResult,
+  PreprocessResultSchema,
+  type ProcessConfigResponse,
+  type ProcessResult,
+  ProcessResultSchema,
+  type StartRequest,
+  StartRequestSchema,
+  type ProcessStreamResponse_Partitions,
+  ProcessStreamResponse_PartitionsSchema,
   ProcessStreamResponse_Partitions_Partition_SysValue,
-  StartRequest,
-  UpdateTemplatesRequest
+  type UpdateTemplatesRequest
 } from '@sentio/protos'
+import { create } from '@bufbuild/protobuf'
 import { IDataBindingContext, IStoreContext } from './db-context.js'
 import { AsyncLocalStorage } from 'node:async_hooks'
 
@@ -29,25 +34,28 @@ export abstract class Plugin {
   }
 
   async processBinding(request: DataBinding, preparedData: PreparedData | undefined): Promise<ProcessResult> {
-    return ProcessResult.create()
+    return create(ProcessResultSchema)
   }
 
   async preprocessBinding(request: DataBinding, preprocessStore: { [k: string]: any }): Promise<PreprocessResult> {
-    return PreprocessResult.create()
+    return create(PreprocessResultSchema)
   }
 
   async partition(request: DataBinding): Promise<ProcessStreamResponse_Partitions> {
-    return {
+    return create(ProcessStreamResponse_PartitionsSchema, {
       partitions: request.handlerIds.reduce(
         (acc, id) => ({
           ...acc,
           [id]: {
-            sysValue: ProcessStreamResponse_Partitions_Partition_SysValue.UNRECOGNIZED
+            value: {
+              case: 'sysValue',
+              value: ProcessStreamResponse_Partitions_Partition_SysValue.BLOCK_NUMBER
+            }
           }
         }),
         {}
       )
-    }
+    })
   }
 
   /**
@@ -147,9 +155,11 @@ export class PluginManager {
 
   async updateTemplates(request: UpdateTemplatesRequest) {
     for (const plugin of this.plugins) {
-      await plugin.start({
-        templateInstances: request.templateInstances
-      })
+      await plugin.start(
+        create(StartRequestSchema, {
+          templateInstances: request.templateInstances
+        })
+      )
     }
   }
 }

package/src/processor-runner.ts

@@ -1,18 +1,15 @@
 #!/usr/bin/env node
 
 import fs from 'fs-extra'
-// import { compressionAlgorithms } from '@grpc/grpc-js'
-import { createServer } from 'nice-grpc'
-import { errorDetailsServerMiddleware } from 'nice-grpc-error-details'
-// import { registry as niceGrpcRegistry } from 'nice-grpc-prometheus'
-// import { openTelemetryServerMiddleware } from 'nice-grpc-opentelemetry'
+import { type ConnectRouter } from '@connectrpc/connect'
+import { connectNodeAdapter } from '@connectrpc/connect-node'
 import http from 'http'
+import http2 from 'node:http2'
 // @ts-ignore inspector promises is not included in @type/node
 import { Session } from 'node:inspector/promises'
 import { fork, ChildProcess } from 'child_process'
 import { fileURLToPath } from 'url'
 
-import { ProcessorDefinition } from './gen/processor/protos/processor.js'
 import { ProcessorServiceImpl } from './service.js'
 import { configureEndpoints } from './endpoints.js'
 import { FullProcessorServiceImpl, FullProcessorServiceV3Impl } from './full-service.js'
@@ -20,7 +17,7 @@ import { setupLogger } from './logger.js'
 
 import { setupOTLP } from './otlp.js'
 import { ActionServer } from './action-server.js'
-import { ProcessorV3Definition } from '@sentio/protos'
+import { Processor, ProcessorV3 } from '@sentio/protos'
 import { ProcessorServiceImplV3 } from './service-v3.js'
 import { dirname, join } from 'path'
 import { program, ProcessorRuntimeOptions } from './processor-runner-program.js'
@@ -69,6 +66,7 @@ if (!isChildProcess) {
 }
 
 let server: any
+let processorHttp2Server: http2.Http2Server | undefined
 let baseService: ProcessorServiceImpl
 let httpServer: http.Server | undefined
 
@@ -82,28 +80,30 @@ if (options.startActionServer) {
   server = new ActionServer(loader)
   server.listen(options.port)
 } else {
-  server = createServer({
-    'grpc.max_send_message_length': 768 * 1024 * 1024,
-    'grpc.max_receive_message_length': 768 * 1024 * 1024
-    // 'grpc.default_compression_algorithm': compressionAlgorithms.gzip
-  })
-    // .use(prometheusServerMiddleware())
-    // .use(openTelemetryServerMiddleware())
-    .use(errorDetailsServerMiddleware)
+  const shutdown = () => processorHttp2Server?.close()
 
   // for V2
-  baseService = new ProcessorServiceImpl(loader, options, server.shutdown)
-  const service = new FullProcessorServiceImpl(baseService)
-
-  server.add(ProcessorDefinition, service)
+  baseService = new ProcessorServiceImpl(loader, options, shutdown)
+  const serviceV2 = new FullProcessorServiceImpl(baseService)
 
   // for V3
-  server.add(
-    ProcessorV3Definition,
-    new FullProcessorServiceV3Impl(new ProcessorServiceImplV3(loader, options, server.shutdown))
-  )
+  const serviceV3 = new FullProcessorServiceV3Impl(new ProcessorServiceImplV3(loader, options, shutdown))
+
+  const routes = (router: ConnectRouter) => {
+    router.service(Processor, serviceV2)
+    router.service(ProcessorV3, serviceV3)
+  }
 
-  server.listen('0.0.0.0:' + options.port)
+  const adapter = connectNodeAdapter({
+    routes,
+    readMaxBytes: 768 * 1024 * 1024,
+    writeMaxBytes: 768 * 1024 * 1024
+  })
+
+  // h2c (cleartext HTTP/2) gRPC server — connect-node serves grpc + grpc-web + connect
+  // on the same port, so the Go launcher's gRPC client is unaffected.
+  processorHttp2Server = http2.createServer(adapter)
+  processorHttp2Server.listen(Number(options.port))
   console.log('Processor Server Started at:', options.port)
 }
 
@@ -245,7 +245,11 @@ async function dumpHeap(file: string): Promise<void> {
 }
 
 function shutdownServers(exitCode: number): void {
-  server.forceShutdown()
+  if (processorHttp2Server) {
+    processorHttp2Server.close()
+  } else if (server?.forceShutdown) {
+    server.forceShutdown()
+  }
   console.log('RPC server shut down')
 
   if (httpServer) {

package/src/service-v3.ts

@@ -1,20 +1,18 @@
 import {
-  DataBinding,
-  DeepPartial,
-  Empty,
+  type DataBinding,
+  EmptySchema,
   HandlerType,
-  ProcessConfigRequest,
-  ProcessConfigResponse,
-  ProcessorV3ServiceImplementation,
-  ProcessResult,
-  ProcessStreamRequest,
-  ProcessStreamResponse,
-  ProcessStreamResponseV3,
-  StartRequest,
-  UpdateTemplatesRequest
+  type ProcessConfigRequest,
+  ProcessConfigResponseSchema,
+  ProcessorV3,
+  ProcessResultSchema,
+  type ProcessStreamRequest,
+  ProcessStreamResponseV3Schema,
+  type StartRequest,
+  type UpdateTemplatesRequest
 } from '@sentio/protos'
-import { CallContext, ServerError, Status } from 'nice-grpc'
-import { AsyncIterable } from 'ix'
+import { clone, create, type MessageInitShape } from '@bufbuild/protobuf'
+import { ConnectError, Code, type HandlerContext, type ServiceImpl } from '@connectrpc/connect'
 import { PluginManager } from './plugin.js'
 import { Subject } from 'rxjs'
 import { from } from 'ix/asynciterable'
@@ -27,12 +25,14 @@ import { DataBindingContext } from './db-context.js'
 import { freezeGlobalConfig } from './global-config.js'
 import { ProcessorRuntimeOptions } from './processor-runner-program.js'
 
+type ProcessStreamResponseV3Init = MessageInitShape<typeof ProcessStreamResponseV3Schema>
+
 const { process_binding_count, process_binding_time, process_binding_error } = processMetrics
 
 const WRITE_V2_EVENT_LOGS = process.env.WRITE_V2_EVENT_LOGS !== 'false'
 const TIME_SERIES_RESULT_BATCH_SIZE = 1000
 
-export class ProcessorServiceImplV3 implements ProcessorV3ServiceImplementation {
+export class ProcessorServiceImplV3 implements ServiceImpl<typeof ProcessorV3> {
   readonly enablePartition: boolean
   private readonly loader: () => Promise<any>
   private readonly shutdownHandler?: () => void
@@ -45,9 +45,9 @@ export class ProcessorServiceImplV3 implements ProcessorV3ServiceImplementation
     this.enablePartition = options?.enablePartition == true
   }
 
-  async start(request: StartRequest, context: CallContext): Promise<Empty> {
+  async start(request: StartRequest, context: HandlerContext) {
     if (this.started) {
-      return {}
+      return create(EmptySchema)
     }
 
     freezeGlobalConfig()
@@ -55,27 +55,27 @@ export class ProcessorServiceImplV3 implements ProcessorV3ServiceImplementation
     try {
       await this.loader()
     } catch (e) {
-      throw new ServerError(Status.INVALID_ARGUMENT, 'Failed to load processor: ' + errorString(e))
+      throw new ConnectError('Failed to load processor: ' + errorString(e), Code.InvalidArgument)
     }
 
     await PluginManager.INSTANCE.start(request)
 
     this.started = true
-    return {}
+    return create(EmptySchema)
   }
 
-  async getConfig(request: ProcessConfigRequest, context: CallContext): Promise<ProcessConfigResponse> {
+  async getConfig(request: ProcessConfigRequest, context: HandlerContext) {
     if (!this.started) {
-      throw new ServerError(Status.UNAVAILABLE, 'Service Not started.')
+      throw new ConnectError('Service Not started.', Code.Unavailable)
     }
 
-    const newConfig = ProcessConfigResponse.fromPartial({})
+    const newConfig = create(ProcessConfigResponseSchema, {})
     await PluginManager.INSTANCE.configure(newConfig)
     return newConfig
   }
 
-  async *processBindingsStream(requests: AsyncIterable<ProcessStreamRequest>, context: CallContext) {
-    const subject = new Subject<DeepPartial<ProcessStreamResponseV3>>()
+  async *processBindingsStream(requests: AsyncIterable<ProcessStreamRequest>, context: HandlerContext) {
+    const subject = new Subject<ProcessStreamResponseV3Init>()
     this.handleRequests(requests, subject)
       .then(() => {
         subject.complete()
@@ -89,14 +89,14 @@ export class ProcessorServiceImplV3 implements ProcessorV3ServiceImplementation
 
   protected async handleRequests(
     requests: AsyncIterable<ProcessStreamRequest>,
-    subject: Subject<DeepPartial<ProcessStreamResponse>>
+    subject: Subject<ProcessStreamResponseV3Init>
   ) {
     let lastBinding: DataBinding | undefined = undefined
     for await (const request of requests) {
       try {
         // console.log('received request:', request, 'lastBinding:', lastBinding)
-        if (request.binding) {
-          lastBinding = request.binding
+        if (request.value.case === 'binding') {
+          lastBinding = request.value.value
         }
         this.handleRequest(request, lastBinding, subject)
       } catch (e) {
@@ -111,26 +111,27 @@ export class ProcessorServiceImplV3 implements ProcessorV3ServiceImplementation
   async handleRequest(
     request: ProcessStreamRequest,
     lastBinding: DataBinding | undefined,
-    subject: Subject<DeepPartial<ProcessStreamResponseV3>>
+    subject: Subject<ProcessStreamResponseV3Init>
   ) {
-    if (request.binding) {
+    const binding = request.value.case === 'binding' ? request.value.value : undefined
+    if (binding) {
       process_binding_count.add(1)
 
-      if (request.binding.handlerType === HandlerType.UNKNOWN) {
+      if (binding.handlerType === HandlerType.UNKNOWN) {
         subject.next({
           processId: request.processId,
-          result: ProcessResult.create()
+          value: { case: 'result', value: create(ProcessResultSchema) }
         })
         return
       }
 
       if (this.enablePartition) {
         try {
-          console.debug('sending partition request', request.binding)
-          const partitions = await PluginManager.INSTANCE.partition(request.binding)
+          console.debug('sending partition request', binding)
+          const partitions = await PluginManager.INSTANCE.partition(binding)
           subject.next({
             processId: request.processId,
-            partitions
+            value: { case: 'partitions', value: partitions }
           })
         } catch (e) {
           console.error('Partition error:', e)
@@ -138,11 +139,11 @@ export class ProcessorServiceImplV3 implements ProcessorV3ServiceImplementation
           return
         }
       } else {
-        this.startProcess(request.processId, request.binding, subject)
+        this.startProcess(request.processId, binding, subject)
       }
     }
 
-    if (request.start) {
+    if (request.value.case === 'start') {
       if (!lastBinding) {
         console.error('start request received without binding')
         subject.error(new Error('start request received without binding'))
@@ -151,44 +152,41 @@ export class ProcessorServiceImplV3 implements ProcessorV3ServiceImplementation
       this.startProcess(request.processId, lastBinding, subject)
     }
 
-    if (request.dbResult) {
+    if (request.value.case === 'dbResult') {
       const context = this.contexts.get(request.processId)
       try {
-        context?.result(request.dbResult)
+        context?.result(request.value.value)
       } catch (e) {
         subject.error(new Error('db result error, process should stop'))
       }
     }
   }
 
-  private startProcess(
-    processId: number,
-    binding: DataBinding,
-    subject: Subject<DeepPartial<ProcessStreamResponseV3>>
-  ) {
+  private startProcess(processId: number, binding: DataBinding, subject: Subject<ProcessStreamResponseV3Init>) {
     const context = this.contexts.new(processId, subject)
     const start = Date.now()
     PluginManager.INSTANCE.processBinding(binding, undefined, context)
       .then(async (result) => {
         await context.awaitPendings()
-        const { timeseriesResult, ...otherResults } = result
+        const timeseriesResult = result.timeseriesResult
         for (let i = 0; i < timeseriesResult.length; i += TIME_SERIES_RESULT_BATCH_SIZE) {
           const batch = timeseriesResult.slice(i, i + TIME_SERIES_RESULT_BATCH_SIZE)
           subject.next({
             processId,
-            tsRequest: {
-              data: batch
-            }
+            value: { case: 'tsRequest', value: { data: batch } }
           })
         }
 
+        // Send everything except the (already-batched) timeseries result back.
+        const otherResults = clone(ProcessResultSchema, result)
+        otherResults.timeseriesResult = []
+
         subject.next({
-          result: WRITE_V2_EVENT_LOGS
-            ? otherResults
-            : {
-                states: otherResults.states
-              },
-          processId: processId
+          processId,
+          value: {
+            case: 'result',
+            value: WRITE_V2_EVENT_LOGS ? otherResults : create(ProcessResultSchema, { states: result.states })
+          }
         })
         recordRuntimeInfo(result, binding.handlerType)
       })
@@ -204,9 +202,9 @@ export class ProcessorServiceImplV3 implements ProcessorV3ServiceImplementation
       })
   }
 
-  async updateTemplates(request: UpdateTemplatesRequest, context: CallContext): Promise<DeepPartial<Empty>> {
+  async updateTemplates(request: UpdateTemplatesRequest, context: HandlerContext) {
     await PluginManager.INSTANCE.updateTemplates(request)
-    return {}
+    return create(EmptySchema)
   }
 }
 
@@ -217,7 +215,7 @@ class Contexts {
     return this.contexts.get(processId)
   }
 
-  new(processId: number, subject: Subject<DeepPartial<ProcessStreamResponseV3>>) {
+  new(processId: number, subject: Subject<ProcessStreamResponseV3Init>) {
     const context = new DataBindingContext(processId, subject)
     this.contexts.set(processId, context)
     return context