@sentio/runtime 2.59.0-rc.9 → 2.59.1-rc.1

package/src/gen/service/common/protos/common.ts

@@ -1592,6 +1592,7 @@ export interface Account {
   address: string;
   paymentMethod: PayMethod;
   usageOverCapLimit: string;
+  status: string;
 }
 
 export interface ImportedProject {
@@ -10316,6 +10317,7 @@ function createBaseAccount(): Account {
     address: "",
     paymentMethod: 0,
     usageOverCapLimit: "",
+    status: "",
   };
 }
 
@@ -10348,6 +10350,9 @@ export const Account = {
     if (message.usageOverCapLimit !== "") {
       writer.uint32(90).string(message.usageOverCapLimit);
     }
+    if (message.status !== "") {
+      writer.uint32(98).string(message.status);
+    }
     return writer;
   },
 
@@ -10421,6 +10426,13 @@ export const Account = {
 
           message.usageOverCapLimit = reader.string();
           continue;
+        case 12:
+          if (tag !== 98) {
+            break;
+          }
+
+          message.status = reader.string();
+          continue;
       }
       if ((tag & 7) === 4 || tag === 0) {
         break;
@@ -10441,6 +10453,7 @@ export const Account = {
       address: isSet(object.address) ? globalThis.String(object.address) : "",
       paymentMethod: isSet(object.paymentMethod) ? payMethodFromJSON(object.paymentMethod) : 0,
       usageOverCapLimit: isSet(object.usageOverCapLimit) ? globalThis.String(object.usageOverCapLimit) : "",
+      status: isSet(object.status) ? globalThis.String(object.status) : "",
     };
   },
 
@@ -10473,6 +10486,9 @@ export const Account = {
     if (message.usageOverCapLimit !== "") {
       obj.usageOverCapLimit = message.usageOverCapLimit;
     }
+    if (message.status !== "") {
+      obj.status = message.status;
+    }
     return obj;
   },
 
@@ -10490,6 +10506,7 @@ export const Account = {
     message.address = object.address ?? "";
     message.paymentMethod = object.paymentMethod ?? 0;
     message.usageOverCapLimit = object.usageOverCapLimit ?? "";
+    message.status = object.status ?? "";
     return message;
   },
 };

package/src/metrics.ts

@@ -77,25 +77,29 @@ export const dbMetrics = {
     get: new C('store_get_send'),
     upsert: new C('store_upsert_send'),
     list: new C('store_list_send'),
-    delete: new C('store_delete_send')
+    delete: new C('store_delete_send'),
+    update: new C('store_update_send')
   },
   recv_counts: {
     get: new C('store_get_recv'),
     upsert: new C('store_upsert_recv'),
     list: new C('store_list_recv'),
-    delete: new C('store_delete_recv')
+    delete: new C('store_delete_recv'),
+    update: new C('store_update_recv')
   },
   request_times: {
     get: new C('store_get_time'),
     upsert: new C('store_upsert_time'),
     list: new C('store_list_time'),
-    delete: new C('store_delete_time')
+    delete: new C('store_delete_time'),
+    update: new C('store_update_time')
   },
   request_errors: {
     get: new C('store_get_error'),
     upsert: new C('store_upsert_error'),
     list: new C('store_list_error'),
-    delete: new C('store_delete_error')
+    delete: new C('store_delete_error'),
+    update: new C('store_update_error')
   },
   batched_total_count: new C('batched_total_count'),
   batched_request_count: new C('batched_request_count'),

package/src/plugin.ts

@@ -5,6 +5,8 @@ import {
   PreprocessResult,
   ProcessConfigResponse,
   ProcessResult,
+  ProcessStreamResponse_Partitions,
+  ProcessStreamResponse_Partitions_Partition_SysValue,
   StartRequest
 } from '@sentio/protos'
 import { IStoreContext, StoreContext } from './db-context.js'
@@ -33,6 +35,20 @@ export abstract class Plugin {
     return PreprocessResult.create()
   }
 
+  async partition(request: DataBinding): Promise<ProcessStreamResponse_Partitions> {
+    return {
+      partitions: request.handlerIds.reduce(
+        (acc, id) => ({
+          ...acc,
+          [id]: {
+            sysValue: ProcessStreamResponse_Partitions_Partition_SysValue.UNRECOGNIZED
+          }
+        }),
+        {}
+      )
+    }
+  }
+
   /**
    * method used by action server only
    * @param port
@@ -104,6 +120,14 @@ export class PluginManager {
     })
   }
 
+  async partition(request: DataBinding): Promise<ProcessStreamResponse_Partitions> {
+    const plugin = this.typesToPlugin.get(request.handlerType)
+    if (!plugin) {
+      throw new Error(`No plugin for ${request.handlerType}`)
+    }
+    return plugin.partition(request)
+  }
+
   preprocessBinding(
     request: DataBinding,
     preprocessStore: { [k: string]: any },

package/src/processor-runner.ts

@@ -49,7 +49,14 @@ export const optionDefinitions = [
   { name: 'debug', type: Boolean, defaultValue: false },
   { name: 'otlp-debug', type: Boolean, defaultValue: false },
   { name: 'start-action-server', type: Boolean, defaultValue: false },
-  { name: 'worker', type: Number, defaultValue: workerNum }
+  { name: 'worker', type: Number, defaultValue: workerNum },
+  { name: 'process-timeout', type: Number, defaultValue: 60 },
+  { name: 'worker-timeout', type: Number, defaultValue: 60 },
+  {
+    name: 'enable-partition',
+    type: Boolean,
+    defaultValue: process.env['SENTIO_ENABLE_BINDING_DATA_PARTITION'] === 'true'
+  }
 ]
 
 const options = commandLineArgs(optionDefinitions, { partial: true })
@@ -88,9 +95,9 @@ if (options['start-action-server']) {
     .use(errorDetailsServerMiddleware)
 
   if (options.worker > 1) {
-    baseService = new ServiceManager(options, loader, server.shutdown)
+    baseService = new ServiceManager(loader, options, server.shutdown)
   } else {
-    baseService = new ProcessorServiceImpl(loader, server.shutdown)
+    baseService = new ProcessorServiceImpl(loader, options, server.shutdown)
   }
 
   const service = new FullProcessorServiceImpl(baseService)

package/src/service-manager.ts

@@ -1,30 +1,21 @@
 import { CallContext } from 'nice-grpc'
 import { Piscina } from 'piscina'
 import {
-  DataBinding,
-  DBRequest,
-  DBResponse,
   DeepPartial,
   Empty,
-  HandlerType,
   ProcessConfigRequest,
   ProcessConfigResponse,
   ProcessResult,
   ProcessStreamRequest,
   ProcessStreamResponse,
+  ProcessStreamResponse_Partitions,
   StartRequest
 } from '@sentio/protos'
-
-import { IStoreContext } from './db-context.js'
 import { Subject } from 'rxjs'
 
-import { processMetrics } from './metrics.js'
 import { MessageChannel } from 'node:worker_threads'
 import { ProcessorServiceImpl } from './service.js'
 import { TemplateInstanceState } from './state.js'
-
-const { process_binding_count, process_binding_time, process_binding_error } = processMetrics
-
 ;(BigInt.prototype as any).toJSON = function () {
   return this.toString()
 }
@@ -34,11 +25,11 @@ export class ServiceManager extends ProcessorServiceImpl {
   private workerData: any = {}
 
   constructor(
-    readonly options: any,
     loader: () => Promise<any>,
+    readonly options: any,
     shutdownHandler?: () => void
   ) {
-    super(loader, shutdownHandler)
+    super(loader, options, shutdownHandler)
     this.workerData.options = options
   }
 
@@ -70,80 +61,76 @@ export class ServiceManager extends ProcessorServiceImpl {
     return await super.stop(request, context)
   }
 
-  async process(request: DataBinding, dbContext?: ChannelStoreContext): Promise<ProcessResult> {
-    if (!this.pool) {
-      await this.initPool()
-    }
-
-    return this.pool.run(
-      { request, workerPort: dbContext?.workerPort },
-      { transferList: dbContext?.workerPort ? [dbContext?.workerPort] : [] }
-    )
-  }
-
   private readonly contexts = new Contexts()
 
   protected async handleRequests(
     requests: AsyncIterable<ProcessStreamRequest>,
     subject: Subject<DeepPartial<ProcessStreamResponse>>
   ) {
+    if (!this.pool) {
+      await this.initPool()
+    }
     for await (const request of requests) {
-      try {
-        // console.debug('received request:', request)
-        if (request.binding) {
-          process_binding_count.add(1)
-
-          // Adjust binding will make some request become invalid by setting UNKNOWN HandlerType
-          // for older SDK version, so we just return empty result for them here
-          if (request.binding.handlerType === HandlerType.UNKNOWN) {
-            subject.next({
-              processId: request.processId,
-              result: ProcessResult.create()
-            })
-            continue
-          }
-
-          const binding = request.binding
-
-          const dbContext = this.contexts.new(request.processId, subject)
-
-          const start = Date.now()
-          this.process(binding, dbContext)
-            .then(async (result) => {
-              subject.next({
-                result,
-                processId: request.processId
-              })
-            })
-            .catch((e) => {
-              dbContext.error(request.processId, e)
-              process_binding_error.add(1)
-            })
-            .finally(() => {
-              const cost = Date.now() - start
-              process_binding_time.add(cost)
-              this.contexts.delete(request.processId)
-            })
-        }
-        if (request.dbResult) {
-          const dbContext = this.contexts.get(request.processId)
-          try {
-            dbContext?.result(request.dbResult)
-          } catch (e) {
-            subject.error(new Error('db result error, process should stop'))
-          }
+      this.handleSingleRequest(request, subject)
+    }
+  }
+
+  async handleSingleRequest(request: ProcessStreamRequest, subject: Subject<DeepPartial<ProcessStreamResponse>>) {
+    const processId = request.processId
+    if (request.binding) {
+      const context = this.contexts.new(processId)
+      context.mainPort.on('message', (resp: ProcessStreamResponse) => {
+        subject.next(resp)
+        if (resp.result) {
+          // last response
+          this.contexts.delete(processId)
         }
-      } catch (e) {
-        // should not happen
-        console.error('unexpect error during handle loop', e)
+      })
+      try {
+        await this.pool.run(
+          { request, workerPort: context.workerPort, processId },
+          { transferList: [context.workerPort] }
+        )
+      } catch (err) {
+        console.error('Error processing request:', err)
+        subject.error(err)
       }
+    } else {
+      const context = this.contexts.get(processId)
+      if (!context) {
+        console.error('No context found for processId:', processId)
+        throw new Error(`No context found for processId: ${processId}`)
+      }
+      context.sendRequest(request)
+    }
+  }
+
+  async process(processId: number, context: ChannelContext): Promise<ProcessResult | ProcessStreamResponse_Partitions> {
+    if (!this.pool) {
+      await this.initPool()
     }
+
+    return this.pool.run(
+      { workerPort: context?.workerPort, processId },
+      { transferList: context?.workerPort ? [context?.workerPort] : [] }
+    )
   }
 
   private async initPool() {
     if (this.pool) {
       await this.pool.close()
     }
+
+    if (this.enablePartition) {
+      const concurrent = parseInt(process.env['PROCESS_CONCURRENCY'] || '0')
+      if (this.options.worker < concurrent) {
+        console.warn(
+          `When partition is enabled, the worker count must >= 'PROCESS_CONCURRENCY', will set worker count to ${concurrent})`
+        )
+        this.options.worker = concurrent
+      }
+    }
+
     console.info('Initializing worker pool with worker count:', this.options.worker)
     this.pool = new Piscina({
       maxThreads: this.options.worker,
@@ -161,17 +148,19 @@ export class ServiceManager extends ProcessorServiceImpl {
   }
 }
 
-export type WorkerMessage = DBRequest & { processId: number }
-
 class Contexts {
-  private contexts: Map<number, ChannelStoreContext> = new Map()
+  private contexts: Map<number, ChannelContext> = new Map()
 
   get(processId: number) {
     return this.contexts.get(processId)
   }
 
-  new(processId: number, subject: Subject<DeepPartial<ProcessStreamResponse>>) {
-    const context = new ChannelStoreContext(subject, processId)
+  new(processId: number) {
+    let context = this.get(processId)
+    if (context) {
+      return context
+    }
+    context = new ChannelContext(processId)
     this.contexts.set(processId, context)
     return context
   }
@@ -181,25 +170,19 @@ class Contexts {
     context?.close()
     this.contexts.delete(processId)
   }
+
+  has(processId: number) {
+    return this.contexts.has(processId)
+  }
 }
 
-export class ChannelStoreContext implements IStoreContext {
+export class ChannelContext {
   channel = new MessageChannel()
 
-  constructor(
-    readonly subject: Subject<DeepPartial<ProcessStreamResponse>>,
-    readonly processId: number
-  ) {
-    this.mainPort.on('message', (req: ProcessStreamRequest) => {
-      subject.next({
-        ...req,
-        processId: processId
-      })
-    })
-  }
+  constructor(readonly processId: number) {}
 
-  sendRequest(request: DeepPartial<Omit<DBRequest, 'opId'>>, timeoutSecs?: number): Promise<DBResponse> {
-    throw new Error('should not be used on main thread')
+  sendRequest(request: ProcessStreamRequest) {
+    this.mainPort.postMessage(request)
   }
 
   get workerPort() {
@@ -210,24 +193,7 @@ export class ChannelStoreContext implements IStoreContext {
     return this.channel.port1
   }
 
-  result(dbResult: DBResponse) {
-    this.mainPort.postMessage(dbResult)
-  }
-
   close(): void {
     this.mainPort.close()
   }
-
-  error(processId: number, e: any): void {
-    console.error('process error', processId, e)
-    const errorResult = ProcessResult.create({
-      states: {
-        error: e?.toString()
-      }
-    })
-    this.subject.next({
-      result: errorResult,
-      processId
-    })
-  }
 }

package/src/service-worker.ts

@@ -1,24 +1,14 @@
-import {
-  DataBinding,
-  DBResponse,
-  DeepPartial,
-  Empty,
-  ProcessConfigRequest,
-  ProcessConfigResponse,
-  ProcessStreamResponse,
-  StartRequest
-} from '@sentio/protos'
+import { DeepPartial, Empty, ProcessStreamRequest, ProcessStreamResponse, StartRequest } from '@sentio/protos'
 import { CallContext, ServerError, Status } from 'nice-grpc'
-import { PluginManager } from './plugin.js'
 import { errorString } from './utils.js'
 import { freezeGlobalConfig } from './global-config.js'
 import { DebugInfo, RichServerError } from 'nice-grpc-error-details'
-import { recordRuntimeInfo } from './service.js'
-import { BroadcastChannel, MessagePort, threadId } from 'worker_threads'
+import { ProcessorServiceImpl } from './service.js'
+import { MessagePort, threadId } from 'worker_threads'
 import { Piscina } from 'piscina'
 import { configureEndpoints } from './endpoints.js'
 import { setupLogger } from './logger.js'
-import { AbstractStoreContext } from './db-context.js'
+import { Subject } from 'rxjs'
 
 let started = false
 
@@ -38,16 +28,11 @@ process
     unhandled = reason as Error
     // shutdownServers(1)
   })
+  .on('exit', () => {
+    console.info('Worker thread exiting, threadId:', threadId)
+  })
 
-async function getConfig(request: ProcessConfigRequest, context?: CallContext): Promise<ProcessConfigResponse> {
-  if (!started) {
-    throw new ServerError(Status.UNAVAILABLE, 'Service Not started.')
-  }
-
-  const newConfig = ProcessConfigResponse.fromPartial({})
-  await PluginManager.INSTANCE.configure(newConfig)
-  return newConfig
-}
+let service: ProcessorServiceImpl | undefined
 
 const loader = async (options: any) => {
   if (options.target) {
@@ -57,10 +42,7 @@ const loader = async (options: any) => {
   }
 }
 
-const configureChannel = new BroadcastChannel('configure_channel')
-configureChannel.onmessage = (request: ProcessConfigRequest) => {
-  getConfig(request)
-}
+const emptyCallContext = <CallContext>{}
 
 async function start(request: StartRequest, options: any): Promise<Empty> {
   if (started) {
@@ -69,24 +51,24 @@ async function start(request: StartRequest, options: any): Promise<Empty> {
   freezeGlobalConfig()
 
   try {
-    await loader(options)
+    service = new ProcessorServiceImpl(() => loader(options), options)
   } catch (e) {
     throw new ServerError(Status.INVALID_ARGUMENT, 'Failed to load processor: ' + errorString(e))
   }
 
-  await PluginManager.INSTANCE.start(request)
+  await service.start(request, emptyCallContext)
   started = true
   return {}
 }
 
 export default async function ({
-  request,
   processId,
+  request: firstRequest,
   workerPort
 }: {
-  request: DataBinding
   processId: number
-  workerPort?: MessagePort
+  request: ProcessStreamRequest
+  workerPort: MessagePort
 }) {
   const { startRequest, configRequest, options } = Piscina.workerData
   if (!started) {
@@ -97,18 +79,19 @@ export default async function ({
 
     if (startRequest) {
       await start(startRequest, options)
-      console.debug('worker started, template instance:', startRequest.templateInstances?.length)
+      console.debug('worker', threadId, ' started, template instance:', startRequest.templateInstances?.length)
     }
 
     if (configRequest) {
-      await getConfig(configRequest)
-      console.debug('worker configured')
+      await service?.getConfig(configRequest, emptyCallContext)
+      console.debug('worker', threadId, ' configured')
     }
   }
 
   if (unhandled) {
     const err = unhandled
     unhandled = undefined
+    console.error('Unhandled exception/rejection in previous request:', err)
     throw new RichServerError(
       Status.UNAVAILABLE,
       'Unhandled exception/rejection in previous request: ' + errorString(err),
@@ -120,31 +103,37 @@ export default async function ({
       ]
     )
   }
-
-  const result = await PluginManager.INSTANCE.processBinding(
-    request,
-    undefined,
-    workerPort ? new WorkerStoreContext(workerPort, processId) : undefined
-  )
-  recordRuntimeInfo(result, request.handlerType)
-  return result
-}
-
-class WorkerStoreContext extends AbstractStoreContext {
-  constructor(
-    readonly port: MessagePort,
-    processId: number
-  ) {
-    super(processId)
-    this.port.on('message', (resp: DBResponse) => {
-      this.result(resp)
+  const timeout = options['worker-timeout'] || 0
+  const enablePartition = options['enable-partition'] || false
+  await new Promise<void>((resolve, reject) => {
+    const subject = new Subject<DeepPartial<ProcessStreamResponse>>()
+    let timeoutId: NodeJS.Timeout | undefined = undefined
+    subject.subscribe((resp: ProcessStreamResponse) => {
+      console.debug('Worker', threadId, 'send response:', resp.result ? 'result' : 'dbResult')
+      workerPort.postMessage(resp)
+      // receive the response from the processor , close and resolve the promise
+      if (resp.result) {
+        if (timeoutId) clearTimeout(timeoutId)
+        resolve()
+        workerPort.close()
+      }
     })
-    this.port.on('close', () => {
-      this.close()
+    workerPort.on('message', (msg: ProcessStreamRequest) => {
+      const request = msg as ProcessStreamRequest
+      console.debug('Worker', threadId, 'received request:', request.start ? 'start' : 'dbResult')
+      service?.handleRequest(request, firstRequest.binding, subject)
+      if (enablePartition && request.start && timeout > 0) {
+        timeoutId = setTimeout(async () => {
+          reject(new RichServerError(Status.DEADLINE_EXCEEDED, 'Worker timeout exceeded'))
+        }, timeout)
+      }
     })
-  }
-
-  doSend(req: DeepPartial<ProcessStreamResponse>): void {
-    this.port.postMessage(req)
-  }
+    console.debug('Worker', threadId, 'handle request: binding')
+    service?.handleRequest(firstRequest, firstRequest.binding, subject)
+    if (!enablePartition && timeout > 0) {
+      timeoutId = setTimeout(() => {
+        reject(new RichServerError(Status.DEADLINE_EXCEEDED, 'Worker timeout exceeded'))
+      }, timeout)
+    }
+  })
 }