@sentio/runtime 2.59.0-rc.8 → 2.59.0

package/src/service.ts

@@ -56,14 +56,17 @@ export class ProcessorServiceImpl implements ProcessorServiceImplementation {
   private readonly enablePreprocess: boolean
 
   private preparedData: PreparedData | undefined
+  readonly enablePartition: boolean
 
-  constructor(loader: () => Promise<any>, shutdownHandler?: () => void) {
+  constructor(loader: () => Promise<any>, options?: any, shutdownHandler?: () => void) {
     this.loader = loader
     this.shutdownHandler = shutdownHandler
 
     this.enablePreprocess = process.env['ENABLE_PREPROCESS']
       ? process.env['ENABLE_PREPROCESS'].toLowerCase() == 'true'
       : false
+
+    this.enablePartition = options?.['enable-partition'] == true
   }
 
   async getConfig(request: ProcessConfigRequest, context: CallContext): Promise<ProcessConfigResponse> {
@@ -413,65 +416,105 @@ export class ProcessorServiceImpl implements ProcessorServiceImplementation {
     yield* from(subject).pipe(withAbort(context.signal))
   }
 
+  private dbContexts = new Contexts()
+
   protected async handleRequests(
     requests: AsyncIterable<ProcessStreamRequest>,
     subject: Subject<DeepPartial<ProcessStreamResponse>>
   ) {
-    const contexts = new Contexts()
+    let lastBinding: DataBinding | undefined = undefined
     for await (const request of requests) {
       try {
-        // console.debug('received request:', request)
+        // console.log('received request:', request, 'lastBinding:', lastBinding)
         if (request.binding) {
-          process_binding_count.add(1)
-
-          // Adjust binding will make some request become invalid by setting UNKNOWN HandlerType
-          // for older SDK version, so we just return empty result for them here
-          if (request.binding.handlerType === HandlerType.UNKNOWN) {
-            subject.next({
-              processId: request.processId,
-              result: ProcessResult.create()
-            })
-            continue
-          }
-
-          const binding = request.binding
-          const dbContext = contexts.new(request.processId, subject)
-          const start = Date.now()
-          PluginManager.INSTANCE.processBinding(binding, this.preparedData, dbContext)
-            .then(async (result) => {
-              // await all pending db requests
-              await dbContext.awaitPendings()
-              subject.next({
-                result,
-                processId: request.processId
-              })
-              recordRuntimeInfo(result, binding.handlerType)
-            })
-            .catch((e) => {
-              console.debug(e)
-              dbContext.error(request.processId, e)
-              process_binding_error.add(1)
-            })
-            .finally(() => {
-              const cost = Date.now() - start
-              process_binding_time.add(cost)
-              contexts.delete(request.processId)
-            })
-        }
-        if (request.dbResult) {
-          const dbContext = contexts.get(request.processId)
-          try {
-            dbContext?.result(request.dbResult)
-          } catch (e) {
-            subject.error(new Error('db result error, process should stop'))
-          }
+          lastBinding = request.binding
         }
+        this.handleRequest(request, lastBinding, subject)
       } catch (e) {
         // should not happen
         console.error('unexpect error during handle loop', e)
       }
     }
   }
+
+  async handleRequest(
+    request: ProcessStreamRequest,
+    lastBinding: DataBinding | undefined,
+    subject: Subject<DeepPartial<ProcessStreamResponse>>
+  ) {
+    if (request.binding) {
+      process_binding_count.add(1)
+
+      // Adjust binding will make some request become invalid by setting UNKNOWN HandlerType
+      // for older SDK version, so we just return empty result for them here
+      if (request.binding.handlerType === HandlerType.UNKNOWN) {
+        subject.next({
+          processId: request.processId,
+          result: ProcessResult.create()
+        })
+        return
+      }
+
+      if (this.enablePartition) {
+        try {
+          const partitions = await PluginManager.INSTANCE.partition(request.binding)
+          subject.next({
+            processId: request.processId,
+            partitions
+          })
+        } catch (e) {
+          console.error('Partition error:', e)
+          subject.error(new Error('Partition error: ' + errorString(e)))
+          return
+        }
+      } else {
+        this.startProcess(request.processId, request.binding, subject)
+      }
+    }
+
+    if (request.start) {
+      if (!lastBinding) {
+        console.error('start request received without binding')
+        subject.error(new Error('start request received without binding'))
+        return
+      }
+      this.startProcess(request.processId, lastBinding, subject)
+    }
+
+    if (request.dbResult) {
+      const dbContext = this.dbContexts.get(request.processId)
+      try {
+        dbContext?.result(request.dbResult)
+      } catch (e) {
+        subject.error(new Error('db result error, process should stop'))
+      }
+    }
+  }
+
+  private startProcess(processId: number, binding: DataBinding, subject: Subject<DeepPartial<ProcessStreamResponse>>) {
+    const dbContext = this.dbContexts.new(processId, subject)
+    const start = Date.now()
+    PluginManager.INSTANCE.processBinding(binding, this.preparedData, dbContext)
+      .then(async (result) => {
+        // await all pending db requests
+        await dbContext.awaitPendings()
+        subject.next({
+          result,
+          processId: processId
+        })
+        recordRuntimeInfo(result, binding.handlerType)
+      })
+      .catch((e) => {
+        console.error(e)
+        dbContext.error(processId, e)
+        process_binding_error.add(1)
+      })
+      .finally(() => {
+        const cost = Date.now() - start
+        process_binding_time.add(cost)
+        this.dbContexts.delete(processId)
+      })
+  }
 }
 
 export function recordRuntimeInfo(results: ProcessResult, handlerType: HandlerType) {

package/src/utils.ts

@@ -10,12 +10,22 @@ export function mergeProcessResults(results: ProcessResult[]): Required<ProcessR
     ...ProcessResultFull.create(),
     states: StateResult.create()
   }
+  return mergeProcessResultsInPlace(res, results)
+}
 
+export function mergeProcessResultsInPlace(
+  res: ProcessResult,
+  results: ProcessResult[]
+): Required<ProcessResult, 'states'> {
+  res.states = res.states || StateResult.create()
   for (const r of results) {
-    res.counters = res.counters.concat(r.counters)
-    res.gauges = res.gauges.concat(r.gauges)
-    res.events = res.events.concat(r.events)
-    res.exports = res.exports.concat(r.exports)
+    // not using spread operator since it puts all element on the stack
+    // cause maximum call stack size exceeded error if it's a large array
+    mergeArrayInPlace(res.counters, r.counters)
+    mergeArrayInPlace(res.gauges, r.gauges)
+    mergeArrayInPlace(res.events, r.events)
+    mergeArrayInPlace(res.exports, r.exports)
+    mergeArrayInPlace(res.timeseriesResult, r.timeseriesResult)
     res.states = {
       configUpdated: res.states?.configUpdated || r.states?.configUpdated || false
     }
@@ -23,6 +33,12 @@ export function mergeProcessResults(results: ProcessResult[]): Required<ProcessR
   return res
 }
 
+function mergeArrayInPlace<T>(dst: T[], src: T[]) {
+  for (const r of src) {
+    dst.push(r)
+  }
+}
+
 export function errorString(e: Error): string {
   return e.message + '\n' + e.stack
 }