@sentio/runtime 2.59.0-rc.3 → 2.59.0-rc.30

package/src/service.ts

@@ -56,6 +56,7 @@ export class ProcessorServiceImpl implements ProcessorServiceImplementation {
   private readonly enablePreprocess: boolean
 
   private preparedData: PreparedData | undefined
+  readonly enablePartition: boolean
 
   constructor(loader: () => Promise<any>, shutdownHandler?: () => void) {
     this.loader = loader
@@ -64,6 +65,8 @@ export class ProcessorServiceImpl implements ProcessorServiceImplementation {
     this.enablePreprocess = process.env['ENABLE_PREPROCESS']
       ? process.env['ENABLE_PREPROCESS'].toLowerCase() == 'true'
       : false
+
+    this.enablePartition = process.env['SENTIO_ENABLE_BINDING_DATA_PARTITION'] == 'true'
   }
 
   async getConfig(request: ProcessConfigRequest, context: CallContext): Promise<ProcessConfigResponse> {
@@ -418,9 +421,10 @@ export class ProcessorServiceImpl implements ProcessorServiceImplementation {
     subject: Subject<DeepPartial<ProcessStreamResponse>>
   ) {
     const contexts = new Contexts()
+    let lastBinding: DataBinding | undefined = undefined
     for await (const request of requests) {
       try {
-        // console.debug('received request:', request)
+        // console.log('received request:', request, 'lastBinding:', lastBinding)
         if (request.binding) {
           process_binding_count.add(1)
 
@@ -433,31 +437,29 @@ export class ProcessorServiceImpl implements ProcessorServiceImplementation {
             })
             continue
           }
+          lastBinding = request.binding
 
-          const binding = request.binding
-          const dbContext = contexts.new(request.processId, subject)
-          const start = Date.now()
-          PluginManager.INSTANCE.processBinding(binding, this.preparedData, dbContext)
-            .then(async (result) => {
-              // await all pending db requests
-              await dbContext.awaitPendings()
+          if (this.enablePartition) {
+            PluginManager.INSTANCE.partition(request.binding).then((partitions) => {
               subject.next({
-                result,
-                processId: request.processId
+                processId: request.processId,
+                partitions
               })
-              recordRuntimeInfo(result, binding.handlerType)
-            })
-            .catch((e) => {
-              console.debug(e)
-              dbContext.error(request.processId, e)
-              process_binding_error.add(1)
-            })
-            .finally(() => {
-              const cost = Date.now() - start
-              process_binding_time.add(cost)
-              contexts.delete(request.processId)
             })
+          } else {
+            this.startProcess(request.processId, request.binding, contexts, subject)
+          }
+        }
+
+        if (request.start) {
+          if (!lastBinding) {
+            console.error('start request received without binding')
+            subject.error(new Error('start request received without binding'))
+            continue
+          }
+          this.startProcess(request.processId, lastBinding, contexts, subject)
         }
+
         if (request.dbResult) {
           const dbContext = contexts.get(request.processId)
           try {
@@ -472,6 +474,36 @@ export class ProcessorServiceImpl implements ProcessorServiceImplementation {
       }
     }
   }
+
+  private startProcess(
+    processId: number,
+    binding: DataBinding,
+    contexts: Contexts,
+    subject: Subject<DeepPartial<ProcessStreamResponse>>
+  ) {
+    const dbContext = contexts.new(processId, subject)
+    const start = Date.now()
+    PluginManager.INSTANCE.processBinding(binding, this.preparedData, dbContext)
+      .then(async (result) => {
+        // await all pending db requests
+        await dbContext.awaitPendings()
+        subject.next({
+          result,
+          processId: processId
+        })
+        recordRuntimeInfo(result, binding.handlerType)
+      })
+      .catch((e) => {
+        console.error(e)
+        dbContext.error(processId, e)
+        process_binding_error.add(1)
+      })
+      .finally(() => {
+        const cost = Date.now() - start
+        process_binding_time.add(cost)
+        contexts.delete(processId)
+      })
+  }
 }
 
 export function recordRuntimeInfo(results: ProcessResult, handlerType: HandlerType) {

package/src/utils.ts

@@ -16,6 +16,7 @@ export function mergeProcessResults(results: ProcessResult[]): Required<ProcessR
     res.gauges = res.gauges.concat(r.gauges)
     res.events = res.events.concat(r.events)
     res.exports = res.exports.concat(r.exports)
+    res.timeseriesResult = res.timeseriesResult.concat(r.timeseriesResult)
     res.states = {
       configUpdated: res.states?.configUpdated || r.states?.configUpdated || false
     }