@sentio/runtime 2.59.0-rc.21 → 2.59.0-rc.23

package/lib/service-worker.js

@@ -1,7 +1,7 @@
 import { createRequire as createRequireShim } from 'module'; const require = createRequireShim(import.meta.url);
 import {
   setupLogger
-} from "./chunk-GWKJGG55.js";
+} from "./chunk-ZYJXCGEJ.js";
 import {
   AbstractStoreContext,
   configureEndpoints,
@@ -10,12 +10,12 @@ import {
   recordRuntimeInfo,
   require_lib3 as require_lib,
   require_lib4 as require_lib2
-} from "./chunk-EPAMG3V5.js";
+} from "./chunk-3G5TU64H.js";
 import {
   PluginManager,
   ProcessConfigResponse,
   __toESM
-} from "./chunk-45FF2A6M.js";
+} from "./chunk-HMNYJB5K.js";
 
 // src/service-worker.ts
 var import_nice_grpc = __toESM(require_lib(), 1);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sentio/runtime",
-  "version": "2.59.0-rc.21",
+  "version": "2.59.0-rc.23",
   "license": "Apache-2.0",
   "type": "module",
   "exports": {

package/src/gen/processor/protos/processor.ts

@@ -912,12 +912,71 @@ export interface ProcessStreamRequest {
   processId: number;
   binding?: DataBinding | undefined;
   dbResult?: DBResponse | undefined;
+  start?: boolean | undefined;
 }
 
 export interface ProcessStreamResponse {
   processId: number;
   dbRequest?: DBRequest | undefined;
   result?: ProcessResult | undefined;
+  partitions?: ProcessStreamResponse_Partitions | undefined;
+}
+
+export interface ProcessStreamResponse_Partitions {
+  partitions: { [key: number]: ProcessStreamResponse_Partitions_Partition };
+}
+
+export interface ProcessStreamResponse_Partitions_Partition {
+  userValue?: string | undefined;
+  sysValue?: ProcessStreamResponse_Partitions_Partition_SysValue | undefined;
+}
+
+export enum ProcessStreamResponse_Partitions_Partition_SysValue {
+  BLOCK_NUMBER = 0,
+  SEQ_MODE = 1,
+  UNIQUE_VALUE = 2,
+  UNRECOGNIZED = -1,
+}
+
+export function processStreamResponse_Partitions_Partition_SysValueFromJSON(
+  object: any,
+): ProcessStreamResponse_Partitions_Partition_SysValue {
+  switch (object) {
+    case 0:
+    case "BLOCK_NUMBER":
+      return ProcessStreamResponse_Partitions_Partition_SysValue.BLOCK_NUMBER;
+    case 1:
+    case "SEQ_MODE":
+      return ProcessStreamResponse_Partitions_Partition_SysValue.SEQ_MODE;
+    case 2:
+    case "UNIQUE_VALUE":
+      return ProcessStreamResponse_Partitions_Partition_SysValue.UNIQUE_VALUE;
+    case -1:
+    case "UNRECOGNIZED":
+    default:
+      return ProcessStreamResponse_Partitions_Partition_SysValue.UNRECOGNIZED;
+  }
+}
+
+export function processStreamResponse_Partitions_Partition_SysValueToJSON(
+  object: ProcessStreamResponse_Partitions_Partition_SysValue,
+): string {
+  switch (object) {
+    case ProcessStreamResponse_Partitions_Partition_SysValue.BLOCK_NUMBER:
+      return "BLOCK_NUMBER";
+    case ProcessStreamResponse_Partitions_Partition_SysValue.SEQ_MODE:
+      return "SEQ_MODE";
+    case ProcessStreamResponse_Partitions_Partition_SysValue.UNIQUE_VALUE:
+      return "UNIQUE_VALUE";
+    case ProcessStreamResponse_Partitions_Partition_SysValue.UNRECOGNIZED:
+    default:
+      return "UNRECOGNIZED";
+  }
+}
+
+export interface ProcessStreamResponse_Partitions_PartitionsEntry {
+  key: number;
+  value: ProcessStreamResponse_Partitions_Partition | undefined;
 }
 
 export interface PreprocessStreamRequest {
@@ -7999,7 +8058,7 @@ export const ProcessBindingResponse = {
 };
 
 function createBaseProcessStreamRequest(): ProcessStreamRequest {
-  return { processId: 0, binding: undefined, dbResult: undefined };
+  return { processId: 0, binding: undefined, dbResult: undefined, start: undefined };
 }
 
 export const ProcessStreamRequest = {
@@ -8013,6 +8072,9 @@ export const ProcessStreamRequest = {
     if (message.dbResult !== undefined) {
       DBResponse.encode(message.dbResult, writer.uint32(26).fork()).ldelim();
     }
+    if (message.start !== undefined) {
+      writer.uint32(32).bool(message.start);
+    }
     return writer;
   },
 
@@ -8044,6 +8106,13 @@ export const ProcessStreamRequest = {
 
           message.dbResult = DBResponse.decode(reader, reader.uint32());
           continue;
+        case 4:
+          if (tag !== 32) {
+            break;
+          }
+
+          message.start = reader.bool();
+          continue;
       }
       if ((tag & 7) === 4 || tag === 0) {
         break;
@@ -8058,6 +8127,7 @@ export const ProcessStreamRequest = {
       processId: isSet(object.processId) ? globalThis.Number(object.processId) : 0,
       binding: isSet(object.binding) ? DataBinding.fromJSON(object.binding) : undefined,
       dbResult: isSet(object.dbResult) ? DBResponse.fromJSON(object.dbResult) : undefined,
+      start: isSet(object.start) ? globalThis.Boolean(object.start) : undefined,
     };
   },
 
@@ -8072,6 +8142,9 @@ export const ProcessStreamRequest = {
     if (message.dbResult !== undefined) {
       obj.dbResult = DBResponse.toJSON(message.dbResult);
     }
+    if (message.start !== undefined) {
+      obj.start = message.start;
+    }
     return obj;
   },
 
@@ -8087,12 +8160,13 @@ export const ProcessStreamRequest = {
     message.dbResult = (object.dbResult !== undefined && object.dbResult !== null)
       ? DBResponse.fromPartial(object.dbResult)
       : undefined;
+    message.start = object.start ?? undefined;
     return message;
   },
 };
 
 function createBaseProcessStreamResponse(): ProcessStreamResponse {
-  return { processId: 0, dbRequest: undefined, result: undefined };
+  return { processId: 0, dbRequest: undefined, result: undefined, partitions: undefined };
 }
 
 export const ProcessStreamResponse = {
@@ -8106,6 +8180,9 @@ export const ProcessStreamResponse = {
     if (message.result !== undefined) {
       ProcessResult.encode(message.result, writer.uint32(26).fork()).ldelim();
     }
+    if (message.partitions !== undefined) {
+      ProcessStreamResponse_Partitions.encode(message.partitions, writer.uint32(34).fork()).ldelim();
+    }
     return writer;
   },
 
@@ -8137,6 +8214,13 @@ export const ProcessStreamResponse = {
 
           message.result = ProcessResult.decode(reader, reader.uint32());
           continue;
+        case 4:
+          if (tag !== 34) {
+            break;
+          }
+
+          message.partitions = ProcessStreamResponse_Partitions.decode(reader, reader.uint32());
+          continue;
       }
       if ((tag & 7) === 4 || tag === 0) {
         break;
@@ -8151,6 +8235,7 @@ export const ProcessStreamResponse = {
       processId: isSet(object.processId) ? globalThis.Number(object.processId) : 0,
       dbRequest: isSet(object.dbRequest) ? DBRequest.fromJSON(object.dbRequest) : undefined,
       result: isSet(object.result) ? ProcessResult.fromJSON(object.result) : undefined,
+      partitions: isSet(object.partitions) ? ProcessStreamResponse_Partitions.fromJSON(object.partitions) : undefined,
     };
   },
 
@@ -8165,6 +8250,9 @@ export const ProcessStreamResponse = {
     if (message.result !== undefined) {
       obj.result = ProcessResult.toJSON(message.result);
     }
+    if (message.partitions !== undefined) {
+      obj.partitions = ProcessStreamResponse_Partitions.toJSON(message.partitions);
+    }
     return obj;
   },
 
@@ -8180,6 +8268,254 @@ export const ProcessStreamResponse = {
     message.result = (object.result !== undefined && object.result !== null)
       ? ProcessResult.fromPartial(object.result)
       : undefined;
+    message.partitions = (object.partitions !== undefined && object.partitions !== null)
+      ? ProcessStreamResponse_Partitions.fromPartial(object.partitions)
+      : undefined;
+    return message;
+  },
+};
+
+function createBaseProcessStreamResponse_Partitions(): ProcessStreamResponse_Partitions {
+  return { partitions: {} };
+}
+
+export const ProcessStreamResponse_Partitions = {
+  encode(message: ProcessStreamResponse_Partitions, writer: _m0.Writer = _m0.Writer.create()): _m0.Writer {
+    Object.entries(message.partitions).forEach(([key, value]) => {
+      ProcessStreamResponse_Partitions_PartitionsEntry.encode({ key: key as any, value }, writer.uint32(10).fork())
+        .ldelim();
+    });
+    return writer;
+  },
+
+  decode(input: _m0.Reader | Uint8Array, length?: number): ProcessStreamResponse_Partitions {
+    const reader = input instanceof _m0.Reader ? input : _m0.Reader.create(input);
+    let end = length === undefined ? reader.len : reader.pos + length;
+    const message = createBaseProcessStreamResponse_Partitions();
+    while (reader.pos < end) {
+      const tag = reader.uint32();
+      switch (tag >>> 3) {
+        case 1:
+          if (tag !== 10) {
+            break;
+          }
+
+          const entry1 = ProcessStreamResponse_Partitions_PartitionsEntry.decode(reader, reader.uint32());
+          if (entry1.value !== undefined) {
+            message.partitions[entry1.key] = entry1.value;
+          }
+          continue;
+      }
+      if ((tag & 7) === 4 || tag === 0) {
+        break;
+      }
+      reader.skipType(tag & 7);
+    }
+    return message;
+  },
+
+  fromJSON(object: any): ProcessStreamResponse_Partitions {
+    return {
+      partitions: isObject(object.partitions)
+        ? Object.entries(object.partitions).reduce<{ [key: number]: ProcessStreamResponse_Partitions_Partition }>(
+          (acc, [key, value]) => {
+            acc[globalThis.Number(key)] = ProcessStreamResponse_Partitions_Partition.fromJSON(value);
+            return acc;
+          },
+          {},
+        )
+        : {},
+    };
+  },
+
+  toJSON(message: ProcessStreamResponse_Partitions): unknown {
+    const obj: any = {};
+    if (message.partitions) {
+      const entries = Object.entries(message.partitions);
+      if (entries.length > 0) {
+        obj.partitions = {};
+        entries.forEach(([k, v]) => {
+          obj.partitions[k] = ProcessStreamResponse_Partitions_Partition.toJSON(v);
+        });
+      }
+    }
+    return obj;
+  },
+
+  create(base?: DeepPartial<ProcessStreamResponse_Partitions>): ProcessStreamResponse_Partitions {
+    return ProcessStreamResponse_Partitions.fromPartial(base ?? {});
+  },
+  fromPartial(object: DeepPartial<ProcessStreamResponse_Partitions>): ProcessStreamResponse_Partitions {
+    const message = createBaseProcessStreamResponse_Partitions();
+    message.partitions = Object.entries(object.partitions ?? {}).reduce<
+      { [key: number]: ProcessStreamResponse_Partitions_Partition }
+    >((acc, [key, value]) => {
+      if (value !== undefined) {
+        acc[globalThis.Number(key)] = ProcessStreamResponse_Partitions_Partition.fromPartial(value);
+      }
+      return acc;
+    }, {});
+    return message;
+  },
+};
+
+function createBaseProcessStreamResponse_Partitions_Partition(): ProcessStreamResponse_Partitions_Partition {
+  return { userValue: undefined, sysValue: undefined };
+}
+
+export const ProcessStreamResponse_Partitions_Partition = {
+  encode(message: ProcessStreamResponse_Partitions_Partition, writer: _m0.Writer = _m0.Writer.create()): _m0.Writer {
+    if (message.userValue !== undefined) {
+      writer.uint32(10).string(message.userValue);
+    }
+    if (message.sysValue !== undefined) {
+      writer.uint32(16).int32(message.sysValue);
+    }
+    return writer;
+  },
+
+  decode(input: _m0.Reader | Uint8Array, length?: number): ProcessStreamResponse_Partitions_Partition {
+    const reader = input instanceof _m0.Reader ? input : _m0.Reader.create(input);
+    let end = length === undefined ? reader.len : reader.pos + length;
+    const message = createBaseProcessStreamResponse_Partitions_Partition();
+    while (reader.pos < end) {
+      const tag = reader.uint32();
+      switch (tag >>> 3) {
+        case 1:
+          if (tag !== 10) {
+            break;
+          }
+
+          message.userValue = reader.string();
+          continue;
+        case 2:
+          if (tag !== 16) {
+            break;
+          }
+
+          message.sysValue = reader.int32() as any;
+          continue;
+      }
+      if ((tag & 7) === 4 || tag === 0) {
+        break;
+      }
+      reader.skipType(tag & 7);
+    }
+    return message;
+  },
+
+  fromJSON(object: any): ProcessStreamResponse_Partitions_Partition {
+    return {
+      userValue: isSet(object.userValue) ? globalThis.String(object.userValue) : undefined,
+      sysValue: isSet(object.sysValue)
+        ? processStreamResponse_Partitions_Partition_SysValueFromJSON(object.sysValue)
+        : undefined,
+    };
+  },
+
+  toJSON(message: ProcessStreamResponse_Partitions_Partition): unknown {
+    const obj: any = {};
+    if (message.userValue !== undefined) {
+      obj.userValue = message.userValue;
+    }
+    if (message.sysValue !== undefined) {
+      obj.sysValue = processStreamResponse_Partitions_Partition_SysValueToJSON(message.sysValue);
+    }
+    return obj;
+  },
+
+  create(base?: DeepPartial<ProcessStreamResponse_Partitions_Partition>): ProcessStreamResponse_Partitions_Partition {
+    return ProcessStreamResponse_Partitions_Partition.fromPartial(base ?? {});
+  },
+  fromPartial(
+    object: DeepPartial<ProcessStreamResponse_Partitions_Partition>,
+  ): ProcessStreamResponse_Partitions_Partition {
+    const message = createBaseProcessStreamResponse_Partitions_Partition();
+    message.userValue = object.userValue ?? undefined;
+    message.sysValue = object.sysValue ?? undefined;
+    return message;
+  },
+};
+
+function createBaseProcessStreamResponse_Partitions_PartitionsEntry(): ProcessStreamResponse_Partitions_PartitionsEntry {
+  return { key: 0, value: undefined };
+}
+
+export const ProcessStreamResponse_Partitions_PartitionsEntry = {
+  encode(
+    message: ProcessStreamResponse_Partitions_PartitionsEntry,
+    writer: _m0.Writer = _m0.Writer.create(),
+  ): _m0.Writer {
+    if (message.key !== 0) {
+      writer.uint32(8).int32(message.key);
+    }
+    if (message.value !== undefined) {
+      ProcessStreamResponse_Partitions_Partition.encode(message.value, writer.uint32(18).fork()).ldelim();
+    }
+    return writer;
+  },
+
+  decode(input: _m0.Reader | Uint8Array, length?: number): ProcessStreamResponse_Partitions_PartitionsEntry {
+    const reader = input instanceof _m0.Reader ? input : _m0.Reader.create(input);
+    let end = length === undefined ? reader.len : reader.pos + length;
+    const message = createBaseProcessStreamResponse_Partitions_PartitionsEntry();
+    while (reader.pos < end) {
+      const tag = reader.uint32();
+      switch (tag >>> 3) {
+        case 1:
+          if (tag !== 8) {
+            break;
+          }
+
+          message.key = reader.int32();
+          continue;
+        case 2:
+          if (tag !== 18) {
+            break;
+          }
+
+          message.value = ProcessStreamResponse_Partitions_Partition.decode(reader, reader.uint32());
+          continue;
+      }
+      if ((tag & 7) === 4 || tag === 0) {
+        break;
+      }
+      reader.skipType(tag & 7);
+    }
+    return message;
+  },
+
+  fromJSON(object: any): ProcessStreamResponse_Partitions_PartitionsEntry {
+    return {
+      key: isSet(object.key) ? globalThis.Number(object.key) : 0,
+      value: isSet(object.value) ? ProcessStreamResponse_Partitions_Partition.fromJSON(object.value) : undefined,
+    };
+  },
+
+  toJSON(message: ProcessStreamResponse_Partitions_PartitionsEntry): unknown {
+    const obj: any = {};
+    if (message.key !== 0) {
+      obj.key = Math.round(message.key);
+    }
+    if (message.value !== undefined) {
+      obj.value = ProcessStreamResponse_Partitions_Partition.toJSON(message.value);
+    }
+    return obj;
+  },
+
+  create(
+    base?: DeepPartial<ProcessStreamResponse_Partitions_PartitionsEntry>,
+  ): ProcessStreamResponse_Partitions_PartitionsEntry {
+    return ProcessStreamResponse_Partitions_PartitionsEntry.fromPartial(base ?? {});
+  },
+  fromPartial(
+    object: DeepPartial<ProcessStreamResponse_Partitions_PartitionsEntry>,
+  ): ProcessStreamResponse_Partitions_PartitionsEntry {
+    const message = createBaseProcessStreamResponse_Partitions_PartitionsEntry();
+    message.key = object.key ?? 0;
+    message.value = (object.value !== undefined && object.value !== null)
+      ? ProcessStreamResponse_Partitions_Partition.fromPartial(object.value)
+      : undefined;
     return message;
   },
 };

package/src/plugin.ts

@@ -5,6 +5,8 @@ import {
   PreprocessResult,
   ProcessConfigResponse,
   ProcessResult,
+  ProcessStreamResponse_Partitions,
+  ProcessStreamResponse_Partitions_Partition_SysValue,
   StartRequest
 } from '@sentio/protos'
 import { IStoreContext, StoreContext } from './db-context.js'
@@ -33,6 +35,20 @@ export abstract class Plugin {
     return PreprocessResult.create()
   }
 
+  async partition(request: DataBinding): Promise<ProcessStreamResponse_Partitions> {
+    return {
+      partitions: request.handlerIds.reduce(
+        (acc, id) => ({
+          ...acc,
+          [id]: {
+            sysValue: ProcessStreamResponse_Partitions_Partition_SysValue.UNRECOGNIZED
+          }
+        }),
+        {}
+      )
+    }
+  }
+
   /**
    * method used by action server only
    * @param port
@@ -104,6 +120,14 @@ export class PluginManager {
     })
   }
 
+  async partition(request: DataBinding): Promise<ProcessStreamResponse_Partitions> {
+    const plugin = this.typesToPlugin.get(request.handlerType)
+    if (!plugin) {
+      throw new Error(`No plugin for ${request.handlerType}`)
+    }
+    return plugin.partition(request)
+  }
+
   preprocessBinding(
     request: DataBinding,
     preprocessStore: { [k: string]: any },

package/src/service-manager.ts

@@ -1,4 +1,4 @@
-import { CallContext } from 'nice-grpc'
+import { CallContext, ServerError, Status } from 'nice-grpc'
 import { Piscina } from 'piscina'
 import {
   DataBinding,
@@ -22,6 +22,7 @@ import { processMetrics } from './metrics.js'
 import { MessageChannel } from 'node:worker_threads'
 import { ProcessorServiceImpl } from './service.js'
 import { TemplateInstanceState } from './state.js'
+import { PluginManager } from './plugin.js'
 
 const { process_binding_count, process_binding_time, process_binding_error } = processMetrics
 
@@ -70,17 +71,6 @@ export class ServiceManager extends ProcessorServiceImpl {
     return await super.stop(request, context)
   }
 
-  async process(request: DataBinding, dbContext?: ChannelStoreContext): Promise<ProcessResult> {
-    if (!this.pool) {
-      await this.initPool()
-    }
-
-    return this.pool.run(
-      { request, workerPort: dbContext?.workerPort },
-      { transferList: dbContext?.workerPort ? [dbContext?.workerPort] : [] }
-    )
-  }
-
   private readonly contexts = new Contexts()
 
   protected async handleRequests(
@@ -88,6 +78,8 @@ export class ServiceManager extends ProcessorServiceImpl {
     subject: Subject<DeepPartial<ProcessStreamResponse>>
   ) {
     for await (const request of requests) {
+      let lastBinding: DataBinding | undefined = undefined
+
       try {
         // console.debug('received request:', request)
         if (request.binding) {
@@ -103,27 +95,22 @@ export class ServiceManager extends ProcessorServiceImpl {
             continue
           }
 
-          const binding = request.binding
-
-          const dbContext = this.contexts.new(request.processId, subject)
-
-          const start = Date.now()
-          this.process(binding, dbContext)
-            .then(async (result) => {
-              subject.next({
-                result,
-                processId: request.processId
-              })
-            })
-            .catch((e) => {
-              dbContext.error(request.processId, e)
-              process_binding_error.add(1)
-            })
-            .finally(() => {
-              const cost = Date.now() - start
-              process_binding_time.add(cost)
-              this.contexts.delete(request.processId)
+          if (this.enablePartition) {
+            const partitions = await PluginManager.INSTANCE.partition(request.binding)
+            subject.next({
+              processId: request.processId,
+              partitions
             })
+            lastBinding = request.binding
+          } else {
+            this.doProcess(request.processId, request.binding, subject)
+          }
+        }
+        if (request.start) {
+          if (!lastBinding) {
+            throw new ServerError(Status.INVALID_ARGUMENT, 'start request received without binding')
+          }
+          this.doProcess(request.processId, lastBinding, subject)
         }
         if (request.dbResult) {
           const dbContext = this.contexts.get(request.processId)
@@ -140,6 +127,39 @@ export class ServiceManager extends ProcessorServiceImpl {
     }
   }
 
+  private doProcess(processId: number, binding: DataBinding, subject: Subject<DeepPartial<ProcessStreamResponse>>) {
+    const dbContext = this.contexts.new(processId, subject)
+
+    const start = Date.now()
+    this.process(binding, dbContext)
+      .then(async (result) => {
+        subject.next({
+          result,
+          processId: processId
+        })
+      })
+      .catch((e) => {
+        dbContext.error(processId, e)
+        process_binding_error.add(1)
+      })
+      .finally(() => {
+        const cost = Date.now() - start
+        process_binding_time.add(cost)
+        this.contexts.delete(processId)
+      })
+  }
+
+  async process(request: DataBinding, dbContext?: ChannelStoreContext): Promise<ProcessResult> {
+    if (!this.pool) {
+      await this.initPool()
+    }
+
+    return this.pool.run(
+      { request, workerPort: dbContext?.workerPort },
+      { transferList: dbContext?.workerPort ? [dbContext?.workerPort] : [] }
+    )
+  }
+
   private async initPool() {
     if (this.pool) {
       await this.pool.close()