@sentio/runtime 2.59.0-rc.31 → 2.59.0-rc.33

package/lib/service-worker.d.ts

@@ -1,11 +1,11 @@
-import * as _sentio_protos from '@sentio/protos';
 import { DataBinding } from '@sentio/protos';
 import { MessagePort } from 'worker_threads';
 
-declare function export_default({ request, processId, workerPort }: {
+declare function export_default({ request, processId, workerPort, partition }: {
     request: DataBinding;
     processId: number;
     workerPort?: MessagePort;
-}): Promise<_sentio_protos.ProcessResult>;
+    partition?: boolean;
+}): Promise<unknown>;
 
 export { export_default as default };

package/lib/service-worker.js

@@ -10,7 +10,7 @@ import {
   recordRuntimeInfo,
   require_lib3 as require_lib,
   require_lib4 as require_lib2
-} from "./chunk-X2VTMTYL.js";
+} from "./chunk-QWKQ7U4H.js";
 import {
   PluginManager,
   ProcessConfigResponse,
@@ -33,6 +33,8 @@ process.on("uncaughtException", (err) => {
   }
   console.error("Unhandled Rejection, please checking if await is properly", reason);
   unhandled = reason;
+}).on("exit", () => {
+  console.info("Worker thread exiting, threadId:", threadId);
 });
 async function getConfig(request, context) {
   if (!started) {
@@ -70,7 +72,8 @@ async function start(request, options) {
 async function service_worker_default({
   request,
   processId,
-  workerPort
+  workerPort,
+  partition
 }) {
   const { startRequest, configRequest, options } = Piscina.workerData;
   if (!started) {
@@ -79,16 +82,17 @@ async function service_worker_default({
     configureEndpoints(options);
     if (startRequest) {
       await start(startRequest, options);
-      console.debug("worker started, template instance:", startRequest.templateInstances?.length);
+      console.debug("worker", threadId, " started, template instance:", startRequest.templateInstances?.length);
     }
     if (configRequest) {
       await getConfig(configRequest);
-      console.debug("worker configured");
+      console.debug("worker", threadId, " configured");
     }
   }
   if (unhandled) {
     const err = unhandled;
     unhandled = void 0;
+    console.error("Unhandled exception/rejection in previous request:", err);
     throw new import_nice_grpc_error_details.RichServerError(
       import_nice_grpc.Status.UNAVAILABLE,
       "Unhandled exception/rejection in previous request: " + errorString(err),
@@ -100,13 +104,62 @@ async function service_worker_default({
       ]
     );
   }
-  const result = await PluginManager.INSTANCE.processBinding(
-    request,
-    void 0,
-    workerPort ? new WorkerStoreContext(workerPort, processId) : void 0
-  );
-  recordRuntimeInfo(result, request.handlerType);
-  return result;
+  console.debug("Worker", threadId, "starting request ", processId);
+  const now = Date.now();
+  const timeoutPromise = new Promise((resolve, reject) => {
+    setTimeout(
+      () => {
+        reject(
+          new import_nice_grpc_error_details.RichServerError(
+            import_nice_grpc.Status.DEADLINE_EXCEEDED,
+            `Request ${processId} timed out after ${options["request-timeout"]} ms`
+          )
+        );
+      },
+      (options["process-timeout"] ?? 60) * 1e3
+    );
+  });
+  let resultPromise;
+  if (partition) {
+    resultPromise = PluginManager.INSTANCE.partition(request);
+  } else {
+    resultPromise = PluginManager.INSTANCE.processBinding(
+      request,
+      void 0,
+      workerPort ? new WorkerStoreContext(workerPort, processId) : void 0
+    ).then((result) => {
+      recordRuntimeInfo(result, request.handlerType);
+      return result;
+    });
+  }
+  try {
+    const result = await Promise.race([resultPromise, timeoutPromise]);
+    console.debug(
+      "worker",
+      threadId,
+      `finished ${partition ? "partition" : "process"} request`,
+      processId,
+      "took",
+      Date.now() - now
+    );
+    return result;
+  } catch (e) {
+    console.error("worker", threadId, `failed ${partition ? "partition" : "process"} request`, processId, "error:", e);
+    if (e instanceof import_nice_grpc_error_details.RichServerError) {
+      throw e;
+    } else {
+      throw new import_nice_grpc_error_details.RichServerError(
+        import_nice_grpc.Status.INTERNAL,
+        `Worker ${threadId} failed to process request ${processId}: ` + errorString(e),
+        [
+          import_nice_grpc_error_details.DebugInfo.fromPartial({
+            detail: errorString(e),
+            stackEntries: e.stack?.split("\n")
+          })
+        ]
+      );
+    }
+  }
 }
 var WorkerStoreContext = class extends AbstractStoreContext {
   constructor(port, processId) {

package/lib/service-worker.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../src/service-worker.ts"],"sourcesContent":["import {\n  DataBinding,\n  DBResponse,\n  DeepPartial,\n  Empty,\n  ProcessConfigRequest,\n  ProcessConfigResponse,\n  ProcessStreamResponse,\n  StartRequest\n} from '@sentio/protos'\nimport { CallContext, ServerError, Status } from 'nice-grpc'\nimport { PluginManager } from './plugin.js'\nimport { errorString } from './utils.js'\nimport { freezeGlobalConfig } from './global-config.js'\nimport { DebugInfo, RichServerError } from 'nice-grpc-error-details'\nimport { recordRuntimeInfo } from './service.js'\nimport { BroadcastChannel, MessagePort, threadId } from 'worker_threads'\nimport { Piscina } from 'piscina'\nimport { configureEndpoints } from './endpoints.js'\nimport { setupLogger } from './logger.js'\nimport { AbstractStoreContext } from './db-context.js'\n\nlet started = false\n\nlet unhandled: Error | undefined\n\nprocess\n  .on('uncaughtException', (err) => {\n    console.error('Uncaught Exception, please checking if await is properly used', err)\n    unhandled = err\n  })\n  .on('unhandledRejection', (reason, p) => {\n    // @ts-ignore ignore invalid ens error\n    if (reason?.message.startsWith('invalid ENS name (disallowed character: \"*\"')) {\n      return\n    }\n    console.error('Unhandled Rejection, please checking if await is properly', reason)\n    unhandled = reason as Error\n    // shutdownServers(1)\n  })\n\nasync function getConfig(request: ProcessConfigRequest, context?: CallContext): Promise<ProcessConfigResponse> {\n  if (!started) {\n    throw new ServerError(Status.UNAVAILABLE, 'Service Not started.')\n  }\n\n  const newConfig = ProcessConfigResponse.fromPartial({})\n  await PluginManager.INSTANCE.configure(newConfig)\n  return newConfig\n}\n\nconst loader = async (options: any) => {\n  if (options.target) {\n    const m = await import(options.target)\n    console.debug('Module loaded, path:', options.target, 'module:', m)\n    return m\n  }\n}\n\nconst configureChannel = new BroadcastChannel('configure_channel')\nconfigureChannel.onmessage = (request: ProcessConfigRequest) => {\n  getConfig(request)\n}\n\nasync function start(request: StartRequest, options: any): Promise<Empty> {\n  if (started) {\n    return {}\n  }\n  freezeGlobalConfig()\n\n  try {\n    await loader(options)\n  } catch (e) {\n    throw new ServerError(Status.INVALID_ARGUMENT, 'Failed to load processor: ' + errorString(e))\n  }\n\n  await PluginManager.INSTANCE.start(request)\n  started = true\n  return {}\n}\n\nexport default async function ({\n  request,\n  processId,\n  workerPort\n}: {\n  request: DataBinding\n  processId: number\n  workerPort?: MessagePort\n}) {\n  const { startRequest, configRequest, options } = Piscina.workerData\n  if (!started) {\n    const logLevel = process.env['LOG_LEVEL']?.toUpperCase()\n    setupLogger(options['log-format'] === 'json', logLevel === 'debug' ? true : options.debug, threadId)\n\n    configureEndpoints(options)\n\n    if (startRequest) {\n      await start(startRequest, options)\n      console.debug('worker started, template instance:', startRequest.templateInstances?.length)\n    }\n\n    if (configRequest) {\n      await getConfig(configRequest)\n      console.debug('worker configured')\n    }\n  }\n\n  if (unhandled) {\n    const err = unhandled\n    unhandled = undefined\n    throw new RichServerError(\n      Status.UNAVAILABLE,\n      'Unhandled exception/rejection in previous request: ' + errorString(err),\n      [\n        DebugInfo.fromPartial({\n          detail: err.message,\n          stackEntries: err.stack?.split('\\n')\n        })\n      ]\n    )\n  }\n\n  const result = await PluginManager.INSTANCE.processBinding(\n    request,\n    undefined,\n    workerPort ? new WorkerStoreContext(workerPort, processId) : undefined\n  )\n  recordRuntimeInfo(result, request.handlerType)\n  return result\n}\n\nclass WorkerStoreContext extends AbstractStoreContext {\n  constructor(\n    readonly port: MessagePort,\n    processId: number\n  ) {\n    super(processId)\n    this.port.on('message', (resp: DBResponse) => {\n      this.result(resp)\n    })\n    this.port.on('close', () => {\n      this.close()\n    })\n  }\n\n  doSend(req: DeepPartial<ProcessStreamResponse>): void {\n    this.port.postMessage(req)\n  }\n}\n;import(\"node:process\").then((p) => p.stdout.write(\"\"));"],"mappings":";;;;;;;;;;;;;;;;;;;;AAUA,uBAAiD;AAIjD,qCAA2C;AAE3C,SAAS,kBAA+B,gBAAgB;AACxD,SAAS,eAAe;AAKxB,IAAI,UAAU;AAEd,IAAI;AAEJ,QACG,GAAG,qBAAqB,CAAC,QAAQ;AAChC,UAAQ,MAAM,iEAAiE,GAAG;AAClF,cAAY;AACd,CAAC,EACA,GAAG,sBAAsB,CAAC,QAAQ,MAAM;AAEvC,MAAI,QAAQ,QAAQ,WAAW,6CAA6C,GAAG;AAC7E;AAAA,EACF;AACA,UAAQ,MAAM,6DAA6D,MAAM;AACjF,cAAY;AAEd,CAAC;AAEH,eAAe,UAAU,SAA+B,SAAuD;AAC7G,MAAI,CAAC,SAAS;AACZ,UAAM,IAAI,6BAAY,wBAAO,aAAa,sBAAsB;AAAA,EAClE;AAEA,QAAM,YAAY,sBAAsB,YAAY,CAAC,CAAC;AACtD,QAAM,cAAc,SAAS,UAAU,SAAS;AAChD,SAAO;AACT;AAEA,IAAM,SAAS,OAAO,YAAiB;AACrC,MAAI,QAAQ,QAAQ;AAClB,UAAM,IAAI,MAAM,OAAO,QAAQ;AAC/B,YAAQ,MAAM,wBAAwB,QAAQ,QAAQ,WAAW,CAAC;AAClE,WAAO;AAAA,EACT;AACF;AAEA,IAAM,mBAAmB,IAAI,iBAAiB,mBAAmB;AACjE,iBAAiB,YAAY,CAAC,YAAkC;AAC9D,YAAU,OAAO;AACnB;AAEA,eAAe,MAAM,SAAuB,SAA8B;AACxE,MAAI,SAAS;AACX,WAAO,CAAC;AAAA,EACV;AACA,qBAAmB;AAEnB,MAAI;AACF,UAAM,OAAO,OAAO;AAAA,EACtB,SAAS,GAAG;AACV,UAAM,IAAI,6BAAY,wBAAO,kBAAkB,+BAA+B,YAAY,CAAC,CAAC;AAAA,EAC9F;AAEA,QAAM,cAAc,SAAS,MAAM,OAAO;AAC1C,YAAU;AACV,SAAO,CAAC;AACV;AAEA,eAAO,uBAAwB;AAAA,EAC7B;AAAA,EACA;AAAA,EACA;AACF,GAIG;AACD,QAAM,EAAE,cAAc,eAAe,QAAQ,IAAI,QAAQ;AACzD,MAAI,CAAC,SAAS;AACZ,UAAM,WAAW,QAAQ,IAAI,WAAW,GAAG,YAAY;AACvD,gBAAY,QAAQ,YAAY,MAAM,QAAQ,aAAa,UAAU,OAAO,QAAQ,OAAO,QAAQ;AAEnG,uBAAmB,OAAO;AAE1B,QAAI,cAAc;AAChB,YAAM,MAAM,cAAc,OAAO;AACjC,cAAQ,MAAM,sCAAsC,aAAa,mBAAmB,MAAM;AAAA,IAC5F;AAEA,QAAI,eAAe;AACjB,YAAM,UAAU,aAAa;AAC7B,cAAQ,MAAM,mBAAmB;AAAA,IACnC;AAAA,EACF;AAEA,MAAI,WAAW;AACb,UAAM,MAAM;AACZ,gBAAY;AACZ,UAAM,IAAI;AAAA,MACR,wBAAO;AAAA,MACP,wDAAwD,YAAY,GAAG;AAAA,MACvE;AAAA,QACE,yCAAU,YAAY;AAAA,UACpB,QAAQ,IAAI;AAAA,UACZ,cAAc,IAAI,OAAO,MAAM,IAAI;AAAA,QACrC,CAAC;AAAA,MACH;AAAA,IACF;AAAA,EACF;AAEA,QAAM,SAAS,MAAM,cAAc,SAAS;AAAA,IAC1C;AAAA,IACA;AAAA,IACA,aAAa,IAAI,mBAAmB,YAAY,SAAS,IAAI;AAAA,EAC/D;AACA,oBAAkB,QAAQ,QAAQ,WAAW;AAC7C,SAAO;AACT;AAEA,IAAM,qBAAN,cAAiC,qBAAqB;AAAA,EACpD,YACW,MACT,WACA;AACA,UAAM,SAAS;AAHN;AAIT,SAAK,KAAK,GAAG,WAAW,CAAC,SAAqB;AAC5C,WAAK,OAAO,IAAI;AAAA,IAClB,CAAC;AACD,SAAK,KAAK,GAAG,SAAS,MAAM;AAC1B,WAAK,MAAM;AAAA,IACb,CAAC;AAAA,EACH;AAAA,EAEA,OAAO,KAA+C;AACpD,SAAK,KAAK,YAAY,GAAG;AAAA,EAC3B;AACF;AACC,OAAO,cAAc,EAAE,KAAK,CAAC,MAAM,EAAE,OAAO,MAAM,EAAE,CAAC;","names":[]}
+{"version":3,"sources":["../src/service-worker.ts"],"sourcesContent":["import {\n  DataBinding,\n  DBResponse,\n  DeepPartial,\n  Empty,\n  ProcessConfigRequest,\n  ProcessConfigResponse,\n  ProcessResult,\n  ProcessStreamResponse,\n  ProcessStreamResponse_Partitions,\n  StartRequest\n} from '@sentio/protos'\nimport { CallContext, ServerError, Status } from 'nice-grpc'\nimport { PluginManager } from './plugin.js'\nimport { errorString } from './utils.js'\nimport { freezeGlobalConfig } from './global-config.js'\nimport { DebugInfo, RichServerError } from 'nice-grpc-error-details'\nimport { recordRuntimeInfo } from './service.js'\nimport { BroadcastChannel, MessagePort, threadId } from 'worker_threads'\nimport { Piscina } from 'piscina'\nimport { configureEndpoints } from './endpoints.js'\nimport { setupLogger } from './logger.js'\nimport { AbstractStoreContext } from './db-context.js'\n\nlet started = false\n\nlet unhandled: Error | undefined\n\nprocess\n  .on('uncaughtException', (err) => {\n    console.error('Uncaught Exception, please checking if await is properly used', err)\n    unhandled = err\n  })\n  .on('unhandledRejection', (reason, p) => {\n    // @ts-ignore ignore invalid ens error\n    if (reason?.message.startsWith('invalid ENS name (disallowed character: \"*\"')) {\n      return\n    }\n    console.error('Unhandled Rejection, please checking if await is properly', reason)\n    unhandled = reason as Error\n    // shutdownServers(1)\n  })\n  .on('exit', () => {\n    console.info('Worker thread exiting, threadId:', threadId)\n  })\n\nasync function getConfig(request: ProcessConfigRequest, context?: CallContext): Promise<ProcessConfigResponse> {\n  if (!started) {\n    throw new ServerError(Status.UNAVAILABLE, 'Service Not started.')\n  }\n\n  const newConfig = ProcessConfigResponse.fromPartial({})\n  await PluginManager.INSTANCE.configure(newConfig)\n  return newConfig\n}\n\nconst loader = async (options: any) => {\n  if (options.target) {\n    const m = await import(options.target)\n    console.debug('Module loaded, path:', options.target, 'module:', m)\n    return m\n  }\n}\n\nconst configureChannel = new BroadcastChannel('configure_channel')\nconfigureChannel.onmessage = (request: ProcessConfigRequest) => {\n  getConfig(request)\n}\n\nasync function start(request: StartRequest, options: any): Promise<Empty> {\n  if (started) {\n    return {}\n  }\n  freezeGlobalConfig()\n\n  try {\n    await loader(options)\n  } catch (e) {\n    throw new ServerError(Status.INVALID_ARGUMENT, 'Failed to load processor: ' + errorString(e))\n  }\n\n  await PluginManager.INSTANCE.start(request)\n  started = true\n  return {}\n}\n\nexport default async function ({\n  request,\n  processId,\n  workerPort,\n  partition\n}: {\n  request: DataBinding\n  processId: number\n  workerPort?: MessagePort\n  partition?: boolean\n}) {\n  const { startRequest, configRequest, options } = Piscina.workerData\n  if (!started) {\n    const logLevel = process.env['LOG_LEVEL']?.toUpperCase()\n    setupLogger(options['log-format'] === 'json', logLevel === 'debug' ? true : options.debug, threadId)\n\n    configureEndpoints(options)\n\n    if (startRequest) {\n      await start(startRequest, options)\n      console.debug('worker', threadId, ' started, template instance:', startRequest.templateInstances?.length)\n    }\n\n    if (configRequest) {\n      await getConfig(configRequest)\n      console.debug('worker', threadId, ' configured')\n    }\n  }\n\n  if (unhandled) {\n    const err = unhandled\n    unhandled = undefined\n    console.error('Unhandled exception/rejection in previous request:', err)\n    throw new RichServerError(\n      Status.UNAVAILABLE,\n      'Unhandled exception/rejection in previous request: ' + errorString(err),\n      [\n        DebugInfo.fromPartial({\n          detail: err.message,\n          stackEntries: err.stack?.split('\\n')\n        })\n      ]\n    )\n  }\n\n  console.debug('Worker', threadId, 'starting request ', processId)\n  const now = Date.now()\n  const timeoutPromise = new Promise((resolve, reject) => {\n    setTimeout(\n      () => {\n        reject(\n          new RichServerError(\n            Status.DEADLINE_EXCEEDED,\n            `Request ${processId} timed out after ${options['request-timeout']} ms`\n          )\n        )\n      },\n      (options['process-timeout'] ?? 60) * 1000\n    )\n  })\n  let resultPromise: Promise<ProcessResult | ProcessStreamResponse_Partitions>\n  if (partition) {\n    resultPromise = PluginManager.INSTANCE.partition(request)\n  } else {\n    resultPromise = PluginManager.INSTANCE.processBinding(\n      request,\n      undefined,\n      workerPort ? new WorkerStoreContext(workerPort, processId) : undefined\n    ).then((result) => {\n      recordRuntimeInfo(result, request.handlerType)\n      return result\n    })\n  }\n\n  try {\n    const result = await Promise.race([resultPromise, timeoutPromise])\n    console.debug(\n      'worker',\n      threadId,\n      `finished ${partition ? 'partition' : 'process'} request`,\n      processId,\n      'took',\n      Date.now() - now\n    )\n    return result\n  } catch (e) {\n    console.error('worker', threadId, `failed ${partition ? 'partition' : 'process'} request`, processId, 'error:', e)\n    if (e instanceof RichServerError) {\n      throw e\n    } else {\n      throw new RichServerError(\n        Status.INTERNAL,\n        `Worker ${threadId} failed to process request ${processId}: ` + errorString(e),\n        [\n          DebugInfo.fromPartial({\n            detail: errorString(e),\n            stackEntries: e.stack?.split('\\n')\n          })\n        ]\n      )\n    }\n  }\n}\n\nclass WorkerStoreContext extends AbstractStoreContext {\n  constructor(\n    readonly port: MessagePort,\n    processId: number\n  ) {\n    super(processId)\n    this.port.on('message', (resp: DBResponse) => {\n      this.result(resp)\n    })\n    this.port.on('close', () => {\n      this.close()\n    })\n  }\n\n  doSend(req: DeepPartial<ProcessStreamResponse>): void {\n    this.port.postMessage(req)\n  }\n}\n;import(\"node:process\").then((p) => p.stdout.write(\"\"));"],"mappings":";;;;;;;;;;;;;;;;;;;;AAYA,uBAAiD;AAIjD,qCAA2C;AAE3C,SAAS,kBAA+B,gBAAgB;AACxD,SAAS,eAAe;AAKxB,IAAI,UAAU;AAEd,IAAI;AAEJ,QACG,GAAG,qBAAqB,CAAC,QAAQ;AAChC,UAAQ,MAAM,iEAAiE,GAAG;AAClF,cAAY;AACd,CAAC,EACA,GAAG,sBAAsB,CAAC,QAAQ,MAAM;AAEvC,MAAI,QAAQ,QAAQ,WAAW,6CAA6C,GAAG;AAC7E;AAAA,EACF;AACA,UAAQ,MAAM,6DAA6D,MAAM;AACjF,cAAY;AAEd,CAAC,EACA,GAAG,QAAQ,MAAM;AAChB,UAAQ,KAAK,oCAAoC,QAAQ;AAC3D,CAAC;AAEH,eAAe,UAAU,SAA+B,SAAuD;AAC7G,MAAI,CAAC,SAAS;AACZ,UAAM,IAAI,6BAAY,wBAAO,aAAa,sBAAsB;AAAA,EAClE;AAEA,QAAM,YAAY,sBAAsB,YAAY,CAAC,CAAC;AACtD,QAAM,cAAc,SAAS,UAAU,SAAS;AAChD,SAAO;AACT;AAEA,IAAM,SAAS,OAAO,YAAiB;AACrC,MAAI,QAAQ,QAAQ;AAClB,UAAM,IAAI,MAAM,OAAO,QAAQ;AAC/B,YAAQ,MAAM,wBAAwB,QAAQ,QAAQ,WAAW,CAAC;AAClE,WAAO;AAAA,EACT;AACF;AAEA,IAAM,mBAAmB,IAAI,iBAAiB,mBAAmB;AACjE,iBAAiB,YAAY,CAAC,YAAkC;AAC9D,YAAU,OAAO;AACnB;AAEA,eAAe,MAAM,SAAuB,SAA8B;AACxE,MAAI,SAAS;AACX,WAAO,CAAC;AAAA,EACV;AACA,qBAAmB;AAEnB,MAAI;AACF,UAAM,OAAO,OAAO;AAAA,EACtB,SAAS,GAAG;AACV,UAAM,IAAI,6BAAY,wBAAO,kBAAkB,+BAA+B,YAAY,CAAC,CAAC;AAAA,EAC9F;AAEA,QAAM,cAAc,SAAS,MAAM,OAAO;AAC1C,YAAU;AACV,SAAO,CAAC;AACV;AAEA,eAAO,uBAAwB;AAAA,EAC7B;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF,GAKG;AACD,QAAM,EAAE,cAAc,eAAe,QAAQ,IAAI,QAAQ;AACzD,MAAI,CAAC,SAAS;AACZ,UAAM,WAAW,QAAQ,IAAI,WAAW,GAAG,YAAY;AACvD,gBAAY,QAAQ,YAAY,MAAM,QAAQ,aAAa,UAAU,OAAO,QAAQ,OAAO,QAAQ;AAEnG,uBAAmB,OAAO;AAE1B,QAAI,cAAc;AAChB,YAAM,MAAM,cAAc,OAAO;AACjC,cAAQ,MAAM,UAAU,UAAU,gCAAgC,aAAa,mBAAmB,MAAM;AAAA,IAC1G;AAEA,QAAI,eAAe;AACjB,YAAM,UAAU,aAAa;AAC7B,cAAQ,MAAM,UAAU,UAAU,aAAa;AAAA,IACjD;AAAA,EACF;AAEA,MAAI,WAAW;AACb,UAAM,MAAM;AACZ,gBAAY;AACZ,YAAQ,MAAM,sDAAsD,GAAG;AACvE,UAAM,IAAI;AAAA,MACR,wBAAO;AAAA,MACP,wDAAwD,YAAY,GAAG;AAAA,MACvE;AAAA,QACE,yCAAU,YAAY;AAAA,UACpB,QAAQ,IAAI;AAAA,UACZ,cAAc,IAAI,OAAO,MAAM,IAAI;AAAA,QACrC,CAAC;AAAA,MACH;AAAA,IACF;AAAA,EACF;AAEA,UAAQ,MAAM,UAAU,UAAU,qBAAqB,SAAS;AAChE,QAAM,MAAM,KAAK,IAAI;AACrB,QAAM,iBAAiB,IAAI,QAAQ,CAAC,SAAS,WAAW;AACtD;AAAA,MACE,MAAM;AACJ;AAAA,UACE,IAAI;AAAA,YACF,wBAAO;AAAA,YACP,WAAW,SAAS,oBAAoB,QAAQ,iBAAiB,CAAC;AAAA,UACpE;AAAA,QACF;AAAA,MACF;AAAA,OACC,QAAQ,iBAAiB,KAAK,MAAM;AAAA,IACvC;AAAA,EACF,CAAC;AACD,MAAI;AACJ,MAAI,WAAW;AACb,oBAAgB,cAAc,SAAS,UAAU,OAAO;AAAA,EAC1D,OAAO;AACL,oBAAgB,cAAc,SAAS;AAAA,MACrC;AAAA,MACA;AAAA,MACA,aAAa,IAAI,mBAAmB,YAAY,SAAS,IAAI;AAAA,IAC/D,EAAE,KAAK,CAAC,WAAW;AACjB,wBAAkB,QAAQ,QAAQ,WAAW;AAC7C,aAAO;AAAA,IACT,CAAC;AAAA,EACH;AAEA,MAAI;AACF,UAAM,SAAS,MAAM,QAAQ,KAAK,CAAC,eAAe,cAAc,CAAC;AACjE,YAAQ;AAAA,MACN;AAAA,MACA;AAAA,MACA,YAAY,YAAY,cAAc,SAAS;AAAA,MAC/C;AAAA,MACA;AAAA,MACA,KAAK,IAAI,IAAI;AAAA,IACf;AACA,WAAO;AAAA,EACT,SAAS,GAAG;AACV,YAAQ,MAAM,UAAU,UAAU,UAAU,YAAY,cAAc,SAAS,YAAY,WAAW,UAAU,CAAC;AACjH,QAAI,aAAa,gDAAiB;AAChC,YAAM;AAAA,IACR,OAAO;AACL,YAAM,IAAI;AAAA,QACR,wBAAO;AAAA,QACP,UAAU,QAAQ,8BAA8B,SAAS,OAAO,YAAY,CAAC;AAAA,QAC7E;AAAA,UACE,yCAAU,YAAY;AAAA,YACpB,QAAQ,YAAY,CAAC;AAAA,YACrB,cAAc,EAAE,OAAO,MAAM,IAAI;AAAA,UACnC,CAAC;AAAA,QACH;AAAA,MACF;AAAA,IACF;AAAA,EACF;AACF;AAEA,IAAM,qBAAN,cAAiC,qBAAqB;AAAA,EACpD,YACW,MACT,WACA;AACA,UAAM,SAAS;AAHN;AAIT,SAAK,KAAK,GAAG,WAAW,CAAC,SAAqB;AAC5C,WAAK,OAAO,IAAI;AAAA,IAClB,CAAC;AACD,SAAK,KAAK,GAAG,SAAS,MAAM;AAC1B,WAAK,MAAM;AAAA,IACb,CAAC;AAAA,EACH;AAAA,EAEA,OAAO,KAA+C;AACpD,SAAK,KAAK,YAAY,GAAG;AAAA,EAC3B;AACF;AACC,OAAO,cAAc,EAAE,KAAK,CAAC,MAAM,EAAE,OAAO,MAAM,EAAE,CAAC;","names":[]}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sentio/runtime",
-  "version": "2.59.0-rc.31",
+  "version": "2.59.0-rc.33",
   "license": "Apache-2.0",
   "type": "module",
   "exports": {

package/src/db-context.ts

@@ -95,9 +95,6 @@ export abstract class AbstractStoreContext implements IStoreContext {
 
     return Promise.race(promises)
       .then((result: DBResponse) => {
-        if (timeoutSecs) {
-          console.debug('db request', requestType, 'op', opId, ' took', Date.now() - start, 'ms')
-        }
         request_times[requestType]?.add(Date.now() - start)
         return result
       })

package/src/processor-runner.ts

@@ -49,7 +49,8 @@ export const optionDefinitions = [
   { name: 'debug', type: Boolean, defaultValue: false },
   { name: 'otlp-debug', type: Boolean, defaultValue: false },
   { name: 'start-action-server', type: Boolean, defaultValue: false },
-  { name: 'worker', type: Number, defaultValue: workerNum }
+  { name: 'worker', type: Number, defaultValue: workerNum },
+  { name: 'process-timeout', type: Number, defaultValue: 60 } // 60 seconds
 ]
 
 const options = commandLineArgs(optionDefinitions, { partial: true })

package/src/service-manager.ts

@@ -22,7 +22,6 @@ import { processMetrics } from './metrics.js'
 import { MessageChannel } from 'node:worker_threads'
 import { ProcessorServiceImpl } from './service.js'
 import { TemplateInstanceState } from './state.js'
-import { PluginManager } from './plugin.js'
 
 const { process_binding_count, process_binding_time, process_binding_error } = processMetrics
 
@@ -82,8 +81,8 @@ export class ServiceManager extends ProcessorServiceImpl {
       try {
         // console.debug('received request:', request)
         if (request.binding) {
+          lastBinding = request.binding
           process_binding_count.add(1)
-
           // Adjust binding will make some request become invalid by setting UNKNOWN HandlerType
           // for older SDK version, so we just return empty result for them here
           if (request.binding.handlerType === HandlerType.UNKNOWN) {
@@ -93,15 +92,8 @@ export class ServiceManager extends ProcessorServiceImpl {
             })
             continue
           }
-
           if (this.enablePartition) {
-            PluginManager.INSTANCE.partition(request.binding).then((partitions) => {
-              subject.next({
-                processId: request.processId,
-                partitions
-              })
-            })
-            lastBinding = request.binding
+            this.doPartition(request.processId, request.binding, subject)
           } else {
             this.doProcess(request.processId, request.binding, subject)
           }
@@ -127,12 +119,44 @@ export class ServiceManager extends ProcessorServiceImpl {
     }
   }
 
+  private async doPartition(
+    processId: number,
+    binding: DataBinding,
+    subject: Subject<DeepPartial<ProcessStreamResponse>>
+  ) {
+    if (!this.pool) {
+      await this.initPool()
+    }
+    const dbContext = this.contexts.new(processId, subject)
+    const start = Date.now()
+
+    this.process(binding, processId, dbContext, true)
+      .then(async (result) => {
+        console.debug('partition', processId, 'finished, took:', Date.now() - start)
+        subject.next({
+          result,
+          processId: processId
+        })
+      })
+      .catch((e) => {
+        dbContext.error(processId, e)
+        process_binding_error.add(1)
+        console.error('partition', processId, 'failed, took:', Date.now() - start)
+      })
+      .finally(() => {
+        const cost = Date.now() - start
+        process_binding_time.add(cost)
+        this.contexts.delete(processId)
+      })
+  }
+
   private doProcess(processId: number, binding: DataBinding, subject: Subject<DeepPartial<ProcessStreamResponse>>) {
     const dbContext = this.contexts.new(processId, subject)
 
     const start = Date.now()
-    this.process(binding, dbContext)
+    this.process(binding, processId, dbContext, false)
       .then(async (result) => {
+        console.debug('process', processId, 'finished, took:', Date.now() - start)
         subject.next({
           result,
           processId: processId
@@ -141,6 +165,7 @@ export class ServiceManager extends ProcessorServiceImpl {
       .catch((e) => {
         dbContext.error(processId, e)
         process_binding_error.add(1)
+        console.error('process', processId, 'failed, took:', Date.now() - start)
       })
       .finally(() => {
         const cost = Date.now() - start
@@ -149,13 +174,18 @@ export class ServiceManager extends ProcessorServiceImpl {
       })
   }
 
-  async process(request: DataBinding, dbContext?: ChannelStoreContext): Promise<ProcessResult> {
+  async process(
+    request: DataBinding,
+    processId: number,
+    dbContext?: ChannelStoreContext,
+    partition: boolean = false
+  ): Promise<ProcessResult> {
     if (!this.pool) {
       await this.initPool()
     }
 
     return this.pool.run(
-      { request, workerPort: dbContext?.workerPort },
+      { request, workerPort: dbContext?.workerPort, partition, processId },
       { transferList: dbContext?.workerPort ? [dbContext?.workerPort] : [] }
     )
   }
@@ -181,8 +211,6 @@ export class ServiceManager extends ProcessorServiceImpl {
   }
 }
 
-export type WorkerMessage = DBRequest & { processId: number }
-
 class Contexts {
   private contexts: Map<number, ChannelStoreContext> = new Map()
 
@@ -191,7 +219,11 @@ class Contexts {
   }
 
   new(processId: number, subject: Subject<DeepPartial<ProcessStreamResponse>>) {
-    const context = new ChannelStoreContext(subject, processId)
+    let context = this.get(processId)
+    if (context) {
+      return context
+    }
+    context = new ChannelStoreContext(subject, processId)
     this.contexts.set(processId, context)
     return context
   }

package/src/service-worker.ts

@@ -5,7 +5,9 @@ import {
   Empty,
   ProcessConfigRequest,
   ProcessConfigResponse,
+  ProcessResult,
   ProcessStreamResponse,
+  ProcessStreamResponse_Partitions,
   StartRequest
 } from '@sentio/protos'
 import { CallContext, ServerError, Status } from 'nice-grpc'
@@ -38,6 +40,9 @@ process
     unhandled = reason as Error
     // shutdownServers(1)
   })
+  .on('exit', () => {
+    console.info('Worker thread exiting, threadId:', threadId)
+  })
 
 async function getConfig(request: ProcessConfigRequest, context?: CallContext): Promise<ProcessConfigResponse> {
   if (!started) {
@@ -82,11 +87,13 @@ async function start(request: StartRequest, options: any): Promise<Empty> {
 export default async function ({
   request,
   processId,
-  workerPort
+  workerPort,
+  partition
 }: {
   request: DataBinding
   processId: number
   workerPort?: MessagePort
+  partition?: boolean
 }) {
   const { startRequest, configRequest, options } = Piscina.workerData
   if (!started) {
@@ -97,18 +104,19 @@ export default async function ({
 
     if (startRequest) {
       await start(startRequest, options)
-      console.debug('worker started, template instance:', startRequest.templateInstances?.length)
+      console.debug('worker', threadId, ' started, template instance:', startRequest.templateInstances?.length)
     }
 
     if (configRequest) {
       await getConfig(configRequest)
-      console.debug('worker configured')
+      console.debug('worker', threadId, ' configured')
     }
   }
 
   if (unhandled) {
     const err = unhandled
     unhandled = undefined
+    console.error('Unhandled exception/rejection in previous request:', err)
     throw new RichServerError(
       Status.UNAVAILABLE,
       'Unhandled exception/rejection in previous request: ' + errorString(err),
@@ -121,13 +129,63 @@ export default async function ({
     )
   }
 
-  const result = await PluginManager.INSTANCE.processBinding(
-    request,
-    undefined,
-    workerPort ? new WorkerStoreContext(workerPort, processId) : undefined
-  )
-  recordRuntimeInfo(result, request.handlerType)
-  return result
+  console.debug('Worker', threadId, 'starting request ', processId)
+  const now = Date.now()
+  const timeoutPromise = new Promise((resolve, reject) => {
+    setTimeout(
+      () => {
+        reject(
+          new RichServerError(
+            Status.DEADLINE_EXCEEDED,
+            `Request ${processId} timed out after ${options['request-timeout']} ms`
+          )
+        )
+      },
+      (options['process-timeout'] ?? 60) * 1000
+    )
+  })
+  let resultPromise: Promise<ProcessResult | ProcessStreamResponse_Partitions>
+  if (partition) {
+    resultPromise = PluginManager.INSTANCE.partition(request)
+  } else {
+    resultPromise = PluginManager.INSTANCE.processBinding(
+      request,
+      undefined,
+      workerPort ? new WorkerStoreContext(workerPort, processId) : undefined
+    ).then((result) => {
+      recordRuntimeInfo(result, request.handlerType)
+      return result
+    })
+  }
+
+  try {
+    const result = await Promise.race([resultPromise, timeoutPromise])
+    console.debug(
+      'worker',
+      threadId,
+      `finished ${partition ? 'partition' : 'process'} request`,
+      processId,
+      'took',
+      Date.now() - now
+    )
+    return result
+  } catch (e) {
+    console.error('worker', threadId, `failed ${partition ? 'partition' : 'process'} request`, processId, 'error:', e)
+    if (e instanceof RichServerError) {
+      throw e
+    } else {
+      throw new RichServerError(
+        Status.INTERNAL,
+        `Worker ${threadId} failed to process request ${processId}: ` + errorString(e),
+        [
+          DebugInfo.fromPartial({
+            detail: errorString(e),
+            stackEntries: e.stack?.split('\n')
+          })
+        ]
+      )
+    }
+  }
 }
 
 class WorkerStoreContext extends AbstractStoreContext {