@sentio/runtime 2.59.0-rc.4 → 2.59.0-rc.40

package/lib/index.js

@@ -22,17 +22,18 @@ import {
   getProvider,
   makeEthCallKey,
   mergeProcessResults,
+  mergeProcessResultsInPlace,
   metricsStorage,
   parseSemver,
   processMetrics,
   providerMetrics,
   recordRuntimeInfo,
   timeoutError
-} from "./chunk-QELD44EL.js";
+} from "./chunk-ELSE7PSY.js";
 import {
   Plugin,
   PluginManager
-} from "./chunk-ZUTD563J.js";
+} from "./chunk-3SLKMZUX.js";
 
 // src/chain-config.ts
 import("node:process").then((p) => p.stdout.write(""));
@@ -62,6 +63,7 @@ export {
   getProvider,
   makeEthCallKey,
   mergeProcessResults,
+  mergeProcessResultsInPlace,
   metricsStorage,
   parseSemver,
   processMetrics,

package/lib/index.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../src/chain-config.ts","../src/index.ts"],"sourcesContent":["export interface ChainConfig {\n  ChainID: string\n  Https?: string[]\n  ChainServer?: string\n}\n;import(\"node:process\").then((p) => p.stdout.write(\"\"));","export * from './plugin.js'\nexport * from './state.js'\nexport * from './utils.js'\nexport * from './endpoints.js'\nexport * from './chain-config.js'\nexport * from './service.js'\nexport { GLOBAL_CONFIG, type GlobalConfig } from './global-config.js'\nexport * from './db-context.js'\nexport * from './provider.js'\nexport * from './metrics.js'\n;import(\"node:process\").then((p) => p.stdout.write(\"\"));"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAKC,OAAO,cAAc,EAAE,KAAK,CAAC,MAAM,EAAE,OAAO,MAAM,EAAE,CAAC;;;ACKrD,OAAO,cAAc,EAAE,KAAK,CAAC,MAAM,EAAE,OAAO,MAAM,EAAE,CAAC;","names":[]}
+{"version":3,"sources":["../src/chain-config.ts","../src/index.ts"],"sourcesContent":["export interface ChainConfig {\n  ChainID: string\n  Https?: string[]\n  ChainServer?: string\n}\n;import(\"node:process\").then((p) => p.stdout.write(\"\"));","export * from './plugin.js'\nexport * from './state.js'\nexport * from './utils.js'\nexport * from './endpoints.js'\nexport * from './chain-config.js'\nexport * from './service.js'\nexport { GLOBAL_CONFIG, type GlobalConfig } from './global-config.js'\nexport * from './db-context.js'\nexport * from './provider.js'\nexport * from './metrics.js'\nexport {\n  type DataBinding,\n  type Data_EthLog,\n  type Data_EthBlock,\n  type Data_EthTransaction,\n  type Data_EthTrace\n} from './gen/processor/protos/processor.js'\n;import(\"node:process\").then((p) => p.stdout.write(\"\"));"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAKC,OAAO,cAAc,EAAE,KAAK,CAAC,MAAM,EAAE,OAAO,MAAM,EAAE,CAAC;;;ACYrD,OAAO,cAAc,EAAE,KAAK,CAAC,MAAM,EAAE,OAAO,MAAM,EAAE,CAAC;","names":[]}

package/lib/processor-runner.js

@@ -5,7 +5,7 @@ import {
 } from "./chunk-6XHWJ2VS.js";
 import {
   setupLogger
-} from "./chunk-BPGFX5S5.js";
+} from "./chunk-FDZMT76B.js";
 import {
   DiagConsoleLogger,
   DiagLogLevel,
@@ -35,7 +35,6 @@ import {
   isValidTraceId,
   metrics,
   parseSemver,
-  processMetrics,
   propagation,
   require_lib,
   require_lib2,
@@ -44,13 +43,11 @@ import {
   require_lodash,
   require_src,
   trace
-} from "./chunk-QELD44EL.js";
+} from "./chunk-ELSE7PSY.js";
 import {
   ExecutionConfig,
-  HandlerType,
   PluginManager,
   ProcessConfigResponse,
-  ProcessResult,
   ProcessorDefinition,
   StartRequest,
   __commonJS,
@@ -60,7 +57,7 @@ import {
   __toCommonJS,
   __toESM,
   require_minimal2 as require_minimal
-} from "./chunk-ZUTD563J.js";
+} from "./chunk-3SLKMZUX.js";
 
 // ../../node_modules/.pnpm/@opentelemetry+semantic-conventions@1.25.1/node_modules/@opentelemetry/semantic-conventions/build/esm/internal/utils.js
 // @__NO_SIDE_EFFECTS__
@@ -25956,6 +25953,7 @@ import { createRequire } from "module";
 var import_fs_extra = __toESM(require_lib(), 1);
 import path from "path";
 import os from "os";
+import { createHash } from "crypto";
 var require2 = createRequire(import.meta.url);
 var FUEL_PROTO_UPDATE_VERSION = parseSemver("2.54.0-rc.7");
 var FUEL_PROTO_NO_FUEL_TRANSACTION_AS_CALL_VERSION = parseSemver("2.55.0-rc.1");
@@ -25964,6 +25962,22 @@ var ETH_USE_RAW_VERSION = parseSemver("2.57.9-rc.12");
 var PROCESSED_MOVE_EVENT_TX_HANDLER = new LRUCache({
   max: 1e4
 });
+var enableTxCache = process.env.ENABLE_PARSE_CACHE === "true";
+var PARSED_DATA_CACHE = new LRUCache({
+  max: enableTxCache ? 5e3 : 1
+});
+function getParsedData(rawData) {
+  if (!enableTxCache) {
+    return JSON.parse(rawData);
+  }
+  const digest = createHash("md5").update(rawData).digest("hex");
+  let parsedData = PARSED_DATA_CACHE.get(digest);
+  if (!parsedData) {
+    parsedData = JSON.parse(rawData);
+    PARSED_DATA_CACHE.set(digest, parsedData);
+  }
+  return parsedData;
+}
 function locatePackageJson(pkgId) {
   const m = require2.resolve(pkgId);
   let dir = path.dirname(m);
@@ -26064,17 +26078,31 @@ var FullProcessorServiceImpl = class {
         const ethLog = dataBinding.data?.ethLog;
         if (ethLog?.log == null && ethLog?.rawLog) {
           ethLog.log = JSON.parse(ethLog.rawLog);
-          ethLog.transaction = ethLog.rawTransaction ? JSON.parse(ethLog.rawTransaction) : void 0;
-          ethLog.block = ethLog.rawBlock ? JSON.parse(ethLog.rawBlock) : void 0;
-          ethLog.transactionReceipt = ethLog.rawTransactionReceipt ? JSON.parse(ethLog.rawTransactionReceipt) : void 0;
+          if (ethLog.rawTransaction) {
+            ethLog.transaction = getParsedData(ethLog.rawTransaction);
+          }
+          if (ethLog.rawBlock) {
+            ethLog.block = getParsedData(ethLog.rawBlock);
+          }
+          if (ethLog.rawTransactionReceipt) {
+            ethLog.transactionReceipt = getParsedData(ethLog.rawTransactionReceipt);
+          }
         }
         break;
       case 11 /* ETH_TRANSACTION */:
         const ethTx = dataBinding.data?.ethTransaction;
         if (ethTx?.transaction == null && ethTx?.rawTransaction) {
-          ethTx.transaction = JSON.parse(ethTx.rawTransaction);
-          ethTx.block = ethTx.rawBlock ? JSON.parse(ethTx.rawBlock) : void 0;
-          ethTx.transactionReceipt = ethTx.rawTransactionReceipt ? JSON.parse(ethTx.rawTransactionReceipt) : void 0;
+          ethTx.transaction = getParsedData(ethTx.rawTransaction);
+          if (ethTx.rawBlock) {
+            ethTx.block = getParsedData(ethTx.rawBlock);
+          } else {
+            ethTx.block = void 0;
+          }
+          if (ethTx.rawTransactionReceipt) {
+            ethTx.transactionReceipt = getParsedData(ethTx.rawTransactionReceipt);
+          } else {
+            ethTx.transactionReceipt = void 0;
+          }
         }
         break;
       case 20 /* FUEL_TRANSACTION */:
@@ -26097,7 +26125,7 @@ var FullProcessorServiceImpl = class {
         const aptEvent = dataBinding.data?.aptEvent;
         if (aptEvent) {
           if (isBeforeMoveUseRawVersion && aptEvent.rawTransaction) {
-            const transaction = JSON.parse(aptEvent.rawTransaction);
+            const transaction = getParsedData(aptEvent.rawTransaction);
             const key = `${transaction.hash}-${dataBinding.handlerIds[0]}`;
             if (PROCESSED_MOVE_EVENT_TX_HANDLER.has(key)) {
               console.debug("skip binding", key);
@@ -26116,7 +26144,7 @@ var FullProcessorServiceImpl = class {
         const aptCall = dataBinding.data?.aptCall;
         if (aptCall) {
           if (isBeforeMoveUseRawVersion && aptCall.rawTransaction) {
-            aptCall.transaction = JSON.parse(aptCall.rawTransaction);
+            aptCall.transaction = getParsedData(aptCall.rawTransaction);
           }
         }
         break;
@@ -26132,7 +26160,7 @@ var FullProcessorServiceImpl = class {
         const suiEvent = dataBinding.data?.suiEvent;
         if (suiEvent) {
           if (isBeforeMoveUseRawVersion && suiEvent.rawTransaction) {
-            const transaction = JSON.parse(suiEvent.rawTransaction);
+            const transaction = getParsedData(suiEvent.rawTransaction);
             const key = `${transaction.digest}-${dataBinding.handlerIds[0]}`;
             if (PROCESSED_MOVE_EVENT_TX_HANDLER.has(key)) {
               console.debug("skip binding", key);
@@ -26151,7 +26179,7 @@ var FullProcessorServiceImpl = class {
         const suiCall = dataBinding.data?.suiCall;
         if (suiCall) {
           if (isBeforeMoveUseRawVersion && suiCall.rawTransaction) {
-            suiCall.transaction = JSON.parse(suiCall.rawTransaction);
+            suiCall.transaction = getParsedData(suiCall.rawTransaction);
           }
         }
         break;
@@ -26258,13 +26286,12 @@ import("node:process").then((p) => p.stdout.write(""));
 // src/service-manager.ts
 import { Piscina } from "piscina";
 import { MessageChannel } from "node:worker_threads";
-var { process_binding_count, process_binding_time, process_binding_error } = processMetrics;
 BigInt.prototype.toJSON = function() {
   return this.toString();
 };
 var ServiceManager = class extends ProcessorServiceImpl {
-  constructor(options2, loader2, shutdownHandler) {
-    super(loader2, shutdownHandler);
+  constructor(loader2, options2, shutdownHandler) {
+    super(loader2, options2, shutdownHandler);
     this.options = options2;
     this.workerData.options = options2;
   }
@@ -26290,58 +26317,47 @@ var ServiceManager = class extends ProcessorServiceImpl {
     await this.pool?.destroy();
     return await super.stop(request3, context2);
   }
-  async process(request3, dbContext) {
+  contexts = new Contexts();
+  async handleRequests(requests, subject) {
     if (!this.pool) {
       await this.initPool();
     }
-    return this.pool.run(
-      { request: request3, workerPort: dbContext?.workerPort },
-      { transferList: dbContext?.workerPort ? [dbContext?.workerPort] : [] }
-    );
-  }
-  contexts = new Contexts();
-  async handleRequests(requests, subject) {
     for await (const request3 of requests) {
-      try {
-        if (request3.binding) {
-          process_binding_count.add(1);
-          if (request3.binding.handlerType === HandlerType.UNKNOWN) {
-            subject.next({
-              processId: request3.processId,
-              result: ProcessResult.create()
-            });
-            continue;
-          }
-          const binding = request3.binding;
-          const dbContext = this.contexts.new(request3.processId, subject);
-          const start = Date.now();
-          this.process(binding, dbContext).then(async (result) => {
-            subject.next({
-              result,
-              processId: request3.processId
-            });
-          }).catch((e) => {
-            dbContext.error(request3.processId, e);
-            process_binding_error.add(1);
-          }).finally(() => {
-            const cost = Date.now() - start;
-            process_binding_time.add(cost);
-            this.contexts.delete(request3.processId);
-          });
-        }
-        if (request3.dbResult) {
-          const dbContext = this.contexts.get(request3.processId);
-          try {
-            dbContext?.result(request3.dbResult);
-          } catch (e) {
-            subject.error(new Error("db result error, process should stop"));
-          }
+      this.handleSingleRequest(request3, subject);
+    }
+  }
+  async handleSingleRequest(request3, subject) {
+    const processId = request3.processId;
+    if (request3.binding) {
+      const context2 = this.contexts.new(processId);
+      context2.mainPort.on("message", (resp) => {
+        subject.next(resp);
+        if (resp.result) {
+          this.contexts.delete(processId);
         }
-      } catch (e) {
-        console.error("unexpect error during handle loop", e);
+      });
+      await this.pool.run(
+        { request: request3, workerPort: context2.workerPort, processId },
+        { transferList: [context2.workerPort] }
+      );
+    } else {
+      const context2 = this.contexts.get(processId);
+      if (!context2) {
+        console.error("No context found for processId:", processId);
+        throw new Error(`No context found for processId: ${processId}`);
       }
+      context2.sendRequest(request3);
     }
   }
+  async process(processId, context2) {
+    if (!this.pool) {
+      await this.initPool();
+    }
+    return this.pool.run(
+      { workerPort: context2?.workerPort, processId },
+      { transferList: context2?.workerPort ? [context2?.workerPort] : [] }
+    );
+  }
   async initPool() {
     if (this.pool) {
       await this.pool.close();
@@ -26366,8 +26382,12 @@ var Contexts = class {
   get(processId) {
     return this.contexts.get(processId);
   }
-  new(processId, subject) {
-    const context2 = new ChannelStoreContext(subject, processId);
+  new(processId) {
+    let context2 = this.get(processId);
+    if (context2) {
+      return context2;
+    }
+    context2 = new ChannelContext(processId);
     this.contexts.set(processId, context2);
     return context2;
   }
@@ -26376,21 +26396,17 @@ var Contexts = class {
     context2?.close();
     this.contexts.delete(processId);
   }
+  has(processId) {
+    return this.contexts.has(processId);
+  }
 };
-var ChannelStoreContext = class {
-  constructor(subject, processId) {
-    this.subject = subject;
+var ChannelContext = class {
+  constructor(processId) {
     this.processId = processId;
-    this.mainPort.on("message", (req) => {
-      subject.next({
-        ...req,
-        processId
-      });
-    });
   }
   channel = new MessageChannel();
-  sendRequest(request3, timeoutSecs) {
-    throw new Error("should not be used on main thread");
+  sendRequest(request3) {
+    this.mainPort.postMessage(request3);
   }
   get workerPort() {
     return this.channel.port2;
@@ -26398,24 +26414,9 @@ var ChannelStoreContext = class {
   get mainPort() {
     return this.channel.port1;
   }
-  result(dbResult) {
-    this.mainPort.postMessage(dbResult);
-  }
   close() {
     this.mainPort.close();
   }
-  error(processId, e) {
-    console.error("process error", processId, e);
-    const errorResult = ProcessResult.create({
-      states: {
-        error: e?.toString()
-      }
-    });
-    this.subject.next({
-      result: errorResult,
-      processId
-    });
-  }
 };
 import("node:process").then((p) => p.stdout.write(""));
 
@@ -26445,7 +26446,14 @@ var optionDefinitions = [
   { name: "debug", type: Boolean, defaultValue: false },
   { name: "otlp-debug", type: Boolean, defaultValue: false },
   { name: "start-action-server", type: Boolean, defaultValue: false },
-  { name: "worker", type: Number, defaultValue: workerNum }
+  { name: "worker", type: Number, defaultValue: workerNum },
+  { name: "process-timeout", type: Number, defaultValue: 60 },
+  { name: "worker-timeout", type: Number, defaultValue: 60 },
+  {
+    name: "enable-partition",
+    type: Boolean,
+    defaultValue: process.env["SENTIO_ENABLE_BINDING_DATA_PARTITION"] === "true"
+  }
 ];
 var options = command_line_args_default(optionDefinitions, { partial: true });
 var logLevel = process.env["LOG_LEVEL"]?.toLowerCase();
@@ -26472,9 +26480,9 @@ if (options["start-action-server"]) {
     "grpc.default_compression_algorithm": import_grpc_js.compressionAlgorithms.gzip
   }).use((0, import_nice_grpc_opentelemetry.openTelemetryServerMiddleware)()).use(import_nice_grpc_error_details.errorDetailsServerMiddleware);
   if (options.worker > 1) {
-    baseService = new ServiceManager(options, loader, server.shutdown);
+    baseService = new ServiceManager(loader, options, server.shutdown);
   } else {
-    baseService = new ProcessorServiceImpl(loader, server.shutdown);
+    baseService = new ProcessorServiceImpl(loader, options, server.shutdown);
   }
   const service = new FullProcessorServiceImpl(baseService);
   server.add(ProcessorDefinition, service);