@sentio/runtime 3.0.2-rc.2 → 3.1.0-rc.1

package/lib/index.d.ts

@@ -84,6 +84,7 @@ declare const program: Command<[string], {
     processTimeout: number;
     workerTimeout: number;
     enablePartition: boolean;
+    multiServer: number;
 }, {}>;
 type ProcessorRuntimeOptions = ReturnType<typeof program.opts> & {
     target: string;

package/lib/processor-runner.js

@@ -18942,6 +18942,8 @@ var import_nice_grpc2 = __toESM(require_lib2(), 1);
 var import_nice_grpc_error_details = __toESM(require_lib3(), 1);
 import http from "http";
 import { Session } from "node:inspector/promises";
+import { fork } from "child_process";
+import { fileURLToPath } from "url";
 
 // src/full-service.ts
 import { createRequire } from "module";
@@ -19637,6 +19639,10 @@ try {
 } catch (e) {
   console.error("Failed to parse worker number", e);
 }
+var serverNum = 1;
+if (process.env["PROCESSOR_SERVER_NUM"]) {
+  serverNum = parseInt(process.env["PROCESSOR_SERVER_NUM"].trim());
+}
 function myParseInt(value, dummyPrevious) {
   const parsedValue = parseInt(value, 10);
   if (isNaN(parsedValue)) {
@@ -19644,7 +19650,7 @@ function myParseInt(value, dummyPrevious) {
   }
   return parsedValue;
 }
-var program2 = new Command("processor-runner").allowUnknownOption().allowExcessArguments().name("processor-runner").description("Sentio Processor Runtime").argument("<target>", "Path to the processor module to load").option("-p, --port <port>", "Port to listen on", "4000").option("--concurrency <number>", "Number of concurrent workers", myParseInt, 4).option("--batch-count <number>", "Batch count for processing", myParseInt, 1).option("-c, --chains-config <path>", "Path to chains configuration file", "chains-config.json").option("--chainquery-server <url>", "Chain query server URL").option("--pricefeed-server <url>", "Price feed server URL").option("--log-format <format>", "Log format (console|json)", "console").option("--debug", "Enable debug mode").option("--otlp-debug", "Enable OTLP debug mode").option("--start-action-server", "Start action server instead of processor server").option("--worker <number>", "Number of worker threads", myParseInt, workerNum).option("--process-timeout <seconds>", "Process timeout in seconds", myParseInt, 60).option(
+var program2 = new Command("processor-runner").allowUnknownOption().allowExcessArguments().name("processor-runner").description("Sentio Processor Runtime").argument("<target>", "Path to the processor module to load").option("-p, --port <port>", "Port to listen on", "4000").option("--concurrency <number>", "Number of concurrent workers(V2 only, deprecated)", myParseInt, 4).option("--batch-count <number>", "Batch count for processing", myParseInt, 1).option("-c, --chains-config <path>", "Path to chains configuration file", "chains-config.json").option("--chainquery-server <url>", "Chain query server URL").option("--pricefeed-server <url>", "Price feed server URL").option("--log-format <format>", "Log format (console|json)", "console").option("--debug", "Enable debug mode").option("--otlp-debug", "Enable OTLP debug mode").option("--start-action-server", "Start action server instead of processor server").option("--worker <number>", "Number of worker threads(V2 only, deprecated) ", myParseInt, workerNum).option("--process-timeout <seconds>", "Process timeout in seconds", myParseInt, 60).option(
   "--worker-timeout <seconds>",
   "Worker timeout in seconds",
   myParseInt,
@@ -19653,7 +19659,7 @@ var program2 = new Command("processor-runner").allowUnknownOption().allowExcessA
   "--enable-partition",
   "Enable binding data partition",
   process.env["SENTIO_ENABLE_BINDING_DATA_PARTITION"] === "true"
-);
+).option("--multi-server <number>", "Enable multi-server mode for processor runtime", myParseInt, serverNum);
 import("node:process").then((p) => p.stdout.write(""));
 
 // src/processor-runner.ts
@@ -19669,8 +19675,42 @@ await setupOTLP(options.otlpDebug);
 Error.stackTraceLimit = 20;
 configureEndpoints(options);
 console.debug("Starting Server", options);
+var isChildProcess = process.env["SENTIO_MULTI_SERVER_CHILD"] === "true";
+var childServerPort = process.env["SENTIO_CHILD_SERVER_PORT"];
+if (options.multiServer > 1 && !isChildProcess) {
+  const childProcesses = [];
+  const basePort = parseInt(options.port);
+  for (let i = 1; i < options.multiServer; i++) {
+    const childPort = basePort + i;
+    const child = fork(fileURLToPath(import.meta.url), process.argv.slice(2), {
+      env: {
+        ...process.env,
+        SENTIO_MULTI_SERVER_CHILD: "true",
+        SENTIO_CHILD_SERVER_PORT: String(childPort)
+      },
+      stdio: "inherit"
+    });
+    child.on("error", (err) => {
+      console.error(`Child process on port ${childPort} error:`, err);
+    });
+    child.on("exit", (code) => {
+      console.log(`Child process on port ${childPort} exited with code ${code}`);
+    });
+    childProcesses.push(child);
+    console.log(`Spawned child server process for port ${childPort}`);
+  }
+  const shutdownChildren = () => {
+    for (const child of childProcesses) {
+      child.kill("SIGINT");
+    }
+  };
+  process.on("SIGINT", shutdownChildren);
+  process.on("SIGTERM", shutdownChildren);
+}
+var actualPort = isChildProcess && childServerPort ? childServerPort : options.port;
 var server;
 var baseService;
+var httpServer;
 var loader = async () => {
   const m = await import(options.target);
   console.debug("Module loaded", m);
@@ -19678,7 +19718,7 @@ var loader = async () => {
 };
 if (options.startActionServer) {
   server = new ActionServer(loader);
-  server.listen(options.port);
+  server.listen(actualPort);
 } else {
   server = (0, import_nice_grpc2.createServer)({
     "grpc.max_send_message_length": 768 * 1024 * 1024,
@@ -19696,59 +19736,77 @@ if (options.startActionServer) {
     ProcessorV3Definition,
     new FullProcessorServiceV3Impl(new ProcessorServiceImplV3(loader, options, server.shutdown))
   );
-  server.listen("0.0.0.0:" + options.port);
-  console.log("Processor Server Started at:", options.port);
+  server.listen("0.0.0.0:" + actualPort);
+  console.log("Processor Server Started at:", actualPort);
 }
-var metricsPort = 4040;
-var httpServer = http.createServer(async function(req, res) {
-  if (req.url) {
-    const reqUrl = new URL(req.url, `http://${req.headers.host}`);
-    const queries = reqUrl.searchParams;
-    switch (reqUrl.pathname) {
-      // case '/metrics':
-      //   const metrics = await mergedRegistry.metrics()
-      //   res.write(metrics)
-      //   break
-      case "/heap": {
-        try {
-          const file = "/tmp/" + Date.now() + ".heapsnapshot";
-          await dumpHeap(file);
-          const readStream = import_fs_extra2.default.createReadStream(file);
-          res.writeHead(200, { "Content-Type": "application/json" });
-          readStream.pipe(res);
-          res.end();
-        } catch {
-          res.writeHead(500);
-          res.end();
+if (!isChildProcess) {
+  const metricsPort = 4040;
+  httpServer = http.createServer(async function(req, res) {
+    if (req.url) {
+      const reqUrl = new URL(req.url, `http://${req.headers.host}`);
+      const queries = reqUrl.searchParams;
+      switch (reqUrl.pathname) {
+        // case '/metrics':
+        //   const metrics = await mergedRegistry.metrics()
+        //   res.write(metrics)
+        //   break
+        case "/heap": {
+          try {
+            const file = "/tmp/" + Date.now() + ".heapsnapshot";
+            await dumpHeap(file);
+            const readStream = import_fs_extra2.default.createReadStream(file);
+            res.writeHead(200, { "Content-Type": "application/json" });
+            readStream.pipe(res);
+            res.end();
+          } catch {
+            res.writeHead(500);
+            res.end();
+          }
+          break;
         }
-        break;
-      }
-      case "/profile": {
-        try {
-          const profileTime = parseInt(queries.get("t") || "1000", 10) || 1e3;
-          const session = new Session();
-          session.connect();
-          await session.post("Profiler.enable");
-          await session.post("Profiler.start");
-          await new Promise((resolve) => setTimeout(resolve, profileTime));
-          const { profile } = await session.post("Profiler.stop");
-          res.writeHead(200, { "Content-Type": "application/json" });
-          res.write(JSON.stringify(profile));
-          session.disconnect();
-        } catch {
-          res.writeHead(500);
+        case "/profile": {
+          try {
+            const profileTime = parseInt(queries.get("t") || "1000", 10) || 1e3;
+            const session = new Session();
+            session.connect();
+            await session.post("Profiler.enable");
+            await session.post("Profiler.start");
+            await new Promise((resolve) => setTimeout(resolve, profileTime));
+            const { profile } = await session.post("Profiler.stop");
+            res.writeHead(200, { "Content-Type": "application/json" });
+            res.write(JSON.stringify(profile));
+            session.disconnect();
+          } catch {
+            res.writeHead(500);
+          }
+          break;
         }
-        break;
+        default:
+          res.writeHead(404);
       }
-      default:
-        res.writeHead(404);
+    } else {
+      res.writeHead(404);
     }
-  } else {
-    res.writeHead(404);
+    res.end();
+  }).listen(metricsPort);
+  console.log("Metric Server Started at:", metricsPort);
+  if (process.env["OOM_DUMP_MEMORY_SIZE_GB"]) {
+    let dumping = false;
+    const memorySize = parseFloat(process.env["OOM_DUMP_MEMORY_SIZE_GB"]);
+    console.log("heap dumping is enabled, limit set to ", memorySize, "gb");
+    const dir = process.env["OOM_DUMP_DIR"] || "/tmp";
+    setInterval(async () => {
+      const mem = process.memoryUsage();
+      console.log("Current Memory Usage", mem);
+      if (mem.heapTotal > memorySize * 1024 * 1024 * 1024 && !dumping) {
+        const file = join(dir, `${Date.now()}.heapsnapshot`);
+        dumping = true;
+        await dumpHeap(file);
+        process.exit(11);
+      }
+    }, 1e3 * 60);
   }
-  res.end();
-}).listen(metricsPort);
-console.log("Metric Server Started at:", metricsPort);
+}
 process.on("SIGINT", function() {
   shutdownServers(0);
 }).on("uncaughtException", (err) => {
@@ -19756,7 +19814,7 @@ process.on("SIGINT", function() {
   if (baseService) {
     baseService.unhandled = err;
   }
-}).on("unhandledRejection", (reason, p) => {
+}).on("unhandledRejection", (reason, _p) => {
   if (reason?.message.startsWith('invalid ENS name (disallowed character: "*"')) {
     return;
   }
@@ -19765,22 +19823,6 @@ process.on("SIGINT", function() {
     baseService.unhandled = reason;
   }
 });
-if (process.env["OOM_DUMP_MEMORY_SIZE_GB"]) {
-  let dumping = false;
-  const memorySize = parseFloat(process.env["OOM_DUMP_MEMORY_SIZE_GB"]);
-  console.log("heap dumping is enabled, limit set to ", memorySize, "gb");
-  const dir = process.env["OOM_DUMP_DIR"] || "/tmp";
-  setInterval(async () => {
-    const mem = process.memoryUsage();
-    console.log("Current Memory Usage", mem);
-    if (mem.heapTotal > memorySize * 1024 * 1024 * 1024 && !dumping) {
-      const file = join(dir, `${Date.now()}.heapsnapshot`);
-      dumping = true;
-      await dumpHeap(file);
-      process.exit(11);
-    }
-  }, 1e3 * 60);
-}
 async function dumpHeap(file) {
   console.log("Heap dumping to", file);
   const session = new Session();
@@ -19801,10 +19843,14 @@ async function dumpHeap(file) {
 function shutdownServers(exitCode) {
   server.forceShutdown();
   console.log("RPC server shut down");
-  httpServer.close(function() {
-    console.log("Http server shut down");
+  if (httpServer) {
+    httpServer.close(function() {
+      console.log("Http server shut down");
+      process.exit(exitCode);
+    });
+  } else {
     process.exit(exitCode);
-  });
+  }
 }
 import("node:process").then((p) => p.stdout.write(""));
 //# sourceMappingURL=processor-runner.js.map