@forzalabs/remora 1.2.8 → 1.2.10

package/index.js

@@ -10293,7 +10293,7 @@ var require_node2 = __commonJS({
 var require_tail_file = __commonJS({
   "../../packages/logger/node_modules/winston/lib/winston/tail-file.js"(exports2, module2) {
     "use strict";
-    var fs25 = require("fs");
+    var fs24 = require("fs");
     var { StringDecoder } = require("string_decoder");
     var { Stream } = require_readable();
     function noop() {
@@ -10314,7 +10314,7 @@ var require_tail_file = __commonJS({
         stream.emit("end");
         stream.emit("close");
       };
-      fs25.open(options.file, "a+", "0644", (err2, fd) => {
+      fs24.open(options.file, "a+", "0644", (err2, fd) => {
         if (err2) {
           if (!iter) {
             stream.emit("error", err2);
@@ -10326,10 +10326,10 @@ var require_tail_file = __commonJS({
         }
         (function read() {
           if (stream.destroyed) {
-            fs25.close(fd, noop);
+            fs24.close(fd, noop);
             return;
           }
-          return fs25.read(fd, buffer, 0, buffer.length, pos, (error, bytes) => {
+          return fs24.read(fd, buffer, 0, buffer.length, pos, (error, bytes) => {
             if (error) {
               if (!iter) {
                 stream.emit("error", error);
@@ -10388,7 +10388,7 @@ var require_tail_file = __commonJS({
 var require_file = __commonJS({
   "../../packages/logger/node_modules/winston/lib/winston/transports/file.js"(exports2, module2) {
     "use strict";
-    var fs25 = require("fs");
+    var fs24 = require("fs");
     var path24 = require("path");
     var asyncSeries = require_series();
     var zlib2 = require("zlib");
@@ -10593,7 +10593,7 @@ var require_file = __commonJS({
         let buff = "";
         let results = [];
         let row = 0;
-        const stream = fs25.createReadStream(file, {
+        const stream = fs24.createReadStream(file, {
           encoding: "utf8"
         });
         stream.on("error", (err2) => {
@@ -10745,7 +10745,7 @@ var require_file = __commonJS({
       stat(callback) {
         const target = this._getFile();
         const fullpath = path24.join(this.dirname, target);
-        fs25.stat(fullpath, (err2, stat) => {
+        fs24.stat(fullpath, (err2, stat) => {
           if (err2 && err2.code === "ENOENT") {
             debug("ENOENT\xA0ok", fullpath);
             this.filename = target;
@@ -10850,7 +10850,7 @@ var require_file = __commonJS({
       _createStream(source) {
         const fullpath = path24.join(this.dirname, this.filename);
         debug("create stream start", fullpath, this.options);
-        const dest = fs25.createWriteStream(fullpath, this.options).on("error", (err2) => debug(err2)).on("close", () => debug("close", dest.path, dest.bytesWritten)).on("open", () => {
+        const dest = fs24.createWriteStream(fullpath, this.options).on("error", (err2) => debug(err2)).on("close", () => debug("close", dest.path, dest.bytesWritten)).on("open", () => {
           debug("file open ok", fullpath);
           this.emit("open", fullpath);
           source.pipe(dest);
@@ -10929,7 +10929,7 @@ var require_file = __commonJS({
         const isZipped = this.zippedArchive ? ".gz" : "";
         const filePath = `${basename}${isOldest}${ext}${isZipped}`;
         const target = path24.join(this.dirname, filePath);
-        fs25.unlink(target, callback);
+        fs24.unlink(target, callback);
       }
       /**
        * Roll files forward based on integer, up to maxFiles. e.g. if base if
@@ -10952,17 +10952,17 @@ var require_file = __commonJS({
           tasks.push(function(i, cb) {
             let fileName = `${basename}${i - 1}${ext}${isZipped}`;
             const tmppath = path24.join(this.dirname, fileName);
-            fs25.exists(tmppath, (exists) => {
+            fs24.exists(tmppath, (exists) => {
               if (!exists) {
                 return cb(null);
               }
               fileName = `${basename}${i}${ext}${isZipped}`;
-              fs25.rename(tmppath, path24.join(this.dirname, fileName), cb);
+              fs24.rename(tmppath, path24.join(this.dirname, fileName), cb);
             });
           }.bind(this, x));
         }
         asyncSeries(tasks, () => {
-          fs25.rename(
+          fs24.rename(
             path24.join(this.dirname, `${basename}${ext}${isZipped}`),
             path24.join(this.dirname, `${basename}1${ext}${isZipped}`),
             callback
@@ -10978,22 +10978,22 @@ var require_file = __commonJS({
        * @private
        */
       _compressFile(src, dest, callback) {
-        fs25.access(src, fs25.F_OK, (err2) => {
+        fs24.access(src, fs24.F_OK, (err2) => {
           if (err2) {
             return callback();
           }
           var gzip = zlib2.createGzip();
-          var inp = fs25.createReadStream(src);
-          var out = fs25.createWriteStream(dest);
+          var inp = fs24.createReadStream(src);
+          var out = fs24.createWriteStream(dest);
           out.on("finish", () => {
-            fs25.unlink(src, callback);
+            fs24.unlink(src, callback);
           });
           inp.pipe(gzip).pipe(out);
         });
       }
       _createLogDirIfNotExist(dirPath) {
-        if (!fs25.existsSync(dirPath)) {
-          fs25.mkdirSync(dirPath, { recursive: true });
+        if (!fs24.existsSync(dirPath)) {
+          fs24.mkdirSync(dirPath, { recursive: true });
         }
       }
     };
@@ -18744,25 +18744,6 @@ var ProcessENVManagerClass = class {
 var ProcessENVManager = new ProcessENVManagerClass();
 var ProcessENVManager_default = ProcessENVManager;
 
-// ../../packages/common/src/SecretManager.ts
-var SecretManagerClass = class {
-  constructor() {
-    /**
-     * If the value is a secret (or .env setting), replace it with the value inside the .env configuration file
-     * Starts with "{" and ends with "}".
-     * e.g. {AWS_ID}
-     */
-    this.replaceSecret = (value) => {
-      if (!value || value.length <= 2 || !value.startsWith("{") || !value.endsWith("}"))
-        return value;
-      const parsedValue = value.slice(1, value.length - 1);
-      return ProcessENVManager_default.getEnvVariable(parsedValue);
-    };
-  }
-};
-var SecretManager = new SecretManagerClass();
-var SecretManager_default = SecretManager;
-
 // ../../packages/common/src/ExecutorScope.ts
 var import_path3 = __toESM(require("path"), 1);
 var import_fs3 = __toESM(require("fs"), 1);
@@ -18770,7 +18751,7 @@ var import_promises = __toESM(require("fs/promises"), 1);
 
 // ../../packages/constants/src/Constants.ts
 var CONSTANTS = {
-  cliVersion: "1.2.8",
+  cliVersion: "1.2.10",
   backendVersion: 1,
   backendPort: 5088,
   workerVersion: 2,
@@ -18816,10 +18797,10 @@ var ExecutorScopeClass = class {
   constructor() {
     this.WORKERS_FOLDER = "workers";
     this.PRODUCERS_FOLDER = "producers";
+    this.getBasePath = () => import_path3.default.join(Constants_default.defaults.REMORA_PATH, Constants_default.defaults.PRODUCER_TEMP_FOLDER);
     this.getWorkerPath = (scope, workerId) => {
       return import_path3.default.join(
-        Constants_default.defaults.REMORA_PATH,
-        Constants_default.defaults.PRODUCER_TEMP_FOLDER,
+        this.getBasePath(),
         // A specific execution sits entirely in this folder, so at the end it's safe to delete it entirely
         scope.folder,
         this.WORKERS_FOLDER,
@@ -18828,8 +18809,7 @@ var ExecutorScopeClass = class {
     };
     this.getProducerPath = (scope, producer, sourceFileKey) => {
       return import_path3.default.join(
-        Constants_default.defaults.REMORA_PATH,
-        Constants_default.defaults.PRODUCER_TEMP_FOLDER,
+        this.getBasePath(),
         // A specific execution sits entirely in this folder, so at the end it's safe to delete it entirely
         scope.folder,
         this.PRODUCERS_FOLDER,
@@ -18839,22 +18819,30 @@ var ExecutorScopeClass = class {
     };
     this.getMainPath = (scope) => {
       return import_path3.default.join(
-        Constants_default.defaults.REMORA_PATH,
-        Constants_default.defaults.PRODUCER_TEMP_FOLDER,
+        this.getBasePath(),
         scope.folder,
         "main.dataset"
       );
     };
     this.clearScope = async (scope) => {
       const scopePath = import_path3.default.join(
-        Constants_default.defaults.REMORA_PATH,
-        Constants_default.defaults.PRODUCER_TEMP_FOLDER,
+        this.getBasePath(),
         scope.folder
       );
       if (import_fs3.default.existsSync(scopePath)) {
         await import_promises.default.rm(scopePath, { recursive: true, force: true });
       }
     };
+    this.deepClear = () => {
+      const basePath = this.getBasePath();
+      const openScopes = this.getOpenScopes();
+      for (const scopeFolder of openScopes) {
+        const scopePath = import_path3.default.join(basePath, scopeFolder);
+        if (import_fs3.default.existsSync(scopePath)) {
+          import_fs3.default.rmSync(scopePath, { recursive: true, force: true });
+        }
+      }
+    };
     this.ensurePath = (fileUri) => {
       const dir = import_path3.default.dirname(fileUri);
       if (!import_fs3.default.existsSync(dir))
@@ -18862,11 +18850,106 @@ var ExecutorScopeClass = class {
       if (!import_fs3.default.existsSync(fileUri))
         import_fs3.default.writeFileSync(fileUri, "");
     };
+    this.getOpenScopes = () => {
+      const basePath = this.getBasePath();
+      if (!import_fs3.default.existsSync(basePath))
+        return [];
+      return import_fs3.default.readdirSync(basePath, { withFileTypes: true }).filter((entry) => entry.isDirectory()).map((entry) => entry.name).filter((folder) => folder !== "logs" && folder !== "usage");
+    };
   }
 };
 var ExecutorScope = new ExecutorScopeClass();
 var ExecutorScope_default = ExecutorScope;
 
+// ../../packages/common/src/ProcessShutdownManager.ts
+var ProcessShutdownManagerClass = class {
+  constructor() {
+    this._initialized = false;
+    this._cleaned = false;
+    this._runtimeName = "Remora process";
+    this.init = (runtimeName) => {
+      if (this._initialized)
+        return;
+      this._initialized = true;
+      if (runtimeName)
+        this._runtimeName = runtimeName;
+      process.once("SIGINT", () => this.handleSignal("SIGINT", 130));
+      process.once("SIGTERM", () => this.handleSignal("SIGTERM", 143));
+      process.once("uncaughtException", (error) => this.handleUnexpectedShutdown("uncaughtException", error));
+      process.once("unhandledRejection", (reason) => this.handleUnexpectedShutdown("unhandledRejection", reason));
+      process.once("beforeExit", (code) => this.handleBeforeExit(code));
+      process.once("exit", (code) => this.handleExit(code));
+    };
+    this.handleSignal = (signal, exitCode) => {
+      this.setShutdownState("intentional", signal);
+      Logger_default.warn(`Received ${signal}. Shutting down ${this._runtimeName}.`);
+      process.exit(exitCode);
+    };
+    this.handleUnexpectedShutdown = (reason, error) => {
+      this.setShutdownState("unintentional", reason);
+      Logger_default.error(this.asError(reason, error));
+      process.exit(1);
+    };
+    this.handleBeforeExit = (code) => {
+      this.setShutdownState(code === 0 ? "intentional" : "unintentional", `beforeExit:${code}`);
+    };
+    this.handleExit = (code) => {
+      if (!this._shutdownState)
+        this.setShutdownState(code === 0 ? "intentional" : "unintentional", `exit:${code}`);
+      this.cleanupOpenScopes(code);
+    };
+    this.cleanupOpenScopes = (code) => {
+      if (this._cleaned)
+        return;
+      this._cleaned = true;
+      const openScopes = ExecutorScope_default.getOpenScopes();
+      const scopeCount = openScopes.length;
+      const shutdownState = this._shutdownState;
+      const shutdownDescription = `${shutdownState?.type ?? "intentional"} shutdown (${shutdownState?.reason ?? `exit:${code}`})`;
+      if (scopeCount === 0) {
+        Logger_default.info(`Detected ${shutdownDescription} for ${this._runtimeName}. No executor scopes to clean up.`);
+        return;
+      }
+      Logger_default.warn(`Detected ${shutdownDescription} for ${this._runtimeName}. Cleaning up ${scopeCount} executor scope${scopeCount === 1 ? "" : "s"}.`);
+      ExecutorScope_default.deepClear();
+    };
+    this.setShutdownState = (type, reason) => {
+      if (this._shutdownState)
+        return;
+      this._shutdownState = { type, reason };
+    };
+    this.asError = (reason, error) => {
+      if (error instanceof Error) {
+        const contextualError = new Error(`Unexpected ${reason} during ${this._runtimeName} execution: ${error.message}`);
+        contextualError.stack = error.stack;
+        return contextualError;
+      }
+      return new Error(`Unexpected ${reason} during ${this._runtimeName} execution: ${String(error)}`);
+    };
+  }
+};
+var ProcessShutdownManager = new ProcessShutdownManagerClass();
+var ProcessShutdownManager_default = ProcessShutdownManager;
+
+// ../../packages/common/src/SecretManager.ts
+var SecretManagerClass = class {
+  constructor() {
+    /**
+     * If the value is a secret (or .env setting), replace it with the value inside the .env configuration file
+     * Starts with "{" and ends with "}".
+     * e.g. {AWS_ID}
+     */
+    this.replaceSecret = (value) => {
+      if (!value || value.length <= 2 || !value.startsWith("{") || !value.endsWith("}"))
+        return value;
+      const parsedValue = value.slice(1, value.length - 1);
+      return ProcessENVManager_default.getEnvVariable(parsedValue);
+    };
+  }
+};
+var SecretManager = new SecretManagerClass();
+var SecretManager_default = SecretManager;
+
 // ../../packages/common/src/Environment.ts
 var import_fs5 = __toESM(require("fs"), 1);
 var import_crypto = __toESM(require("crypto"), 1);
@@ -19952,7 +20035,7 @@ var DeltaShareSourceDriver = class {
     this.readAll = async (request) => {
       Affirm_default(request, "Invalid download request");
       const table = this._resolveTable(request.fileKey);
-      const deltaFiles = await this._getAllFilesInTable(table);
+      const deltaFiles = await this._getAllFilesInTable(table, request.disableHistory);
       const hyparquet = await import("hyparquet");
       const lines = [];
       for (const deltaFile of deltaFiles) {
@@ -19966,7 +20049,7 @@ var DeltaShareSourceDriver = class {
       Affirm_default(request.options, "Invalid read options");
       Affirm_default(request.options.lineFrom !== void 0 && request.options.lineTo !== void 0, "Missing read range");
       const table = this._resolveTable(request.fileKey);
-      const deltaFiles = await this._getAllFilesInTable(table);
+      const deltaFiles = await this._getAllFilesInTable(table, request.disableHistory);
       const hyparquet = await import("hyparquet");
       const { options: { lineFrom, lineTo } } = request;
       const lines = [];
@@ -20067,9 +20150,9 @@ var DeltaShareSourceDriver = class {
 `);
       return true;
     };
-    this._getAllFilesInTable = async (table) => {
+    this._getAllFilesInTable = async (table, disableHistory = false) => {
       const url = this._getTableUrl(this._query, table);
-      const body = {
+      const body = disableHistory ? {} : {
         version: await this._getVersion(table)
       };
       const res = await fetch(url, {
@@ -20091,7 +20174,7 @@ var DeltaShareSourceDriver = class {
       Affirm_default(producer, "Invalid producer");
       Affirm_default(scope, "Invalid executor scope");
       const table = this._resolveTable(producer.settings.fileKey);
-      const deltaFiles = await this._getAllFilesInTable(table);
+      const deltaFiles = await this._getAllFilesInTable(table, producer.settings.disableHistory);
       const hyparquet = await import("hyparquet");
       const delimiter = producer.settings.delimiter ?? ",";
       const files = [];
@@ -20514,6 +20597,19 @@ var Helper = {
 };
 var Helper_default = Helper;
 
+// ../../packages/helper/src/Formatter.ts
+var Formatter = {
+  bytes: (bytes, decimals = 2) => {
+    if (!+bytes) return "0 Bytes";
+    const k = 1024;
+    const dm = decimals < 0 ? 0 : decimals;
+    const sizes = ["Bytes", "KB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB"];
+    const i = Math.floor(Math.log(bytes) / Math.log(k));
+    return `${parseFloat((bytes / Math.pow(k, i)).toFixed(dm))} ${sizes[i]}`;
+  }
+};
+var Formatter_default = Formatter;
+
 // ../../packages/helper/src/Settings.ts
 var SETTINGS = {
   db: {
@@ -22172,10 +22268,10 @@ var LicenceManager = new LicenceManagerClass();
 var LicenceManager_default = LicenceManager;
 
 // ../../packages/executors/src/ConsumerExecutor.ts
-var import_path19 = __toESM(require("path"));
-var import_fs13 = __toESM(require("fs"));
+var import_path18 = __toESM(require("path"));
+var import_fs12 = __toESM(require("fs"));
 var import_readline6 = __toESM(require("readline"));
-var import_promises9 = __toESM(require("fs/promises"));
+var import_promises8 = __toESM(require("fs/promises"));
 var import_crypto5 = __toESM(require("crypto"));
 
 // ../../packages/engines/src/CryptoEngine.ts
@@ -22700,6 +22796,7 @@ var ProducerEngineClass = class {
             fileKey,
             fileType: effectiveFileType,
             options: { lineFrom: options.lines.from, lineTo: options.lines.to, sheetName, hasHeaderRow, startRow, startColumn },
+            disableHistory: producer.settings?.disableHistory,
             httpApi: producer.settings?.httpApi
           });
           break;
@@ -22708,6 +22805,7 @@ var ProducerEngineClass = class {
             fileKey,
             fileType: effectiveFileType,
             options: { sheetName, hasHeaderRow, startRow, startColumn },
+            disableHistory: producer.settings?.disableHistory,
             httpApi: producer.settings?.httpApi
           });
           break;
@@ -24601,69 +24699,8 @@ var UsageManager = new UsageManagerClass();
 var UsageManager_default = UsageManager;
 
 // ../../packages/executors/src/OutputExecutor.ts
-var fs18 = __toESM(require("fs"));
-
-// ../../packages/executors/src/ExecutorScope.ts
+var fs17 = __toESM(require("fs"));
 var import_path17 = __toESM(require("path"));
-var import_fs12 = __toESM(require("fs"));
-var import_promises8 = __toESM(require("fs/promises"));
-var ExecutorScopeClass2 = class {
-  constructor() {
-    this.WORKERS_FOLDER = "workers";
-    this.PRODUCERS_FOLDER = "producers";
-    this.getWorkerPath = (scope, workerId) => {
-      return import_path17.default.join(
-        Constants_default.defaults.REMORA_PATH,
-        Constants_default.defaults.PRODUCER_TEMP_FOLDER,
-        // A specific execution sits entirely in this folder, so at the end it's safe to delete it entirely
-        scope.folder,
-        this.WORKERS_FOLDER,
-        `${workerId}.dataset`
-      );
-    };
-    this.getProducerPath = (scope, producer, sourceFileKey) => {
-      return import_path17.default.join(
-        Constants_default.defaults.REMORA_PATH,
-        Constants_default.defaults.PRODUCER_TEMP_FOLDER,
-        // A specific execution sits entirely in this folder, so at the end it's safe to delete it entirely
-        scope.folder,
-        this.PRODUCERS_FOLDER,
-        producer.name,
-        `${sourceFileKey}.dataset`
-      );
-    };
-    this.getMainPath = (scope) => {
-      return import_path17.default.join(
-        Constants_default.defaults.REMORA_PATH,
-        Constants_default.defaults.PRODUCER_TEMP_FOLDER,
-        scope.folder,
-        "main.dataset"
-      );
-    };
-    this.clearScope = async (scope) => {
-      const scopePath = import_path17.default.join(
-        Constants_default.defaults.REMORA_PATH,
-        Constants_default.defaults.PRODUCER_TEMP_FOLDER,
-        scope.folder
-      );
-      if (import_fs12.default.existsSync(scopePath)) {
-        await import_promises8.default.rm(scopePath, { recursive: true, force: true });
-      }
-    };
-    this.ensurePath = (fileUri) => {
-      const dir = import_path17.default.dirname(fileUri);
-      if (!import_fs12.default.existsSync(dir))
-        import_fs12.default.mkdirSync(dir, { recursive: true });
-      if (!import_fs12.default.existsSync(fileUri))
-        import_fs12.default.writeFileSync(fileUri, "");
-    };
-  }
-};
-var ExecutorScope2 = new ExecutorScopeClass2();
-var ExecutorScope_default2 = ExecutorScope2;
-
-// ../../packages/executors/src/OutputExecutor.ts
-var import_path18 = __toESM(require("path"));
 var OutputExecutorClass = class {
   constructor() {
     this._getInternalRecordFormat = (consumer) => {
@@ -24707,13 +24744,13 @@ var OutputExecutorClass = class {
       for (const output of consumer.outputs) {
         const destination = Environment_default.getSource(output.exportDestination);
         const driver = await DriverFactory_default.instantiateDestination(destination);
-        const currentPath = import_path18.default.dirname(ExecutorScope_default2.getMainPath(scope));
+        const currentPath = import_path17.default.dirname(ExecutorScope_default.getMainPath(scope));
         const destinationName = this._composeFileName(consumer, output, this._getExtension(output));
         Logger_default.log(`Exporting consumer "${consumer.name}" to "${output.exportDestination}" as ${output.format} (${destinationName})`);
-        const filenameArray = fs18.readdirSync(currentPath).filter((filename) => filename.includes(".dataset"));
+        const filenameArray = fs17.readdirSync(currentPath).filter((filename) => filename.includes(".dataset"));
         for (const filename in filenameArray) {
           const destinationPath = this.getCompletedPath(destinationName, filename);
-          const startingPath = import_path18.default.join(currentPath, filenameArray[filename]);
+          const startingPath = import_path17.default.join(currentPath, filenameArray[filename]);
           if (output.format === internalFormat) {
             results.push(await driver.move(startingPath, destinationPath));
           } else {
@@ -24773,31 +24810,31 @@ var OutputExecutor_default = OutputExecutor;
 var ConsumerExecutorClass = class {
   constructor() {
     this._getWorkPath = (consumer, executionId) => {
-      const execFolder = import_path19.default.join(consumer.name, executionId);
-      const workPath = import_path19.default.join("./remora", Constants_default.defaults.PRODUCER_TEMP_FOLDER, execFolder, ".dataset");
+      const execFolder = import_path18.default.join(consumer.name, executionId);
+      const workPath = import_path18.default.join("./remora", Constants_default.defaults.PRODUCER_TEMP_FOLDER, execFolder, ".dataset");
       return workPath;
     };
     this._clearWorkPath = async (workPath) => {
       try {
-        if (import_fs13.default.existsSync(workPath)) {
-          await import_promises9.default.unlink(workPath);
+        if (import_fs12.default.existsSync(workPath)) {
+          await import_promises8.default.unlink(workPath);
         }
       } catch (error) {
       }
       try {
-        const dir = import_path19.default.dirname(workPath);
-        if (import_fs13.default.existsSync(dir)) {
-          await import_promises9.default.rmdir(dir);
+        const dir = import_path18.default.dirname(workPath);
+        if (import_fs12.default.existsSync(dir)) {
+          await import_promises8.default.rmdir(dir);
         }
       } catch (error) {
       }
     };
     this._ensurePath = (pathUri) => {
-      const dir = import_path19.default.dirname(pathUri);
-      if (!import_fs13.default.existsSync(dir))
-        import_fs13.default.mkdirSync(dir, { recursive: true });
-      if (!import_fs13.default.existsSync(pathUri))
-        import_fs13.default.writeFileSync(pathUri, "");
+      const dir = import_path18.default.dirname(pathUri);
+      if (!import_fs12.default.existsSync(dir))
+        import_fs12.default.mkdirSync(dir, { recursive: true });
+      if (!import_fs12.default.existsSync(pathUri))
+        import_fs12.default.writeFileSync(pathUri, "");
     };
     this.processRecord = (options) => {
       const { consumer, fields, dimensions, producer, record, requestOptions, index: recordIndex } = options;
@@ -24927,10 +24964,10 @@ var ConsumerExecutorClass = class {
       return record;
     };
     this.processDistinct = async (datasetPath) => {
-      const reader = import_fs13.default.createReadStream(datasetPath);
+      const reader = import_fs12.default.createReadStream(datasetPath);
       const lineReader = import_readline6.default.createInterface({ input: reader, crlfDelay: Infinity });
       const tempWorkPath = datasetPath + "_tmp";
-      const writer = import_fs13.default.createWriteStream(tempWorkPath);
+      const writer = import_fs12.default.createWriteStream(tempWorkPath);
       const waitForDrain = () => new Promise((resolve) => writer.once("drain", resolve));
       let newLineCount = 0;
       const seen = /* @__PURE__ */ new Set();
@@ -24955,12 +24992,12 @@ var ConsumerExecutorClass = class {
           reader.destroy();
         });
       }
-      await import_promises9.default.unlink(datasetPath);
-      await import_promises9.default.rename(tempWorkPath, datasetPath);
+      await import_promises8.default.unlink(datasetPath);
+      await import_promises8.default.rename(tempWorkPath, datasetPath);
       return newLineCount;
     };
     this.processDistinctOn = async (consumer, datasetPath) => {
-      const reader = import_fs13.default.createReadStream(datasetPath);
+      const reader = import_fs12.default.createReadStream(datasetPath);
       const lineReader = import_readline6.default.createInterface({ input: reader, crlfDelay: Infinity });
       const { distinctOn } = consumer.options;
       const { keys, resolution } = distinctOn;
@@ -24983,7 +25020,7 @@ var ConsumerExecutorClass = class {
       }
       lineReader.close();
       const tempWorkPath = datasetPath + "_tmp";
-      const writer = import_fs13.default.createWriteStream(tempWorkPath);
+      const writer = import_fs12.default.createWriteStream(tempWorkPath);
       const waitForDrain = () => new Promise((resolve) => writer.once("drain", resolve));
       for (const { line } of winners.values()) {
         if (!writer.write(line + "\n"))
@@ -25000,8 +25037,8 @@ var ConsumerExecutorClass = class {
           reader.destroy();
         });
       }
-      await import_promises9.default.unlink(datasetPath);
-      await import_promises9.default.rename(tempWorkPath, datasetPath);
+      await import_promises8.default.unlink(datasetPath);
+      await import_promises8.default.rename(tempWorkPath, datasetPath);
       return winners.size;
     };
     this.processPivot = async (consumer, datasetPath) => {
@@ -25013,7 +25050,7 @@ var ConsumerExecutorClass = class {
       if (!pivotValues) {
         pivotValues = [];
         const discoverySet = /* @__PURE__ */ new Set();
-        const discoverReader = import_fs13.default.createReadStream(datasetPath);
+        const discoverReader = import_fs12.default.createReadStream(datasetPath);
         const discoverLineReader = import_readline6.default.createInterface({ input: discoverReader, crlfDelay: Infinity });
         for await (const line of discoverLineReader) {
           const record = this._parseLine(line, internalRecordFormat, internalFields);
@@ -25032,7 +25069,7 @@ var ConsumerExecutorClass = class {
         }
       }
       const groups = /* @__PURE__ */ new Map();
-      const reader = import_fs13.default.createReadStream(datasetPath);
+      const reader = import_fs12.default.createReadStream(datasetPath);
       const lineReader = import_readline6.default.createInterface({ input: reader, crlfDelay: Infinity });
       for await (const line of lineReader) {
         const record = this._parseLine(line, internalRecordFormat, internalFields);
@@ -25060,7 +25097,7 @@ var ConsumerExecutorClass = class {
         ...pivotValues.map((pv) => ({ cField: { key: columnPrefix + pv }, finalKey: columnPrefix + pv }))
       ];
       const tempWorkPath = datasetPath + "_tmp";
-      const writer = import_fs13.default.createWriteStream(tempWorkPath);
+      const writer = import_fs12.default.createWriteStream(tempWorkPath);
       const waitForDrain = () => new Promise((resolve) => writer.once("drain", resolve));
       let outputCount = 0;
       for (const { rowRecord, cells } of groups.values()) {
@@ -25106,8 +25143,8 @@ var ConsumerExecutorClass = class {
           reader.destroy();
         });
       }
-      await import_promises9.default.unlink(datasetPath);
-      await import_promises9.default.rename(tempWorkPath, datasetPath);
+      await import_promises8.default.unlink(datasetPath);
+      await import_promises8.default.rename(tempWorkPath, datasetPath);
       return outputCount;
     };
     this._parseLine = (line, format2, fields) => {
@@ -25153,7 +25190,7 @@ var ConsumerExecutorClass = class {
       for (const fieldKey of uniqueFieldKeys) {
         fieldValueSets.set(fieldKey, /* @__PURE__ */ new Set());
       }
-      const reader = import_fs13.default.createReadStream(datasetPath);
+      const reader = import_fs12.default.createReadStream(datasetPath);
       const lineReader = import_readline6.default.createInterface({ input: reader, crlfDelay: Infinity });
       for await (const line of lineReader) {
         rowCount++;
@@ -25201,7 +25238,7 @@ var ConsumerExecutor = new ConsumerExecutorClass();
 var ConsumerExecutor_default = ConsumerExecutor;
 
 // ../../packages/executors/src/ProducerExecutor.ts
-var import_path20 = __toESM(require("path"));
+var import_path19 = __toESM(require("path"));
 var ProducerExecutorClass = class {
   constructor() {
     this.ready = async (producer, scope) => {
@@ -25227,7 +25264,7 @@ var ProducerExecutorClass = class {
       counter = performance.now();
       for (const dimension of dimensions) {
         if (dimension.prodDimension.sourceFilename === true)
-          record[dimension.name] = import_path20.default.basename(chunk.fileUri);
+          record[dimension.name] = import_path19.default.basename(chunk.fileUri);
         const maskType = ProducerManager_default.getMask(dimension.prodDimension);
         if (Algo_default.hasVal(maskType))
           record[dimension.name] = CryptoEngine_default.hashValue(maskType, record[dimension.name]?.toString(), dimension.prodDimension.type);
@@ -25258,14 +25295,13 @@ var ExecutorPerformance = class {
 var ExecutorPerformance_default = ExecutorPerformance;
 
 // ../../packages/executors/src/ExecutorOrchestrator.ts
-var import_os = __toESM(require("os"));
-var import_fs14 = __toESM(require("fs"));
-var import_promises10 = __toESM(require("fs/promises"));
+var import_fs13 = __toESM(require("fs"));
+var import_promises9 = __toESM(require("fs/promises"));
 var import_path21 = __toESM(require("path"));
 var import_workerpool = __toESM(require("workerpool"));
 
 // ../../packages/executors/src/ExecutorWriter.ts
-var fs20 = __toESM(require("fs"));
+var fs19 = __toESM(require("fs"));
 var import_readline7 = __toESM(require("readline"));
 var ExecutorWriter = class {
   constructor() {
@@ -25282,11 +25318,11 @@ var ExecutorWriter = class {
     };
     this.splitBySize = async (scope, sourcePath) => {
       const maxOutputFileSize = scope.limitFileSize * this.FAKE_GB;
-      const readStream = fs20.createReadStream(sourcePath);
+      const readStream = fs19.createReadStream(sourcePath);
       const reader = import_readline7.default.createInterface({ input: readStream, crlfDelay: Infinity });
       let writerIndex = 0;
       let destPath = this.getCompletedPath(sourcePath, writerIndex);
-      let writeStream = fs20.createWriteStream(destPath, { flags: "a" });
+      let writeStream = fs19.createWriteStream(destPath, { flags: "a" });
       const waitForDrain = () => new Promise((resolve) => writeStream.once("drain", resolve));
       for await (const line of reader) {
         if (readStream.bytesRead > maxOutputFileSize * (writerIndex + 1)) {
@@ -25297,7 +25333,7 @@ var ExecutorWriter = class {
           });
           writerIndex++;
           destPath = this.getCompletedPath(sourcePath, writerIndex);
-          writeStream = fs20.createWriteStream(destPath, { flags: "a" });
+          writeStream = fs19.createWriteStream(destPath, { flags: "a" });
         }
         if (!writeStream.write(line + "\n"))
           await waitForDrain();
@@ -25307,7 +25343,7 @@ var ExecutorWriter = class {
         writeStream.on("finish", resolve);
         writeStream.on("error", reject);
       });
-      await fs20.promises.unlink(sourcePath);
+      await fs19.promises.unlink(sourcePath);
     };
     /**
      * Manage the Writestream for main.dataset
@@ -25364,7 +25400,7 @@ var ExecutorWriter = class {
 var ExecutorWriter_default = ExecutorWriter;
 
 // ../../packages/executors/src/ExecutorOrchestrator.ts
-var import_promises11 = require("stream/promises");
+var import_promises10 = require("stream/promises");
 
 // ../../packages/executors/src/cli_progress/ExecutorProgress2.ts
 var ExecutorProgress2 = class {
@@ -25400,19 +25436,111 @@ var ExecutorProgress2 = class {
 };
 var ExecutorProgress2_default = ExecutorProgress2;
 
+// ../../packages/executors/src/OrchestratorHelper.ts
+var import_os = __toESM(require("os"));
+var import_path20 = __toESM(require("path"));
+var OrchestratorHelper = {
+  getMemoryUsage: () => {
+    const processMemory = process.memoryUsage();
+    const freeSystemMemory = import_os.default.freemem();
+    return {
+      /**
+       * resident set size (heap + code + stack)
+       */
+      rss: Formatter_default.bytes(processMemory.rss),
+      heapUsed: Formatter_default.bytes(processMemory.heapUsed),
+      heapTotal: Formatter_default.bytes(processMemory.heapTotal),
+      heapPercent: Algo_default.round(processMemory.heapUsed / processMemory.heapTotal, 1),
+      external: Formatter_default.bytes(processMemory.external),
+      free: Formatter_default.bytes(freeSystemMemory)
+    };
+  },
+  formatMemoryUsage: () => {
+    return `Memory [total: ${OrchestratorHelper.getMemoryUsage().rss} - heap: ${OrchestratorHelper.getMemoryUsage().heapPercent}%]`;
+  },
+  computeFinalResult: (tracker, executorResults, executionId, resultUri) => {
+    const result = {
+      cycles: Algo_default.max(executorResults.map((x) => x.cycles)),
+      elapsedMS: Algo_default.sum(executorResults.map((x) => x.elapsedMS)),
+      inputCount: Algo_default.sum(executorResults.map((x) => x.inputCount)),
+      outputCount: Algo_default.sum(executorResults.map((x) => x.outputCount)),
+      workerCount: executorResults.length,
+      executionId,
+      resultUri,
+      operations: {}
+    };
+    for (const res of executorResults) {
+      for (const opKey of Object.keys(res.operations)) {
+        const op = res.operations[opKey];
+        let label = result.operations[opKey];
+        if (!label) {
+          result.operations[opKey] = { avg: -1, max: -1, min: -1, elapsedMS: [] };
+          label = result.operations[opKey];
+        }
+        label.elapsedMS.push(op.elapsedMS);
+      }
+      for (const opKey of Object.keys(result.operations)) {
+        const operation = result.operations[opKey];
+        if (operation.elapsedMS.length > 0) {
+          operation.min = Math.min(...operation.elapsedMS);
+          operation.max = Math.max(...operation.elapsedMS);
+          operation.avg = Algo_default.mean(operation.elapsedMS);
+        }
+      }
+    }
+    const trackerOperations = tracker.getOperations();
+    for (const opKey of Object.keys(trackerOperations)) {
+      const trackerOp = trackerOperations[opKey];
+      const value = trackerOp.elapsedMS;
+      if (!result.operations[opKey]) {
+        result.operations[opKey] = { avg: value, max: value, min: value, elapsedMS: [] };
+      }
+      result.operations[opKey].elapsedMS.push(value);
+    }
+    return result;
+  },
+  /**
+   * Returns the path to the worker thread file in the build (different between dev, cli and prod (docker)).
+   * IMPORTANT!: when moving this (OrchestratorHelper.ts) file, or the workers output, you have to make sure to update these paths
+   */
+  getPhysicalWorkerPath: () => {
+    const currentDir = __dirname;
+    if (ProcessENVManager_default.getEnvVariable("NODE_ENV") === "dev" || ProcessENVManager_default.getEnvVariable("NODE_ENV") === "development")
+      return import_path20.default.resolve("./.build/workers");
+    const forcedPath = ProcessENVManager_default.getEnvVariable("REMORA_WORKERS_PATH");
+    if (forcedPath && forcedPath.length > 0)
+      return import_path20.default.join(__dirname, forcedPath);
+    if (!currentDir.includes(".build")) {
+      return import_path20.default.join(__dirname, "../workers");
+    } else {
+      return import_path20.default.resolve("./.build/workers");
+    }
+  },
+  getParallelWorkerCount: () => {
+    const cpuBoundWorkers = Math.max(1, Math.floor(import_os.default.cpus().length * 0.7));
+    const totalMemoryMB = Math.floor(import_os.default.totalmem() / (1024 * 1024));
+    const reservedMemoryMB = Constants_default.defaults.MIN_RUNTIME_HEAP_MB * 2;
+    const availableMemoryForWorkersMB = Math.max(Constants_default.defaults.MIN_RUNTIME_HEAP_MB, totalMemoryMB - reservedMemoryMB);
+    const memoryBoundWorkers = Math.max(1, Math.floor(availableMemoryForWorkersMB / Constants_default.defaults.MIN_RUNTIME_HEAP_MB));
+    return Math.min(cpuBoundWorkers, Constants_default.defaults.MAX_THREAD_COUNT, memoryBoundWorkers);
+  }
+};
+var OrchestratorHelper_default = OrchestratorHelper;
+
 // ../../packages/executors/src/ExecutorOrchestrator.ts
 var ExecutorOrchestratorClass = class {
   constructor() {
-    this.createPool = () => {
+    this.createPool = (maxWorkers) => {
       const options = {
+        maxWorkers,
         workerThreadOpts: {
           resourceLimits: {
             maxOldGenerationSizeMb: Constants_default.defaults.MIN_RUNTIME_HEAP_MB
           }
         }
       };
-      const workerPath = this._getWorkerPath();
-      Logger_default.log(`Initializing worker pool from ${workerPath} (heap limit: ${Constants_default.defaults.MIN_RUNTIME_HEAP_MB}MB)`);
+      const workerPath = OrchestratorHelper_default.getPhysicalWorkerPath();
+      Logger_default.log(`Initializing worker pool from ${workerPath} (workers: ${maxWorkers}, heap limit: ${Constants_default.defaults.MIN_RUNTIME_HEAP_MB}MB) | ${OrchestratorHelper_default.formatMemoryUsage()}`);
       return import_workerpool.default.pool(import_path21.default.join(workerPath, "ExecutorWorker.js"), options);
     };
     this.launch = async (request) => {
@@ -25426,11 +25554,11 @@ var ExecutorOrchestratorClass = class {
       const _progress = new ExecutorProgress2_default(logProgress);
       const { usageId } = UsageManager_default.startUsage(consumer, details);
       const scope = { id: usageId, folder: `${consumer.name}_${usageId}`, workersId: [], limitFileSize: consumer.maximumFileSize };
-      const pool = this.createPool();
+      let activePool = null;
       try {
         const start = performance.now();
         const executorResults = [];
-        Logger_default.log(`[${usageId}] Launching consumer "${consumer.name}" (invoked by: ${details.invokedBy}, user: ${details.user?.name ?? "unknown"}, producer(s): ${consumer.producers.length})`);
+        Logger_default.log(`[${usageId}] Launching consumer "${consumer.name}" (invoked by: ${details.invokedBy}, user: ${details.user?.name ?? "unknown"}, producer(s): ${consumer.producers.length}) | ${OrchestratorHelper_default.formatMemoryUsage()}`);
         let counter = performance.now();
         _progress.update({ phase: "Preparing source data", progress: 0 });
         let sourceFilesByProducer = await this.readySourceFiles(consumer, scope);
@@ -25446,10 +25574,10 @@ var ExecutorOrchestratorClass = class {
         let globalWorkerIndex = 0;
         for (const pair of sourceFilesByProducer) {
           const { prod, cProd, response } = pair;
-          if (!import_fs14.default.existsSync(response.files[0].fullUri)) {
-            if (!cProd.isOptional)
+          if (!import_fs13.default.existsSync(response.files[0].fullUri)) {
+            if (!cProd.isOptional) {
               throw new Error(`Expected data file ${response.files[0].fullUri} of producer ${prod.name} in consumer ${consumer.name} is missing.`);
-            else if (cProd.isOptional === true) {
+            } else if (cProd.isOptional === true) {
               Logger_default.log(`[${usageId}] Producer "${prod.name}": data file missing but marked optional, skipping`);
               continue;
             }
@@ -25462,35 +25590,40 @@ var ExecutorOrchestratorClass = class {
           for (const [fileIndex, file] of response.files.entries()) {
             const chunks = ExecutorOrchestrator.scopeWork(file.fullUri);
             const workerThreads = [];
-            Logger_default.log(`[${usageId}] Producer "${prod.name}" file ${fileIndex + 1}/${totalFiles}: split into ${chunks.length} chunk(s)`);
-            for (const chunk of chunks) {
-              const workerId = `${usageId}_${globalWorkerIndex}`;
-              globalWorkerIndex++;
-              const workerData = {
-                producer: prod,
-                chunk,
-                consumer,
-                prodDimensions,
-                workerId,
-                scope,
-                options,
-                loggerConfig: Logger_default.getConfig()
-              };
-              scope.workersId.push(workerId);
-              Logger_default.log(`[${usageId}] Spawning worker ${workerId} for producer "${prod.name}" \u2014 chunk ${chunk.start}-${chunk.end} (${Math.round((chunk.end - chunk.start) / 1024)}KB)`);
-              workerThreads.push(pool.exec("executor", [workerData], {
-                on: (payload) => this.onWorkAdvanced(payload, workerId, _progress, totalBytesToProcess, bytesProcessedByWorker)
-              }).catch((error) => {
-                Logger_default.error(error);
-                return null;
-              }));
+            activePool = this.createPool(chunks.length);
+            Logger_default.log(`[${usageId}] Producer "${prod.name}" file ${fileIndex + 1}/${totalFiles}: split into ${chunks.length} chunk(s) | ${OrchestratorHelper_default.formatMemoryUsage()}`);
+            try {
+              for (const chunk of chunks) {
+                const workerId = `${usageId}_${globalWorkerIndex}`;
+                globalWorkerIndex++;
+                const workerData = {
+                  producer: prod,
+                  chunk,
+                  consumer,
+                  prodDimensions,
+                  workerId,
+                  scope,
+                  options,
+                  loggerConfig: Logger_default.getConfig()
+                };
+                scope.workersId.push(workerId);
+                Logger_default.log(`[${usageId}] Spawning worker ${workerId} for producer "${prod.name}" \u2014 chunk ${chunk.start}-${chunk.end} (${Math.round((chunk.end - chunk.start) / 1024)}KB)`);
+                workerThreads.push(activePool.exec("executor", [workerData], {
+                  on: (payload) => this.onWorkAdvanced(payload, workerId, _progress, totalBytesToProcess, bytesProcessedByWorker)
+                }).catch((error) => {
+                  Logger_default.error(error);
+                  return null;
+                }));
+              }
+              Logger_default.log(`[${usageId}] Waiting for ${workerThreads.length} worker(s) to complete  | ${OrchestratorHelper_default.formatMemoryUsage()}`);
+              executorResults.push(...await Promise.all(workerThreads));
+              Logger_default.log(`[${usageId}] All ${workerThreads.length} worker(s) finished for producer "${prod.name}" file ${fileIndex + 1}/${totalFiles} | ${OrchestratorHelper_default.formatMemoryUsage()}`);
+            } finally {
+              await activePool.terminate();
+              activePool = null;
             }
-            Logger_default.log(`[${usageId}] Waiting for ${workerThreads.length} worker(s) to complete`);
-            executorResults.push(...await Promise.all(workerThreads));
-            Logger_default.log(`[${usageId}] All ${workerThreads.length} worker(s) finished for producer "${prod.name}" file ${fileIndex + 1}/${totalFiles}`);
           }
         }
-        await pool.terminate();
         _progress.update({ phase: "Processing data", progress: 1 });
         if (executorResults.some((x) => !Algo_default.hasVal(x)))
           throw new Error(`${executorResults.filter((x) => !Algo_default.hasVal(x)).length} worker(s) failed to produce valid results`);
@@ -25505,7 +25638,7 @@ var ExecutorOrchestratorClass = class {
           if (consumer.options?.distinct === true) {
             Logger_default.log(`[${usageId}] Running unified distinct pass across merged workers`);
             counter = performance.now();
-            const unifiedOutputCount = await ConsumerExecutor_default.processDistinct(ExecutorScope_default2.getMainPath(scope));
+            const unifiedOutputCount = await ConsumerExecutor_default.processDistinct(ExecutorScope_default.getMainPath(scope));
             tracker.measure("process-distinct:main", performance.now() - counter);
             postOperation.totalOutputCount = unifiedOutputCount;
             Logger_default.log(`[${usageId}] Distinct pass complete: ${unifiedOutputCount} unique rows in ${Math.round(performance.now() - counter)}ms`);
@@ -25513,7 +25646,7 @@ var ExecutorOrchestratorClass = class {
           if (consumer.options?.distinctOn) {
             Logger_default.log(`[${usageId}] Running unified distinctOn pass (${consumer.options.distinctOn})`);
             counter = performance.now();
-            const unifiedOutputCount = await ConsumerExecutor_default.processDistinctOn(consumer, ExecutorScope_default2.getMainPath(scope));
+            const unifiedOutputCount = await ConsumerExecutor_default.processDistinctOn(consumer, ExecutorScope_default.getMainPath(scope));
             tracker.measure("process-distinct-on:main", performance.now() - counter);
             postOperation.totalOutputCount = unifiedOutputCount;
             Logger_default.log(`[${usageId}] DistinctOn pass complete: ${unifiedOutputCount} rows in ${Math.round(performance.now() - counter)}ms`);
@@ -25522,7 +25655,7 @@ var ExecutorOrchestratorClass = class {
         if (consumer.options?.pivot) {
           Logger_default.log(`[${usageId}] Running pivot operation`);
           counter = performance.now();
-          const unifiedOutputCount = await ConsumerExecutor_default.processPivot(consumer, ExecutorScope_default2.getMainPath(scope));
+          const unifiedOutputCount = await ConsumerExecutor_default.processPivot(consumer, ExecutorScope_default.getMainPath(scope));
           tracker.measure("process-pivot:main", performance.now() - counter);
           postOperation.totalOutputCount = unifiedOutputCount;
           Logger_default.log(`[${usageId}] Pivot complete: ${unifiedOutputCount} rows in ${Math.round(performance.now() - counter)}ms`);
@@ -25530,7 +25663,7 @@ var ExecutorOrchestratorClass = class {
         if (consumer.validate && consumer.validate.length > 0) {
           Logger_default.log(`[${usageId}] Running dataset-level validations`);
           counter = performance.now();
-          const validationResults = await ConsumerExecutor_default.processDatasetValidation(consumer, ExecutorScope_default2.getMainPath(scope));
+          const validationResults = await ConsumerExecutor_default.processDatasetValidation(consumer, ExecutorScope_default.getMainPath(scope));
           tracker.measure("dataset-validation", performance.now() - counter);
           for (const result of validationResults) {
             if (result.onFail === "fail") {
@@ -25547,7 +25680,7 @@ var ExecutorOrchestratorClass = class {
           Logger_default.log(`[${usageId}] Splitting output by size limit (${scope.limitFileSize})`);
           counter = performance.now();
           const writer = new ExecutorWriter_default();
-          await writer.splitBySize(scope, ExecutorScope_default2.getMainPath(scope));
+          await writer.splitBySize(scope, ExecutorScope_default.getMainPath(scope));
           tracker.measure("split-by-size", performance.now() - counter);
           Logger_default.log(`[${usageId}] Split complete in ${Math.round(performance.now() - counter)}ms`);
         }
@@ -25566,9 +25699,9 @@ var ExecutorOrchestratorClass = class {
           tracker.measure("on-success-actions", performance.now() - counter);
           Logger_default.log(`[${usageId}] On-success actions complete in ${Math.round(performance.now() - counter)}ms`);
         }
-        Logger_default.log(`[${usageId}] Starting cleanup operations`);
+        Logger_default.log(`[${usageId}] Starting cleanup operations | ${OrchestratorHelper_default.formatMemoryUsage()}`);
         await this.performCleanupOperations(scope, tracker);
-        const finalResult = this.computeFinalResult(tracker, executorResults, usageId, exportRes.key);
+        const finalResult = OrchestratorHelper_default.computeFinalResult(tracker, executorResults, usageId, exportRes.key);
         finalResult.elapsedMS = performance.now() - start;
         if (Algo_default.hasVal(postOperation.totalOutputCount))
           finalResult.outputCount = postOperation.totalOutputCount;
@@ -25577,9 +25710,10 @@ var ExecutorOrchestratorClass = class {
         await Logger_default.flush();
         return finalResult;
       } catch (error) {
-        Logger_default.log(`[${usageId}] Consumer "${consumer.name}" failed: ${Helper_default.asError(error).message}`);
+        Logger_default.log(`[${usageId}] Consumer "${consumer.name}" failed: ${Helper_default.asError(error).message} | ${OrchestratorHelper_default.formatMemoryUsage()}`);
         Logger_default.error(error);
-        await pool.terminate();
+        if (activePool)
+          await activePool.terminate();
         await ConsumerOnFinishManager_default.onConsumerError(consumer, usageId);
         Logger_default.log(`[${usageId}] Running cleanup after failure`);
         await this.performCleanupOperations(scope, tracker);
@@ -25594,18 +25728,17 @@ var ExecutorOrchestratorClass = class {
      * Returns a single chunk for small files where parallelism overhead isn't worth it.
      */
     this.scopeWork = (fileUri, numChunks) => {
-      const fileSize = import_fs14.default.statSync(fileUri).size;
+      const fileSize = import_fs13.default.statSync(fileUri).size;
       if (fileSize === 0) return [];
       if (fileSize < Constants_default.defaults.MIN_FILE_SIZE_FOR_PARALLEL) {
         return [{ start: 0, end: fileSize, isFirstChunk: true, fileUri, index: 0 }];
       }
-      const availableCores = Math.max(1, Math.floor(import_os.default.cpus().length * 0.7));
-      const cpus = numChunks ?? Math.min(availableCores, Constants_default.defaults.MAX_THREAD_COUNT);
+      const targetWorkers = numChunks ?? OrchestratorHelper_default.getParallelWorkerCount();
       const maxChunksBySize = Math.floor(fileSize / Constants_default.defaults.MIN_CHUNK_SIZE);
-      const effectiveChunks = Math.min(cpus, maxChunksBySize);
+      const effectiveChunks = Math.min(targetWorkers, maxChunksBySize);
       if (effectiveChunks <= 1) return [{ start: 0, end: fileSize, isFirstChunk: true, fileUri, index: 0 }];
       const targetChunkSize = Math.floor(fileSize / effectiveChunks);
-      const fd = import_fs14.default.openSync(fileUri, "r");
+      const fd = import_fs13.default.openSync(fileUri, "r");
       try {
         const offsets = [];
         let currentStart = 0;
@@ -25623,7 +25756,7 @@ var ExecutorOrchestratorClass = class {
         }
         return offsets;
       } finally {
-        import_fs14.default.closeSync(fd);
+        import_fs13.default.closeSync(fd);
       }
     };
     /**
@@ -25636,7 +25769,7 @@ var ExecutorOrchestratorClass = class {
       let currentPos = position;
       while (currentPos < fileSize) {
         const bytesToRead = Math.min(BUFFER_SIZE, fileSize - currentPos);
-        const bytesRead = import_fs14.default.readSync(fd, buffer, 0, bytesToRead, currentPos);
+        const bytesRead = import_fs13.default.readSync(fd, buffer, 0, bytesToRead, currentPos);
         if (bytesRead === 0) break;
         for (let i = 0; i < bytesRead; i++) {
           if (buffer[i] === 10) {
@@ -25700,21 +25833,21 @@ var ExecutorOrchestratorClass = class {
               startRow: prod.settings.startRow,
               startColumn: prod.settings.startColumn
             });
-            await (0, import_promises11.pipeline)(
+            await (0, import_promises10.pipeline)(
               xlsCsvStream,
-              import_fs14.default.createWriteStream(decodedPath)
+              import_fs13.default.createWriteStream(decodedPath)
             );
-            const fileStats = await import_promises10.default.stat(decodedPath);
+            const fileStats = await import_promises9.default.stat(decodedPath);
             decodedFiles.push({ fullUri: decodedPath, fileSize: fileStats.size });
             decodedCount++;
             continue;
           }
           if (inferredType === "XML") {
-            const fileContent = await import_promises10.default.readFile(file.fullUri, "utf-8");
+            const fileContent = await import_promises9.default.readFile(file.fullUri, "utf-8");
             const jsonData = XMLParser_default.xmlToJson(fileContent);
             const records = normalizeXmlRows(jsonData);
             if (records.length === 0) {
-              await import_promises10.default.writeFile(decodedPath, "", "utf-8");
+              await import_promises9.default.writeFile(decodedPath, "", "utf-8");
             } else {
               const columns = [];
               for (const record of records) {
@@ -25730,9 +25863,9 @@ var ExecutorOrchestratorClass = class {
                 const row = columns.map((column) => csvSafeValue(record[column]));
                 lines.push(CSVParser_default.stringifyRow(row));
               }
-              await import_promises10.default.writeFile(decodedPath, lines.join("\n"), "utf-8");
+              await import_promises9.default.writeFile(decodedPath, lines.join("\n"), "utf-8");
             }
-            const fileStats = await import_promises10.default.stat(decodedPath);
+            const fileStats = await import_promises9.default.stat(decodedPath);
             decodedFiles.push({ fullUri: decodedPath, fileSize: fileStats.size });
             decodedCount++;
             continue;
@@ -25751,86 +25884,32 @@ var ExecutorOrchestratorClass = class {
       }));
       return decodedResults;
     };
-    this._getWorkerPath = () => {
-      const currentDir = __dirname;
-      if (ProcessENVManager_default.getEnvVariable("NODE_ENV") === "dev" || ProcessENVManager_default.getEnvVariable("NODE_ENV") === "development")
-        return import_path21.default.resolve("./.build/workers");
-      const forcedPath = ProcessENVManager_default.getEnvVariable("REMORA_WORKERS_PATH");
-      if (forcedPath && forcedPath.length > 0)
-        return import_path21.default.join(__dirname, forcedPath);
-      if (!currentDir.includes(".build")) {
-        return import_path21.default.join(__dirname, "../workers");
-      } else {
-        return import_path21.default.resolve("./.build/workers");
-      }
-    };
     this.reconcileExecutorThreadsResults = async (scope, executorResults, tracker) => {
-      const mainPath = ExecutorScope_default2.getMainPath(scope);
+      const mainPath = ExecutorScope_default.getMainPath(scope);
       ConsumerExecutor_default._ensurePath(mainPath);
       if (executorResults.length > 1) {
         Logger_default.log(`[${scope.id}] Merging ${executorResults.length} worker output files into ${mainPath}`);
         const perf = performance.now();
         for (const workerResult of executorResults) {
-          await (0, import_promises11.pipeline)(
-            import_fs14.default.createReadStream(workerResult.resultUri),
-            import_fs14.default.createWriteStream(mainPath, { flags: "a" })
+          await (0, import_promises10.pipeline)(
+            import_fs13.default.createReadStream(workerResult.resultUri),
+            import_fs13.default.createWriteStream(mainPath, { flags: "a" })
           );
-          await import_promises10.default.unlink(workerResult.resultUri);
+          await import_promises9.default.unlink(workerResult.resultUri);
         }
         tracker.measure("merge-workers", performance.now() - perf);
         Logger_default.log(`[${scope.id}] Merge complete in ${Math.round(performance.now() - perf)}ms`);
       } else if (executorResults.length === 1) {
         Logger_default.log(`[${scope.id}] Single worker \u2014 renaming output to ${mainPath}`);
-        await import_promises10.default.rename(executorResults[0].resultUri, mainPath);
+        await import_promises9.default.rename(executorResults[0].resultUri, mainPath);
       }
       return mainPath;
     };
     this.performCleanupOperations = async (scope, tracker) => {
       const start = performance.now();
-      await ExecutorScope_default2.clearScope(scope);
+      await ExecutorScope_default.clearScope(scope);
       tracker.measure("cleanup-operations", performance.now() - start);
     };
-    this.computeFinalResult = (tracker, executorResults, executionId, resultUri) => {
-      const result = {
-        cycles: Algo_default.max(executorResults.map((x) => x.cycles)),
-        elapsedMS: Algo_default.sum(executorResults.map((x) => x.elapsedMS)),
-        inputCount: Algo_default.sum(executorResults.map((x) => x.inputCount)),
-        outputCount: Algo_default.sum(executorResults.map((x) => x.outputCount)),
-        workerCount: executorResults.length,
-        executionId,
-        resultUri,
-        operations: {}
-      };
-      for (const res of executorResults) {
-        for (const opKey of Object.keys(res.operations)) {
-          const op = res.operations[opKey];
-          let label = result.operations[opKey];
-          if (!label) {
-            result.operations[opKey] = { avg: -1, max: -1, min: -1, elapsedMS: [] };
-            label = result.operations[opKey];
-          }
-          label.elapsedMS.push(op.elapsedMS);
-        }
-        for (const opKey of Object.keys(result.operations)) {
-          const operation = result.operations[opKey];
-          if (operation.elapsedMS.length > 0) {
-            operation.min = Math.min(...operation.elapsedMS);
-            operation.max = Math.max(...operation.elapsedMS);
-            operation.avg = Algo_default.mean(operation.elapsedMS);
-          }
-        }
-      }
-      const trackerOperations = tracker.getOperations();
-      for (const opKey of Object.keys(trackerOperations)) {
-        const trackerOp = trackerOperations[opKey];
-        const value = trackerOp.elapsedMS;
-        if (!result.operations[opKey]) {
-          result.operations[opKey] = { avg: value, max: value, min: value, elapsedMS: [] };
-        }
-        result.operations[opKey].elapsedMS.push(value);
-      }
-      return result;
-    };
     this.onWorkAdvanced = (packet, workerId, progress, totalBytesToProcess, bytesProcessedByWorker) => {
       const { processed } = packet;
       bytesProcessedByWorker[workerId] = processed;
@@ -25970,21 +26049,21 @@ var discover = async (producerName) => {
 
 // src/actions/create_producer.ts
 var import_chalk8 = __toESM(require("chalk"));
-var import_fs15 = __toESM(require("fs"));
+var import_fs14 = __toESM(require("fs"));
 var import_path22 = __toESM(require("path"));
 var create_producer = async (name) => {
   try {
-    if (!import_fs15.default.existsSync("./remora/producers")) {
+    if (!import_fs14.default.existsSync("./remora/producers")) {
       throw new Error(import_chalk8.default.red("Missing directory: ") + import_chalk8.default.yellow("./remora/producers"));
     }
-    const defaultProducerTemplate = import_fs15.default.readFileSync(
+    const defaultProducerTemplate = import_fs14.default.readFileSync(
       import_path22.default.join(DOCUMENTATION_DIR, "default_resources/producer.json"),
       "utf-8"
     );
     const defaultProducer = JSON.parse(defaultProducerTemplate);
     defaultProducer.name = name;
     const producerPath = import_path22.default.join("remora/producers", `${name}.json`);
-    import_fs15.default.writeFileSync(producerPath, JSON.stringify(defaultProducer, null, 4));
+    import_fs14.default.writeFileSync(producerPath, JSON.stringify(defaultProducer, null, 4));
     console.log(import_chalk8.default.green(`\u2705 Created producer config at ${producerPath}`));
     console.log(import_chalk8.default.blue("Remember to:"));
     console.log(import_chalk8.default.blue("1. Set the correct source name"));
@@ -25999,14 +26078,14 @@ var create_producer = async (name) => {
 
 // src/actions/create_consumer.ts
 var import_chalk9 = __toESM(require("chalk"));
-var import_fs16 = __toESM(require("fs"));
+var import_fs15 = __toESM(require("fs"));
 var import_path23 = __toESM(require("path"));
 var create_consumer = async (name, producerName) => {
   try {
-    if (!import_fs16.default.existsSync("./remora/consumers")) {
+    if (!import_fs15.default.existsSync("./remora/consumers")) {
       throw new Error(import_chalk9.default.red("Missing directory: ") + import_chalk9.default.yellow("./remora/consumers"));
     }
-    const defaultConsumerTemplate = import_fs16.default.readFileSync(
+    const defaultConsumerTemplate = import_fs15.default.readFileSync(
       import_path23.default.join(DOCUMENTATION_DIR, "default_resources/consumer.json"),
       "utf-8"
     );
@@ -26014,10 +26093,10 @@ var create_consumer = async (name, producerName) => {
     defaultConsumer.name = name;
     if (producerName) {
       const producerPath = import_path23.default.join("remora/producers", `${producerName}.json`);
-      if (!import_fs16.default.existsSync(producerPath)) {
+      if (!import_fs15.default.existsSync(producerPath)) {
         throw new Error(import_chalk9.default.red("Producer not found: ") + import_chalk9.default.yellow(producerPath));
       }
-      const producerConfig = JSON.parse(import_fs16.default.readFileSync(producerPath, "utf-8"));
+      const producerConfig = JSON.parse(import_fs15.default.readFileSync(producerPath, "utf-8"));
       defaultConsumer.producers = [{ name: producerName }];
       defaultConsumer.fields = producerConfig.dimensions.map((dim) => ({
         key: dim.name,
@@ -26035,7 +26114,7 @@ var create_consumer = async (name, producerName) => {
       defaultConsumer.metadata = void 0;
     }
     const consumerPath = import_path23.default.join("remora/consumers", `${name}.json`);
-    import_fs16.default.writeFileSync(consumerPath, JSON.stringify(defaultConsumer, null, 4));
+    import_fs15.default.writeFileSync(consumerPath, JSON.stringify(defaultConsumer, null, 4));
     console.log(import_chalk9.default.green(`\u2705 Created consumer config at ${consumerPath}`));
     if (!producerName) {
       console.log(import_chalk9.default.blue("Remember to:"));
@@ -26058,7 +26137,7 @@ var create_consumer = async (name, producerName) => {
 // src/actions/automap.ts
 var import_chalk10 = __toESM(require("chalk"));
 var import_ora5 = __toESM(require("ora"));
-var import_fs17 = __toESM(require("fs"));
+var import_fs16 = __toESM(require("fs"));
 var import_path24 = __toESM(require("path"));
 var automap = async (producerName, schemaNames) => {
   try {
@@ -26085,12 +26164,12 @@ var automap = async (producerName, schemaNames) => {
     const mapResult = await AutoMapperEngine_default.map(sampleStrings, schemas, producer.settings.fileKey, [source]);
     for (const producer2 of mapResult.producers) {
       const producerPath = import_path24.default.join("remora/producers", `${producer2.name}.json`);
-      import_fs17.default.writeFileSync(producerPath, JSON.stringify(producer2, null, 4));
+      import_fs16.default.writeFileSync(producerPath, JSON.stringify(producer2, null, 4));
       console.log(import_chalk10.default.blue(`Created producer: ${producer2.name}`));
     }
     for (const consumer of mapResult.consumers) {
       const consumerPath = import_path24.default.join("remora/consumers", `${consumer.name}.json`);
-      import_fs17.default.writeFileSync(consumerPath, JSON.stringify(consumer, null, 4));
+      import_fs16.default.writeFileSync(consumerPath, JSON.stringify(consumer, null, 4));
       console.log(import_chalk10.default.blue(`Created consumer: ${consumer.name}`));
     }
     spinner.succeed("Producer has been successfully mapped");
@@ -26288,6 +26367,7 @@ if (!process.env.REMORA_RUNTIME_CONTEXT) {
   process.env.REMORA_RUNTIME_CONTEXT = "cli";
   Logger_default.warn('Missing property for REMORA_RUNTIME_CONTEXT during the Remora CLI startup. Defaulting to "cli" for this run. Set it manually in your environment.');
 }
+ProcessShutdownManager_default.init("Remora CLI");
 var program = new import_commander.Command();
 var remoraLicenceKey = ProcessENVManager_default.getEnvVariable("REMORA_LICENCE_KEY");
 var check = LicenceManager_default.validate(remoraLicenceKey);