@forzalabs/remora 1.2.9 → 1.2.11

package/workers/ExecutorWorker.js

@@ -10292,7 +10292,7 @@ var require_node2 = __commonJS({
 var require_tail_file = __commonJS({
   "../../packages/logger/node_modules/winston/lib/winston/tail-file.js"(exports2, module2) {
     "use strict";
-    var fs19 = require("fs");
+    var fs18 = require("fs");
     var { StringDecoder } = require("string_decoder");
     var { Stream } = require_readable();
     function noop() {
@@ -10313,7 +10313,7 @@ var require_tail_file = __commonJS({
         stream.emit("end");
         stream.emit("close");
       };
-      fs19.open(options.file, "a+", "0644", (err2, fd) => {
+      fs18.open(options.file, "a+", "0644", (err2, fd) => {
         if (err2) {
           if (!iter) {
             stream.emit("error", err2);
@@ -10325,10 +10325,10 @@ var require_tail_file = __commonJS({
         }
         (function read() {
           if (stream.destroyed) {
-            fs19.close(fd, noop);
+            fs18.close(fd, noop);
             return;
           }
-          return fs19.read(fd, buffer, 0, buffer.length, pos, (error, bytes) => {
+          return fs18.read(fd, buffer, 0, buffer.length, pos, (error, bytes) => {
             if (error) {
               if (!iter) {
                 stream.emit("error", error);
@@ -10387,7 +10387,7 @@ var require_tail_file = __commonJS({
 var require_file = __commonJS({
   "../../packages/logger/node_modules/winston/lib/winston/transports/file.js"(exports2, module2) {
     "use strict";
-    var fs19 = require("fs");
+    var fs18 = require("fs");
     var path18 = require("path");
     var asyncSeries = require_series();
     var zlib2 = require("zlib");
@@ -10592,7 +10592,7 @@ var require_file = __commonJS({
         let buff = "";
         let results = [];
         let row = 0;
-        const stream = fs19.createReadStream(file, {
+        const stream = fs18.createReadStream(file, {
           encoding: "utf8"
         });
         stream.on("error", (err2) => {
@@ -10744,7 +10744,7 @@ var require_file = __commonJS({
       stat(callback) {
         const target = this._getFile();
         const fullpath = path18.join(this.dirname, target);
-        fs19.stat(fullpath, (err2, stat) => {
+        fs18.stat(fullpath, (err2, stat) => {
           if (err2 && err2.code === "ENOENT") {
             debug("ENOENT\xA0ok", fullpath);
             this.filename = target;
@@ -10849,7 +10849,7 @@ var require_file = __commonJS({
       _createStream(source) {
         const fullpath = path18.join(this.dirname, this.filename);
         debug("create stream start", fullpath, this.options);
-        const dest = fs19.createWriteStream(fullpath, this.options).on("error", (err2) => debug(err2)).on("close", () => debug("close", dest.path, dest.bytesWritten)).on("open", () => {
+        const dest = fs18.createWriteStream(fullpath, this.options).on("error", (err2) => debug(err2)).on("close", () => debug("close", dest.path, dest.bytesWritten)).on("open", () => {
           debug("file open ok", fullpath);
           this.emit("open", fullpath);
           source.pipe(dest);
@@ -10928,7 +10928,7 @@ var require_file = __commonJS({
         const isZipped = this.zippedArchive ? ".gz" : "";
         const filePath = `${basename}${isOldest}${ext}${isZipped}`;
         const target = path18.join(this.dirname, filePath);
-        fs19.unlink(target, callback);
+        fs18.unlink(target, callback);
       }
       /**
        * Roll files forward based on integer, up to maxFiles. e.g. if base if
@@ -10951,17 +10951,17 @@ var require_file = __commonJS({
           tasks.push(function(i, cb) {
             let fileName = `${basename}${i - 1}${ext}${isZipped}`;
             const tmppath = path18.join(this.dirname, fileName);
-            fs19.exists(tmppath, (exists) => {
+            fs18.exists(tmppath, (exists) => {
               if (!exists) {
                 return cb(null);
               }
               fileName = `${basename}${i}${ext}${isZipped}`;
-              fs19.rename(tmppath, path18.join(this.dirname, fileName), cb);
+              fs18.rename(tmppath, path18.join(this.dirname, fileName), cb);
             });
           }.bind(this, x));
         }
         asyncSeries(tasks, () => {
-          fs19.rename(
+          fs18.rename(
             path18.join(this.dirname, `${basename}${ext}${isZipped}`),
             path18.join(this.dirname, `${basename}1${ext}${isZipped}`),
             callback
@@ -10977,22 +10977,22 @@ var require_file = __commonJS({
        * @private
        */
       _compressFile(src, dest, callback) {
-        fs19.access(src, fs19.F_OK, (err2) => {
+        fs18.access(src, fs18.F_OK, (err2) => {
           if (err2) {
             return callback();
           }
           var gzip = zlib2.createGzip();
-          var inp = fs19.createReadStream(src);
-          var out = fs19.createWriteStream(dest);
+          var inp = fs18.createReadStream(src);
+          var out = fs18.createWriteStream(dest);
           out.on("finish", () => {
-            fs19.unlink(src, callback);
+            fs18.unlink(src, callback);
           });
           inp.pipe(gzip).pipe(out);
         });
       }
       _createLogDirIfNotExist(dirPath) {
-        if (!fs19.existsSync(dirPath)) {
-          fs19.mkdirSync(dirPath, { recursive: true });
+        if (!fs18.existsSync(dirPath)) {
+          fs18.mkdirSync(dirPath, { recursive: true });
         }
       }
     };
@@ -18738,25 +18738,6 @@ var ProcessENVManagerClass = class {
 var ProcessENVManager = new ProcessENVManagerClass();
 var ProcessENVManager_default = ProcessENVManager;
 
-// ../../packages/common/src/SecretManager.ts
-var SecretManagerClass = class {
-  constructor() {
-    /**
-     * If the value is a secret (or .env setting), replace it with the value inside the .env configuration file
-     * Starts with "{" and ends with "}".
-     * e.g. {AWS_ID}
-     */
-    this.replaceSecret = (value) => {
-      if (!value || value.length <= 2 || !value.startsWith("{") || !value.endsWith("}"))
-        return value;
-      const parsedValue = value.slice(1, value.length - 1);
-      return ProcessENVManager_default.getEnvVariable(parsedValue);
-    };
-  }
-};
-var SecretManager = new SecretManagerClass();
-var SecretManager_default = SecretManager;
-
 // ../../packages/common/src/ExecutorScope.ts
 var import_path3 = __toESM(require("path"), 1);
 var import_fs3 = __toESM(require("fs"), 1);
@@ -18764,7 +18745,7 @@ var import_promises = __toESM(require("fs/promises"), 1);
 
 // ../../packages/constants/src/Constants.ts
 var CONSTANTS = {
-  cliVersion: "1.2.9",
+  cliVersion: "1.2.11",
   backendVersion: 1,
   backendPort: 5088,
   workerVersion: 2,
@@ -18810,10 +18791,10 @@ var ExecutorScopeClass = class {
   constructor() {
     this.WORKERS_FOLDER = "workers";
     this.PRODUCERS_FOLDER = "producers";
+    this.getBasePath = () => import_path3.default.join(Constants_default.defaults.REMORA_PATH, Constants_default.defaults.PRODUCER_TEMP_FOLDER);
     this.getWorkerPath = (scope, workerId) => {
       return import_path3.default.join(
-        Constants_default.defaults.REMORA_PATH,
-        Constants_default.defaults.PRODUCER_TEMP_FOLDER,
+        this.getBasePath(),
         // A specific execution sits entirely in this folder, so at the end it's safe to delete it entirely
         scope.folder,
         this.WORKERS_FOLDER,
@@ -18822,8 +18803,7 @@ var ExecutorScopeClass = class {
     };
     this.getProducerPath = (scope, producer, sourceFileKey) => {
       return import_path3.default.join(
-        Constants_default.defaults.REMORA_PATH,
-        Constants_default.defaults.PRODUCER_TEMP_FOLDER,
+        this.getBasePath(),
         // A specific execution sits entirely in this folder, so at the end it's safe to delete it entirely
         scope.folder,
         this.PRODUCERS_FOLDER,
@@ -18833,22 +18813,30 @@ var ExecutorScopeClass = class {
     };
     this.getMainPath = (scope) => {
       return import_path3.default.join(
-        Constants_default.defaults.REMORA_PATH,
-        Constants_default.defaults.PRODUCER_TEMP_FOLDER,
+        this.getBasePath(),
         scope.folder,
         "main.dataset"
       );
     };
     this.clearScope = async (scope) => {
       const scopePath = import_path3.default.join(
-        Constants_default.defaults.REMORA_PATH,
-        Constants_default.defaults.PRODUCER_TEMP_FOLDER,
+        this.getBasePath(),
         scope.folder
       );
       if (import_fs3.default.existsSync(scopePath)) {
         await import_promises.default.rm(scopePath, { recursive: true, force: true });
       }
     };
+    this.deepClear = () => {
+      const basePath = this.getBasePath();
+      const openScopes = this.getOpenScopes();
+      for (const scopeFolder of openScopes) {
+        const scopePath = import_path3.default.join(basePath, scopeFolder);
+        if (import_fs3.default.existsSync(scopePath)) {
+          import_fs3.default.rmSync(scopePath, { recursive: true, force: true });
+        }
+      }
+    };
     this.ensurePath = (fileUri) => {
       const dir = import_path3.default.dirname(fileUri);
       if (!import_fs3.default.existsSync(dir))
@@ -18856,11 +18844,105 @@ var ExecutorScopeClass = class {
       if (!import_fs3.default.existsSync(fileUri))
         import_fs3.default.writeFileSync(fileUri, "");
     };
+    this.getOpenScopes = () => {
+      const basePath = this.getBasePath();
+      if (!import_fs3.default.existsSync(basePath))
+        return [];
+      return import_fs3.default.readdirSync(basePath, { withFileTypes: true }).filter((entry) => entry.isDirectory()).map((entry) => entry.name).filter((folder) => folder !== "logs" && folder !== "usage");
+    };
   }
 };
 var ExecutorScope = new ExecutorScopeClass();
 var ExecutorScope_default = ExecutorScope;
 
+// ../../packages/common/src/ProcessShutdownManager.ts
+var ProcessShutdownManagerClass = class {
+  constructor() {
+    this._initialized = false;
+    this._cleaned = false;
+    this._runtimeName = "Remora process";
+    this.init = (runtimeName) => {
+      if (this._initialized)
+        return;
+      this._initialized = true;
+      if (runtimeName)
+        this._runtimeName = runtimeName;
+      process.once("SIGINT", () => this.handleSignal("SIGINT", 130));
+      process.once("SIGTERM", () => this.handleSignal("SIGTERM", 143));
+      process.once("uncaughtException", (error) => this.handleUnexpectedShutdown("uncaughtException", error));
+      process.once("unhandledRejection", (reason) => this.handleUnexpectedShutdown("unhandledRejection", reason));
+      process.once("beforeExit", (code) => this.handleBeforeExit(code));
+      process.once("exit", (code) => this.handleExit(code));
+    };
+    this.handleSignal = (signal, exitCode) => {
+      this.setShutdownState("intentional", signal);
+      Logger_default.warn(`Received ${signal}. Shutting down ${this._runtimeName}.`);
+      process.exit(exitCode);
+    };
+    this.handleUnexpectedShutdown = (reason, error) => {
+      this.setShutdownState("unintentional", reason);
+      Logger_default.error(this.asError(reason, error));
+      process.exit(1);
+    };
+    this.handleBeforeExit = (code) => {
+      this.setShutdownState(code === 0 ? "intentional" : "unintentional", `beforeExit:${code}`);
+    };
+    this.handleExit = (code) => {
+      if (!this._shutdownState)
+        this.setShutdownState(code === 0 ? "intentional" : "unintentional", `exit:${code}`);
+      this.cleanupOpenScopes(code);
+    };
+    this.cleanupOpenScopes = (code) => {
+      if (this._cleaned)
+        return;
+      this._cleaned = true;
+      const openScopes = ExecutorScope_default.getOpenScopes();
+      const scopeCount = openScopes.length;
+      const shutdownState = this._shutdownState;
+      const shutdownDescription = `${shutdownState?.type ?? "intentional"} shutdown (${shutdownState?.reason ?? `exit:${code}`})`;
+      if (scopeCount === 0) {
+        Logger_default.info(`Detected ${shutdownDescription} for ${this._runtimeName}. No executor scopes to clean up.`);
+        return;
+      }
+      Logger_default.warn(`Detected ${shutdownDescription} for ${this._runtimeName}. Cleaning up ${scopeCount} executor scope${scopeCount === 1 ? "" : "s"}.`);
+      ExecutorScope_default.deepClear();
+    };
+    this.setShutdownState = (type, reason) => {
+      if (this._shutdownState)
+        return;
+      this._shutdownState = { type, reason };
+    };
+    this.asError = (reason, error) => {
+      if (error instanceof Error) {
+        const contextualError = new Error(`Unexpected ${reason} during ${this._runtimeName} execution: ${error.message}`);
+        contextualError.stack = error.stack;
+        return contextualError;
+      }
+      return new Error(`Unexpected ${reason} during ${this._runtimeName} execution: ${String(error)}`);
+    };
+  }
+};
+var ProcessShutdownManager = new ProcessShutdownManagerClass();
+
+// ../../packages/common/src/SecretManager.ts
+var SecretManagerClass = class {
+  constructor() {
+    /**
+     * If the value is a secret (or .env setting), replace it with the value inside the .env configuration file
+     * Starts with "{" and ends with "}".
+     * e.g. {AWS_ID}
+     */
+    this.replaceSecret = (value) => {
+      if (!value || value.length <= 2 || !value.startsWith("{") || !value.endsWith("}"))
+        return value;
+      const parsedValue = value.slice(1, value.length - 1);
+      return ProcessENVManager_default.getEnvVariable(parsedValue);
+    };
+  }
+};
+var SecretManager = new SecretManagerClass();
+var SecretManager_default = SecretManager;
+
 // ../../packages/common/src/Environment.ts
 var import_fs5 = __toESM(require("fs"), 1);
 var import_crypto = __toESM(require("crypto"), 1);
@@ -19102,7 +19184,16 @@ var ValidatorClass = class {
         }
         const allAvailableFields = [...availableFieldsByProducer.values()].flat();
         for (const field of consumer.fields) {
-          if (field.key === "*" || field.fixed || field.copyFrom) continue;
+          const syntheticFieldModes = [field.fixed === true, !!field.copyFrom, field.lineCount === true].filter(Boolean);
+          if (syntheticFieldModes.length > 1)
+            errors.push(`Field "${field.alias ?? field.key}" in consumer "${consumer.name}" can only use one synthetic field mode among "fixed", "copyFrom", and "lineCount".`);
+          if (field.lineCount && field.key === "*")
+            errors.push(`Field "${field.alias ?? field.key}" in consumer "${consumer.name}" cannot use "lineCount" with the wildcard key "*".`);
+          if (field.lineCount && field.transform)
+            errors.push(`Field "${field.alias ?? field.key}" in consumer "${consumer.name}" cannot use "transform" together with "lineCount".`);
+          if (field.lineCount && field.validate && field.validate.length > 0)
+            errors.push(`Field "${field.alias ?? field.key}" in consumer "${consumer.name}" cannot use "validate" together with "lineCount".`);
+          if (field.key === "*" || field.fixed || field.copyFrom || field.lineCount) continue;
           if (field.from) {
             const producerFields = availableFieldsByProducer.get(field.from);
             if (producerFields && !producerFields.includes(field.key))
@@ -19123,6 +19214,8 @@ var ValidatorClass = class {
           const found = precedingFields.find((f) => (f.alias ?? f.key) === field.copyFrom);
           if (!found)
             errors.push(`Field "${field.alias ?? field.key}" uses copyFrom "${field.copyFrom}" but no field with that name/alias exists before it in consumer "${consumer.name}".`);
+          else if (found.lineCount)
+            errors.push(`Field "${field.alias ?? field.key}" in consumer "${consumer.name}" cannot use copyFrom on a field with "lineCount" because line counts are normalized after worker merge.`);
         }
         if (consumer.filters && consumer.filters.length > 0) {
           if (consumer.filters.some((x) => x.sql && x.rule))
@@ -19143,10 +19236,14 @@ var ValidatorClass = class {
             const missingRules = rulesWithMatchingFields.filter((x) => !x.match);
             errors.push(`Filter(s) on member(s) "${missingRules.map((x) => x.rule.rule.member).join(", ")}" is invalid since the member specified is not present in the consumer. Check the member value or add the missing field to the consumer.`);
           }
+          const lineCountRuleMembers = rulesWithMatchingFields.filter((x) => x.match?.lineCount);
+          if (lineCountRuleMembers.length > 0)
+            errors.push(`Consumer "${consumer.name}" cannot use rule-based filters on field(s) with "lineCount": ${lineCountRuleMembers.map((x) => `"${x.rule.rule.member}"`).join(", ")}.`);
         }
         const validateTransformations = (fields) => {
           const errors2 = [];
           const trxsFields = fields.filter((x) => x.transform);
+          const lineCountFieldNames = new Set(fields.filter((x) => x.lineCount).map((x) => x.alias ?? x.key));
           for (const field of trxsFields) {
             const trxToValidate = [];
             if (Array.isArray(field.transform))
@@ -19154,6 +19251,16 @@ var ValidatorClass = class {
             else
               trxToValidate.push(field.transform);
             for (const trans of trxToValidate) {
+              if ("multiplyBy" in trans) {
+                const invalidFields = trans.multiplyBy.fields.filter((x) => lineCountFieldNames.has(x));
+                if (invalidFields.length > 0)
+                  errors2.push(`Transformation on field "${field.alias ?? field.key}" cannot reference lineCount field(s) in multiplyBy: ${invalidFields.map((x) => `"${x}"`).join(", ")}.`);
+              }
+              if ("addBy" in trans) {
+                const invalidFields = trans.addBy.fields.filter((x) => lineCountFieldNames.has(x));
+                if (invalidFields.length > 0)
+                  errors2.push(`Transformation on field "${field.alias ?? field.key}" cannot reference lineCount field(s) in addBy: ${invalidFields.map((x) => `"${x}"`).join(", ")}.`);
+              }
               if ("combine_fields" in trans) {
                 const { combine_fields } = trans;
                 if (!combine_fields.fields || combine_fields.fields.length === 0)
@@ -19161,6 +19268,9 @@ var ValidatorClass = class {
                 const missingFieldsInConsumer = combine_fields.fields.map((x) => ({ field: x, found: fields.find((k) => (k.alias ?? k.key) === x) })).filter((x) => !x.found);
                 if (missingFieldsInConsumer.length > 0)
                   errors2.push(`The requested field(s) for a transformation is missing in the consumer -> missing field(s): "${missingFieldsInConsumer.map((x) => x.field).join(", ")}"; field transformation: "${field.alias ?? field.key}";`);
+                const invalidFields = combine_fields.fields.filter((x) => lineCountFieldNames.has(x));
+                if (invalidFields.length > 0)
+                  errors2.push(`Transformation on field "${field.alias ?? field.key}" cannot reference lineCount field(s) in combine_fields: ${invalidFields.map((x) => `"${x}"`).join(", ")}.`);
               }
             }
           }
@@ -19183,7 +19293,7 @@ var ValidatorClass = class {
               errors.push(`The export destination "${output.exportDestination}" was not found in the sources.`);
           }
         }
-        const dimensionFields = consumer.fields.filter((x) => x.key !== "*" && !x.fixed && !x.copyFrom);
+        const dimensionFields = consumer.fields.filter((x) => x.key !== "*" && !x.fixed && !x.copyFrom && !x.lineCount);
         const dimensionKeys = dimensionFields.map((x) => `${x.from ?? "_default_"}::${x.key}`);
         const duplicateDimensionKeys = dimensionKeys.filter((k, i) => dimensionKeys.indexOf(k) !== i);
         if (duplicateDimensionKeys.length > 0) {
@@ -19621,10 +19731,10 @@ var Environment = new EnvironmentClass();
 var Environment_default = Environment;
 
 // ../../packages/executors/src/ConsumerExecutor.ts
-var import_path16 = __toESM(require("path"));
-var import_fs11 = __toESM(require("fs"));
+var import_path15 = __toESM(require("path"));
+var import_fs10 = __toESM(require("fs"));
 var import_readline6 = __toESM(require("readline"));
-var import_promises9 = __toESM(require("fs/promises"));
+var import_promises8 = __toESM(require("fs/promises"));
 var import_crypto4 = __toESM(require("crypto"));
 
 // ../../packages/engines/src/CryptoEngine.ts
@@ -20858,6 +20968,19 @@ var Helper = {
 };
 var Helper_default = Helper;
 
+// ../../packages/helper/src/Formatter.ts
+var Formatter = {
+  bytes: (bytes, decimals = 2) => {
+    if (!+bytes) return "0 Bytes";
+    const k = 1024;
+    const dm = decimals < 0 ? 0 : decimals;
+    const sizes = ["Bytes", "KB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB"];
+    const i = Math.floor(Math.log(bytes) / Math.log(k));
+    return `${parseFloat((bytes / Math.pow(k, i)).toFixed(dm))} ${sizes[i]}`;
+  }
+};
+var Formatter_default = Formatter;
+
 // ../../packages/helper/src/Settings.ts
 var SETTINGS = {
   db: {
@@ -22688,6 +22811,25 @@ var ConsumerManagerClass = class {
       const expandedFields = convertedFields.flatMap((x) => this.expandField(consumer, x, availableColumns));
       return expandedFields;
     };
+    this.getLineCountFields = (consumer, visibleOnly = false) => {
+      Affirm_default(consumer, "Invalid consumer");
+      return this.getLineCountFieldsFromExpanded(this.getExpandedFields(consumer), visibleOnly);
+    };
+    this.getLineCountFieldsFromExpanded = (fields, visibleOnly = false) => {
+      Affirm_default(fields, "Invalid consumer fields");
+      return fields.filter((field) => this.isLineCountField(field.cField) && (!visibleOnly || !field.cField.hidden));
+    };
+    this.hasLineCountFields = (consumer, visibleOnly = false) => {
+      return this.getLineCountFields(consumer, visibleOnly).length > 0;
+    };
+    this.isSyntheticField = (field) => {
+      Affirm_default(field, "Invalid field");
+      return field.fixed === true || !!field.copyFrom || this.isLineCountField(field);
+    };
+    this.isLineCountField = (field) => {
+      Affirm_default(field, "Invalid field");
+      return field.lineCount === true;
+    };
     this.getAvailableColumns = (consumer) => {
       return ResourcesUtils_default.getAvailableColumns(consumer);
     };
@@ -22740,7 +22882,7 @@ var ConsumerManagerClass = class {
         column = columns.find((x) => x.owner === field.from && x.nameInProducer === field.key);
       } else if (consumer.producers.length === 1 && !field.from) {
         column = columns.find((x) => x.nameInProducer === field.key);
-      } else if (!field.fixed && !field.copyFrom) {
+      } else if (!this.isSyntheticField(field)) {
         const matches = columns.filter((x) => x.nameInProducer === field.key);
         Affirm_default(matches.length > 0, `Consumer "${consumer.name}" misconfiguration: the field "${field.key}" is not found in any of the included producers (${consumer.producers.map((x) => x.name).join(", ")})`);
         if (matches.length === 1) {
@@ -22751,14 +22893,14 @@ var ConsumerManagerClass = class {
         column = matches[0];
       }
       if (!column) {
-        if (field.fixed === true && Algo_default.hasVal(field.default) || field.copyFrom) {
+        if (field.fixed === true && Algo_default.hasVal(field.default) || this.isSyntheticField(field)) {
           column = {
             aliasInProducer: field.key,
             nameInProducer: field.alias ?? field.key,
             consumerAlias: field.alias ?? field.key,
             consumerKey: field.key,
             owner: field.from,
-            dimension: { name: field.key, type: typeof field.default }
+            dimension: { name: field.key, type: this.isLineCountField(field) ? "number" : typeof field.default }
           };
         }
       }
@@ -24197,69 +24339,8 @@ var UsageManager = new UsageManagerClass();
 var UsageManager_default = UsageManager;
 
 // ../../packages/executors/src/OutputExecutor.ts
-var fs14 = __toESM(require("fs"));
-
-// ../../packages/executors/src/ExecutorScope.ts
+var fs13 = __toESM(require("fs"));
 var import_path14 = __toESM(require("path"));
-var import_fs10 = __toESM(require("fs"));
-var import_promises8 = __toESM(require("fs/promises"));
-var ExecutorScopeClass2 = class {
-  constructor() {
-    this.WORKERS_FOLDER = "workers";
-    this.PRODUCERS_FOLDER = "producers";
-    this.getWorkerPath = (scope, workerId) => {
-      return import_path14.default.join(
-        Constants_default.defaults.REMORA_PATH,
-        Constants_default.defaults.PRODUCER_TEMP_FOLDER,
-        // A specific execution sits entirely in this folder, so at the end it's safe to delete it entirely
-        scope.folder,
-        this.WORKERS_FOLDER,
-        `${workerId}.dataset`
-      );
-    };
-    this.getProducerPath = (scope, producer, sourceFileKey) => {
-      return import_path14.default.join(
-        Constants_default.defaults.REMORA_PATH,
-        Constants_default.defaults.PRODUCER_TEMP_FOLDER,
-        // A specific execution sits entirely in this folder, so at the end it's safe to delete it entirely
-        scope.folder,
-        this.PRODUCERS_FOLDER,
-        producer.name,
-        `${sourceFileKey}.dataset`
-      );
-    };
-    this.getMainPath = (scope) => {
-      return import_path14.default.join(
-        Constants_default.defaults.REMORA_PATH,
-        Constants_default.defaults.PRODUCER_TEMP_FOLDER,
-        scope.folder,
-        "main.dataset"
-      );
-    };
-    this.clearScope = async (scope) => {
-      const scopePath = import_path14.default.join(
-        Constants_default.defaults.REMORA_PATH,
-        Constants_default.defaults.PRODUCER_TEMP_FOLDER,
-        scope.folder
-      );
-      if (import_fs10.default.existsSync(scopePath)) {
-        await import_promises8.default.rm(scopePath, { recursive: true, force: true });
-      }
-    };
-    this.ensurePath = (fileUri) => {
-      const dir = import_path14.default.dirname(fileUri);
-      if (!import_fs10.default.existsSync(dir))
-        import_fs10.default.mkdirSync(dir, { recursive: true });
-      if (!import_fs10.default.existsSync(fileUri))
-        import_fs10.default.writeFileSync(fileUri, "");
-    };
-  }
-};
-var ExecutorScope2 = new ExecutorScopeClass2();
-var ExecutorScope_default2 = ExecutorScope2;
-
-// ../../packages/executors/src/OutputExecutor.ts
-var import_path15 = __toESM(require("path"));
 var OutputExecutorClass = class {
   constructor() {
     this._getInternalRecordFormat = (consumer) => {
@@ -24303,13 +24384,13 @@ var OutputExecutorClass = class {
       for (const output of consumer.outputs) {
         const destination = Environment_default.getSource(output.exportDestination);
         const driver = await DriverFactory_default.instantiateDestination(destination);
-        const currentPath = import_path15.default.dirname(ExecutorScope_default2.getMainPath(scope));
+        const currentPath = import_path14.default.dirname(ExecutorScope_default.getMainPath(scope));
         const destinationName = this._composeFileName(consumer, output, this._getExtension(output));
         Logger_default.log(`Exporting consumer "${consumer.name}" to "${output.exportDestination}" as ${output.format} (${destinationName})`);
-        const filenameArray = fs14.readdirSync(currentPath).filter((filename) => filename.includes(".dataset"));
+        const filenameArray = fs13.readdirSync(currentPath).filter((filename) => filename.includes(".dataset"));
         for (const filename in filenameArray) {
           const destinationPath = this.getCompletedPath(destinationName, filename);
-          const startingPath = import_path15.default.join(currentPath, filenameArray[filename]);
+          const startingPath = import_path14.default.join(currentPath, filenameArray[filename]);
           if (output.format === internalFormat) {
             results.push(await driver.move(startingPath, destinationPath));
           } else {
@@ -24369,47 +24450,54 @@ var OutputExecutor_default = OutputExecutor;
 var ConsumerExecutorClass = class {
   constructor() {
     this._getWorkPath = (consumer, executionId) => {
-      const execFolder = import_path16.default.join(consumer.name, executionId);
-      const workPath = import_path16.default.join("./remora", Constants_default.defaults.PRODUCER_TEMP_FOLDER, execFolder, ".dataset");
+      const execFolder = import_path15.default.join(consumer.name, executionId);
+      const workPath = import_path15.default.join("./remora", Constants_default.defaults.PRODUCER_TEMP_FOLDER, execFolder, ".dataset");
       return workPath;
     };
     this._clearWorkPath = async (workPath) => {
       try {
-        if (import_fs11.default.existsSync(workPath)) {
-          await import_promises9.default.unlink(workPath);
+        if (import_fs10.default.existsSync(workPath)) {
+          await import_promises8.default.unlink(workPath);
         }
       } catch (error) {
       }
       try {
-        const dir = import_path16.default.dirname(workPath);
-        if (import_fs11.default.existsSync(dir)) {
-          await import_promises9.default.rmdir(dir);
+        const dir = import_path15.default.dirname(workPath);
+        if (import_fs10.default.existsSync(dir)) {
+          await import_promises8.default.rmdir(dir);
         }
       } catch (error) {
       }
     };
     this._ensurePath = (pathUri) => {
-      const dir = import_path16.default.dirname(pathUri);
-      if (!import_fs11.default.existsSync(dir))
-        import_fs11.default.mkdirSync(dir, { recursive: true });
-      if (!import_fs11.default.existsSync(pathUri))
-        import_fs11.default.writeFileSync(pathUri, "");
+      const dir = import_path15.default.dirname(pathUri);
+      if (!import_fs10.default.existsSync(dir))
+        import_fs10.default.mkdirSync(dir, { recursive: true });
+      if (!import_fs10.default.existsSync(pathUri))
+        import_fs10.default.writeFileSync(pathUri, "");
     };
     this.processRecord = (options) => {
       const { consumer, fields, dimensions, producer, record, requestOptions, index: recordIndex } = options;
       for (const field of fields) {
         const { cField } = field;
         const fieldKey = cField.alias ?? cField.key;
+        if (cField.fixed && Algo_default.hasVal(cField.default)) {
+          record[fieldKey] = cField.default;
+          continue;
+        }
+        if (ConsumerManager_default.isLineCountField(cField)) {
+          record[fieldKey] = recordIndex + 1;
+          continue;
+        }
+        if (cField.copyFrom) {
+          record[fieldKey] = record[cField.copyFrom];
+          continue;
+        }
         let dimension;
         try {
           dimension = dimensions.find((x) => x.name === cField.key);
           if (!dimension) {
-            if (cField.fixed && Algo_default.hasVal(cField.default))
-              record[fieldKey] = cField.default;
-            else if (cField.copyFrom)
-              record[fieldKey] = record[cField.copyFrom];
-            else
-              throw new Error(`The requested field "${cField.key}" from the consumer is not present in the underlying producer "${producer.name}" (${dimensions.map((x) => x.name).join(", ")})`);
+            throw new Error(`The requested field "${cField.key}" from the consumer is not present in the underlying producer "${producer.name}" (${dimensions.map((x) => x.name).join(", ")})`);
           }
         } catch (error) {
           const err2 = new Error(`Resolving dimension for field "${fieldKey}" of producer "${producer.name}" failed (index: ${recordIndex}): ${error.message}`, { cause: error });
@@ -24522,11 +24610,69 @@ var ConsumerExecutorClass = class {
       }
       return record;
     };
+    this.processLineCount = async (consumer, datasetPath, executorResults) => {
+      const internalRecordFormat = OutputExecutor_default._getInternalRecordFormat(consumer);
+      const internalFields = ConsumerManager_default.getExpandedFields(consumer);
+      const lineCountFields = ConsumerManager_default.getLineCountFieldsFromExpanded(internalFields, true);
+      if (lineCountFields.length === 0)
+        return 0;
+      const tempWorkPath = datasetPath + "_tmp";
+      const reader = import_fs10.default.createReadStream(datasetPath);
+      const lineReader = import_readline6.default.createInterface({ input: reader, crlfDelay: Infinity });
+      const writer = import_fs10.default.createWriteStream(tempWorkPath);
+      const waitForDrain = () => new Promise((resolve) => writer.once("drain", resolve));
+      let workerIndex = 0;
+      let currentWorker = executorResults[workerIndex] ?? null;
+      let remainingLinesInWorker = currentWorker?.outputCount ?? 0;
+      let processedLineCount = 0;
+      const advanceWorker = () => {
+        while (currentWorker && remainingLinesInWorker <= 0) {
+          workerIndex++;
+          currentWorker = executorResults[workerIndex] ?? null;
+          remainingLinesInWorker = currentWorker?.outputCount ?? 0;
+        }
+      };
+      advanceWorker();
+      for await (const line of lineReader) {
+        advanceWorker();
+        Affirm_default(currentWorker, `Unable to resolve worker metadata while processing lineCount for ${datasetPath}`);
+        const rowOffset = currentWorker.rowOffset ?? 0;
+        if (rowOffset > 0) {
+          const record = this._parseLine(line, internalRecordFormat, internalFields);
+          for (const field of lineCountFields) {
+            const currentValue = Number(record[field.finalKey]);
+            Affirm_default(!Number.isNaN(currentValue), `Invalid lineCount value found while processing ${datasetPath}`);
+            record[field.finalKey] = currentValue + rowOffset;
+          }
+          if (!writer.write(OutputExecutor_default.outputRecord(record, consumer, internalFields) + "\n"))
+            await waitForDrain();
+        } else if (!writer.write(line + "\n")) {
+          await waitForDrain();
+        }
+        remainingLinesInWorker--;
+        processedLineCount++;
+      }
+      lineReader.close();
+      await new Promise((resolve, reject) => {
+        writer.on("close", resolve);
+        writer.on("error", reject);
+        writer.end();
+      });
+      if (!reader.destroyed) {
+        await new Promise((resolve) => {
+          reader.once("close", resolve);
+          reader.destroy();
+        });
+      }
+      await import_promises8.default.unlink(datasetPath);
+      await import_promises8.default.rename(tempWorkPath, datasetPath);
+      return processedLineCount;
+    };
     this.processDistinct = async (datasetPath) => {
-      const reader = import_fs11.default.createReadStream(datasetPath);
+      const reader = import_fs10.default.createReadStream(datasetPath);
       const lineReader = import_readline6.default.createInterface({ input: reader, crlfDelay: Infinity });
       const tempWorkPath = datasetPath + "_tmp";
-      const writer = import_fs11.default.createWriteStream(tempWorkPath);
+      const writer = import_fs10.default.createWriteStream(tempWorkPath);
       const waitForDrain = () => new Promise((resolve) => writer.once("drain", resolve));
       let newLineCount = 0;
       const seen = /* @__PURE__ */ new Set();
@@ -24551,12 +24697,12 @@ var ConsumerExecutorClass = class {
           reader.destroy();
         });
       }
-      await import_promises9.default.unlink(datasetPath);
-      await import_promises9.default.rename(tempWorkPath, datasetPath);
+      await import_promises8.default.unlink(datasetPath);
+      await import_promises8.default.rename(tempWorkPath, datasetPath);
       return newLineCount;
     };
     this.processDistinctOn = async (consumer, datasetPath) => {
-      const reader = import_fs11.default.createReadStream(datasetPath);
+      const reader = import_fs10.default.createReadStream(datasetPath);
       const lineReader = import_readline6.default.createInterface({ input: reader, crlfDelay: Infinity });
       const { distinctOn } = consumer.options;
       const { keys, resolution } = distinctOn;
@@ -24579,7 +24725,7 @@ var ConsumerExecutorClass = class {
       }
       lineReader.close();
       const tempWorkPath = datasetPath + "_tmp";
-      const writer = import_fs11.default.createWriteStream(tempWorkPath);
+      const writer = import_fs10.default.createWriteStream(tempWorkPath);
       const waitForDrain = () => new Promise((resolve) => writer.once("drain", resolve));
       for (const { line } of winners.values()) {
         if (!writer.write(line + "\n"))
@@ -24596,8 +24742,8 @@ var ConsumerExecutorClass = class {
           reader.destroy();
         });
       }
-      await import_promises9.default.unlink(datasetPath);
-      await import_promises9.default.rename(tempWorkPath, datasetPath);
+      await import_promises8.default.unlink(datasetPath);
+      await import_promises8.default.rename(tempWorkPath, datasetPath);
       return winners.size;
     };
     this.processPivot = async (consumer, datasetPath) => {
@@ -24609,7 +24755,7 @@ var ConsumerExecutorClass = class {
       if (!pivotValues) {
         pivotValues = [];
         const discoverySet = /* @__PURE__ */ new Set();
-        const discoverReader = import_fs11.default.createReadStream(datasetPath);
+        const discoverReader = import_fs10.default.createReadStream(datasetPath);
         const discoverLineReader = import_readline6.default.createInterface({ input: discoverReader, crlfDelay: Infinity });
         for await (const line of discoverLineReader) {
           const record = this._parseLine(line, internalRecordFormat, internalFields);
@@ -24628,7 +24774,7 @@ var ConsumerExecutorClass = class {
         }
       }
       const groups = /* @__PURE__ */ new Map();
-      const reader = import_fs11.default.createReadStream(datasetPath);
+      const reader = import_fs10.default.createReadStream(datasetPath);
       const lineReader = import_readline6.default.createInterface({ input: reader, crlfDelay: Infinity });
       for await (const line of lineReader) {
         const record = this._parseLine(line, internalRecordFormat, internalFields);
@@ -24656,7 +24802,7 @@ var ConsumerExecutorClass = class {
         ...pivotValues.map((pv) => ({ cField: { key: columnPrefix + pv }, finalKey: columnPrefix + pv }))
       ];
       const tempWorkPath = datasetPath + "_tmp";
-      const writer = import_fs11.default.createWriteStream(tempWorkPath);
+      const writer = import_fs10.default.createWriteStream(tempWorkPath);
       const waitForDrain = () => new Promise((resolve) => writer.once("drain", resolve));
       let outputCount = 0;
       for (const { rowRecord, cells } of groups.values()) {
@@ -24702,8 +24848,8 @@ var ConsumerExecutorClass = class {
           reader.destroy();
         });
       }
-      await import_promises9.default.unlink(datasetPath);
-      await import_promises9.default.rename(tempWorkPath, datasetPath);
+      await import_promises8.default.unlink(datasetPath);
+      await import_promises8.default.rename(tempWorkPath, datasetPath);
       return outputCount;
     };
     this._parseLine = (line, format2, fields) => {
@@ -24749,7 +24895,7 @@ var ConsumerExecutorClass = class {
       for (const fieldKey of uniqueFieldKeys) {
         fieldValueSets.set(fieldKey, /* @__PURE__ */ new Set());
       }
-      const reader = import_fs11.default.createReadStream(datasetPath);
+      const reader = import_fs10.default.createReadStream(datasetPath);
       const lineReader = import_readline6.default.createInterface({ input: reader, crlfDelay: Infinity });
       for await (const line of lineReader) {
         rowCount++;
@@ -24797,11 +24943,11 @@ var ConsumerExecutor = new ConsumerExecutorClass();
 var ConsumerExecutor_default = ConsumerExecutor;
 
 // ../../packages/executors/src/Executor.ts
-var import_fs12 = __toESM(require("fs"));
+var import_fs11 = __toESM(require("fs"));
 var import_readline7 = __toESM(require("readline"));
 
 // ../../packages/executors/src/ProducerExecutor.ts
-var import_path17 = __toESM(require("path"));
+var import_path16 = __toESM(require("path"));
 var ProducerExecutorClass = class {
   constructor() {
     this.ready = async (producer, scope) => {
@@ -24827,7 +24973,7 @@ var ProducerExecutorClass = class {
       counter = performance.now();
       for (const dimension of dimensions) {
         if (dimension.prodDimension.sourceFilename === true)
-          record[dimension.name] = import_path17.default.basename(chunk.fileUri);
+          record[dimension.name] = import_path16.default.basename(chunk.fileUri);
         const maskType = ProducerManager_default.getMask(dimension.prodDimension);
         if (Algo_default.hasVal(maskType))
           record[dimension.name] = CryptoEngine_default.hashValue(maskType, record[dimension.name]?.toString(), dimension.prodDimension.type);
@@ -24878,10 +25024,14 @@ var Executor = class {
         elapsedMS: -1,
         inputCount: -1,
         outputCount: -1,
-        resultUri: ExecutorScope_default2.getWorkerPath(scope, workerId),
+        chunkIndex: chunk.index,
+        fileUri: chunk.fileUri,
+        rowOffset: chunk.rowOffset ?? -1,
+        rowCount: -1,
+        resultUri: ExecutorScope_default.getWorkerPath(scope, workerId),
         operations: {}
       };
-      ExecutorScope_default2.ensurePath(result.resultUri);
+      ExecutorScope_default.ensurePath(result.resultUri);
       Logger_default.log(`[${workerId}] Starting execution for producer "${producer.name}" \u2192 consumer "${consumer.name}" (file: ${chunk.fileUri})${recordLimit ? ` (limit: ${recordLimit})` : ""}`);
       let totalOutputCount = 0, totalCycles = 1, perf = 0, lineIndex = 0;
       const readStream = this.openReadStream(chunk);
@@ -24987,17 +25137,18 @@ var Executor = class {
       result.cycles = totalCycles;
       result.inputCount = lineIndex;
       result.outputCount = totalOutputCount;
+      result.rowCount = lineIndex;
       result.operations = this._performance.getOperations();
       Logger_default.log(`[${workerId}] Finished: ${lineIndex} input \u2192 ${totalOutputCount} output in ${Math.round(result.elapsedMS)}ms (${totalCycles} cycle(s))`);
       return result;
     };
     this.openReadStream = (chunk) => {
       const { end, fileUri, start } = chunk;
-      return import_fs12.default.createReadStream(fileUri, { start, end: end - 1 });
+      return import_fs11.default.createReadStream(fileUri, { start, end: end - 1 });
     };
     this.openWriteStream = (scope, workerId) => {
-      const workerPath = ExecutorScope_default2.getWorkerPath(scope, workerId);
-      return import_fs12.default.createWriteStream(workerPath);
+      const workerPath = ExecutorScope_default.getWorkerPath(scope, workerId);
+      return import_fs11.default.createWriteStream(workerPath);
     };
     this.shouldProcessFirstLine = (producer) => {
       Affirm_default(producer, "Invalid producer");
@@ -25028,14 +25179,13 @@ var Executor = class {
 var Executor_default = Executor;
 
 // ../../packages/executors/src/ExecutorOrchestrator.ts
-var import_os = __toESM(require("os"));
-var import_fs13 = __toESM(require("fs"));
-var import_promises10 = __toESM(require("fs/promises"));
+var import_fs12 = __toESM(require("fs"));
+var import_promises9 = __toESM(require("fs/promises"));
 var import_path18 = __toESM(require("path"));
 var import_workerpool = __toESM(require("workerpool"));
 
 // ../../packages/executors/src/ExecutorWriter.ts
-var fs17 = __toESM(require("fs"));
+var fs16 = __toESM(require("fs"));
 var import_readline8 = __toESM(require("readline"));
 var ExecutorWriter = class {
   constructor() {
@@ -25052,11 +25202,11 @@ var ExecutorWriter = class {
     };
     this.splitBySize = async (scope, sourcePath) => {
       const maxOutputFileSize = scope.limitFileSize * this.FAKE_GB;
-      const readStream = fs17.createReadStream(sourcePath);
+      const readStream = fs16.createReadStream(sourcePath);
       const reader = import_readline8.default.createInterface({ input: readStream, crlfDelay: Infinity });
       let writerIndex = 0;
       let destPath = this.getCompletedPath(sourcePath, writerIndex);
-      let writeStream = fs17.createWriteStream(destPath, { flags: "a" });
+      let writeStream = fs16.createWriteStream(destPath, { flags: "a" });
       const waitForDrain = () => new Promise((resolve) => writeStream.once("drain", resolve));
       for await (const line of reader) {
         if (readStream.bytesRead > maxOutputFileSize * (writerIndex + 1)) {
@@ -25067,7 +25217,7 @@ var ExecutorWriter = class {
           });
           writerIndex++;
           destPath = this.getCompletedPath(sourcePath, writerIndex);
-          writeStream = fs17.createWriteStream(destPath, { flags: "a" });
+          writeStream = fs16.createWriteStream(destPath, { flags: "a" });
         }
         if (!writeStream.write(line + "\n"))
           await waitForDrain();
@@ -25077,7 +25227,7 @@ var ExecutorWriter = class {
         writeStream.on("finish", resolve);
         writeStream.on("error", reject);
       });
-      await fs17.promises.unlink(sourcePath);
+      await fs16.promises.unlink(sourcePath);
     };
     /**
      * Manage the Writestream for main.dataset
@@ -25134,7 +25284,7 @@ var ExecutorWriter = class {
 var ExecutorWriter_default = ExecutorWriter;
 
 // ../../packages/executors/src/ExecutorOrchestrator.ts
-var import_promises11 = require("stream/promises");
+var import_promises10 = require("stream/promises");
 
 // ../../packages/executors/src/cli_progress/ExecutorProgress2.ts
 var ExecutorProgress2 = class {
@@ -25170,19 +25320,111 @@ var ExecutorProgress2 = class {
 };
 var ExecutorProgress2_default = ExecutorProgress2;
 
+// ../../packages/executors/src/OrchestratorHelper.ts
+var import_os = __toESM(require("os"));
+var import_path17 = __toESM(require("path"));
+var OrchestratorHelper = {
+  getMemoryUsage: () => {
+    const processMemory = process.memoryUsage();
+    const freeSystemMemory = import_os.default.freemem();
+    return {
+      /**
+       * resident set size (heap + code + stack)
+       */
+      rss: Formatter_default.bytes(processMemory.rss),
+      heapUsed: Formatter_default.bytes(processMemory.heapUsed),
+      heapTotal: Formatter_default.bytes(processMemory.heapTotal),
+      heapPercent: Algo_default.round(processMemory.heapUsed / processMemory.heapTotal, 1),
+      external: Formatter_default.bytes(processMemory.external),
+      free: Formatter_default.bytes(freeSystemMemory)
+    };
+  },
+  formatMemoryUsage: () => {
+    return `Memory [total: ${OrchestratorHelper.getMemoryUsage().rss} - heap: ${OrchestratorHelper.getMemoryUsage().heapPercent}%]`;
+  },
+  computeFinalResult: (tracker, executorResults, executionId, resultUri) => {
+    const result = {
+      cycles: Algo_default.max(executorResults.map((x) => x.cycles)),
+      elapsedMS: Algo_default.sum(executorResults.map((x) => x.elapsedMS)),
+      inputCount: Algo_default.sum(executorResults.map((x) => x.inputCount)),
+      outputCount: Algo_default.sum(executorResults.map((x) => x.outputCount)),
+      workerCount: executorResults.length,
+      executionId,
+      resultUri,
+      operations: {}
+    };
+    for (const res of executorResults) {
+      for (const opKey of Object.keys(res.operations)) {
+        const op = res.operations[opKey];
+        let label = result.operations[opKey];
+        if (!label) {
+          result.operations[opKey] = { avg: -1, max: -1, min: -1, elapsedMS: [] };
+          label = result.operations[opKey];
+        }
+        label.elapsedMS.push(op.elapsedMS);
+      }
+      for (const opKey of Object.keys(result.operations)) {
+        const operation = result.operations[opKey];
+        if (operation.elapsedMS.length > 0) {
+          operation.min = Math.min(...operation.elapsedMS);
+          operation.max = Math.max(...operation.elapsedMS);
+          operation.avg = Algo_default.mean(operation.elapsedMS);
+        }
+      }
+    }
+    const trackerOperations = tracker.getOperations();
+    for (const opKey of Object.keys(trackerOperations)) {
+      const trackerOp = trackerOperations[opKey];
+      const value = trackerOp.elapsedMS;
+      if (!result.operations[opKey]) {
+        result.operations[opKey] = { avg: value, max: value, min: value, elapsedMS: [] };
+      }
+      result.operations[opKey].elapsedMS.push(value);
+    }
+    return result;
+  },
+  /**
+   * Returns the path to the worker thread file in the build (different between dev, cli and prod (docker)).
+   * IMPORTANT!: when moving this (OrchestratorHelper.ts) file, or the workers output, you have to make sure to update these paths
+   */
+  getPhysicalWorkerPath: () => {
+    const currentDir = __dirname;
+    if (ProcessENVManager_default.getEnvVariable("NODE_ENV") === "dev" || ProcessENVManager_default.getEnvVariable("NODE_ENV") === "development")
+      return import_path17.default.resolve("./.build/workers");
+    const forcedPath = ProcessENVManager_default.getEnvVariable("REMORA_WORKERS_PATH");
+    if (forcedPath && forcedPath.length > 0)
+      return import_path17.default.join(__dirname, forcedPath);
+    if (!currentDir.includes(".build")) {
+      return import_path17.default.join(__dirname, "../workers");
+    } else {
+      return import_path17.default.resolve("./.build/workers");
+    }
+  },
+  getParallelWorkerCount: () => {
+    const cpuBoundWorkers = Math.max(1, Math.floor(import_os.default.cpus().length * 0.7));
+    const totalMemoryMB = Math.floor(import_os.default.totalmem() / (1024 * 1024));
+    const reservedMemoryMB = Constants_default.defaults.MIN_RUNTIME_HEAP_MB * 2;
+    const availableMemoryForWorkersMB = Math.max(Constants_default.defaults.MIN_RUNTIME_HEAP_MB, totalMemoryMB - reservedMemoryMB);
+    const memoryBoundWorkers = Math.max(1, Math.floor(availableMemoryForWorkersMB / Constants_default.defaults.MIN_RUNTIME_HEAP_MB));
+    return Math.min(cpuBoundWorkers, Constants_default.defaults.MAX_THREAD_COUNT, memoryBoundWorkers);
+  }
+};
+var OrchestratorHelper_default = OrchestratorHelper;
+
 // ../../packages/executors/src/ExecutorOrchestrator.ts
 var ExecutorOrchestratorClass = class {
   constructor() {
-    this.createPool = () => {
+    this.createPool = (maxWorkers) => {
       const options = {
+        maxWorkers,
         workerThreadOpts: {
           resourceLimits: {
             maxOldGenerationSizeMb: Constants_default.defaults.MIN_RUNTIME_HEAP_MB
           }
         }
       };
-      const workerPath = this._getWorkerPath();
-      Logger_default.log(`Initializing worker pool from ${workerPath} (heap limit: ${Constants_default.defaults.MIN_RUNTIME_HEAP_MB}MB)`);
+      const workerPath = OrchestratorHelper_default.getPhysicalWorkerPath();
+      Logger_default.log(`Initializing worker pool from ${workerPath} (workers: ${maxWorkers}, heap limit: ${Constants_default.defaults.MIN_RUNTIME_HEAP_MB}MB) | ${OrchestratorHelper_default.formatMemoryUsage()}`);
       return import_workerpool.default.pool(import_path18.default.join(workerPath, "ExecutorWorker.js"), options);
     };
     this.launch = async (request) => {
@@ -25196,11 +25438,12 @@ var ExecutorOrchestratorClass = class {
       const _progress = new ExecutorProgress2_default(logProgress);
       const { usageId } = UsageManager_default.startUsage(consumer, details);
       const scope = { id: usageId, folder: `${consumer.name}_${usageId}`, workersId: [], limitFileSize: consumer.maximumFileSize };
-      const pool = this.createPool();
+      const expandedFields = ConsumerManager_default.getExpandedFields(consumer);
+      let activePool = null;
       try {
         const start = performance.now();
         const executorResults = [];
-        Logger_default.log(`[${usageId}] Launching consumer "${consumer.name}" (invoked by: ${details.invokedBy}, user: ${details.user?.name ?? "unknown"}, producer(s): ${consumer.producers.length})`);
+        Logger_default.log(`[${usageId}] Launching consumer "${consumer.name}" (invoked by: ${details.invokedBy}, user: ${details.user?.name ?? "unknown"}, producer(s): ${consumer.producers.length}) | ${OrchestratorHelper_default.formatMemoryUsage()}`);
         let counter = performance.now();
         _progress.update({ phase: "Preparing source data", progress: 0 });
         let sourceFilesByProducer = await this.readySourceFiles(consumer, scope);
@@ -25216,10 +25459,10 @@ var ExecutorOrchestratorClass = class {
         let globalWorkerIndex = 0;
         for (const pair of sourceFilesByProducer) {
           const { prod, cProd, response } = pair;
-          if (!import_fs13.default.existsSync(response.files[0].fullUri)) {
-            if (!cProd.isOptional)
+          if (!import_fs12.default.existsSync(response.files[0].fullUri)) {
+            if (!cProd.isOptional) {
               throw new Error(`Expected data file ${response.files[0].fullUri} of producer ${prod.name} in consumer ${consumer.name} is missing.`);
-            else if (cProd.isOptional === true) {
+            } else if (cProd.isOptional === true) {
               Logger_default.log(`[${usageId}] Producer "${prod.name}": data file missing but marked optional, skipping`);
               continue;
             }
@@ -25232,35 +25475,42 @@ var ExecutorOrchestratorClass = class {
           for (const [fileIndex, file] of response.files.entries()) {
             const chunks = ExecutorOrchestrator.scopeWork(file.fullUri);
             const workerThreads = [];
-            Logger_default.log(`[${usageId}] Producer "${prod.name}" file ${fileIndex + 1}/${totalFiles}: split into ${chunks.length} chunk(s)`);
-            for (const chunk of chunks) {
-              const workerId = `${usageId}_${globalWorkerIndex}`;
-              globalWorkerIndex++;
-              const workerData = {
-                producer: prod,
-                chunk,
-                consumer,
-                prodDimensions,
-                workerId,
-                scope,
-                options,
-                loggerConfig: Logger_default.getConfig()
-              };
-              scope.workersId.push(workerId);
-              Logger_default.log(`[${usageId}] Spawning worker ${workerId} for producer "${prod.name}" \u2014 chunk ${chunk.start}-${chunk.end} (${Math.round((chunk.end - chunk.start) / 1024)}KB)`);
-              workerThreads.push(pool.exec("executor", [workerData], {
-                on: (payload) => this.onWorkAdvanced(payload, workerId, _progress, totalBytesToProcess, bytesProcessedByWorker)
-              }).catch((error) => {
-                Logger_default.error(error);
-                return null;
-              }));
+            activePool = this.createPool(chunks.length);
+            Logger_default.log(`[${usageId}] Producer "${prod.name}" file ${fileIndex + 1}/${totalFiles}: split into ${chunks.length} chunk(s) | ${OrchestratorHelper_default.formatMemoryUsage()}`);
+            try {
+              for (const chunk of chunks) {
+                const workerId = `${usageId}_${globalWorkerIndex}`;
+                globalWorkerIndex++;
+                const workerData = {
+                  producer: prod,
+                  chunk,
+                  consumer,
+                  prodDimensions,
+                  workerId,
+                  scope,
+                  options,
+                  loggerConfig: Logger_default.getConfig()
+                };
+                scope.workersId.push(workerId);
+                Logger_default.log(`[${usageId}] Spawning worker ${workerId} for producer "${prod.name}" \u2014 chunk ${chunk.start}-${chunk.end} (${Math.round((chunk.end - chunk.start) / 1024)}KB)`);
+                workerThreads.push(activePool.exec("executor", [workerData], {
+                  on: (payload) => this.onWorkAdvanced(payload, workerId, _progress, totalBytesToProcess, bytesProcessedByWorker)
+                }).catch((error) => {
+                  Logger_default.error(error);
+                  return null;
+                }));
+              }
+              Logger_default.log(`[${usageId}] Waiting for ${workerThreads.length} worker(s) to complete  | ${OrchestratorHelper_default.formatMemoryUsage()}`);
+              const fileExecutorResults = await Promise.all(workerThreads);
+              this.assignChunkRowMetadata(chunks, fileExecutorResults);
+              executorResults.push(...fileExecutorResults);
+              Logger_default.log(`[${usageId}] All ${workerThreads.length} worker(s) finished for producer "${prod.name}" file ${fileIndex + 1}/${totalFiles} | ${OrchestratorHelper_default.formatMemoryUsage()}`);
+            } finally {
+              await activePool.terminate();
+              activePool = null;
             }
-            Logger_default.log(`[${usageId}] Waiting for ${workerThreads.length} worker(s) to complete`);
-            executorResults.push(...await Promise.all(workerThreads));
-            Logger_default.log(`[${usageId}] All ${workerThreads.length} worker(s) finished for producer "${prod.name}" file ${fileIndex + 1}/${totalFiles}`);
           }
         }
-        await pool.terminate();
         _progress.update({ phase: "Processing data", progress: 1 });
         if (executorResults.some((x) => !Algo_default.hasVal(x)))
           throw new Error(`${executorResults.filter((x) => !Algo_default.hasVal(x)).length} worker(s) failed to produce valid results`);
@@ -25275,7 +25525,7 @@ var ExecutorOrchestratorClass = class {
           if (consumer.options?.distinct === true) {
             Logger_default.log(`[${usageId}] Running unified distinct pass across merged workers`);
             counter = performance.now();
-            const unifiedOutputCount = await ConsumerExecutor_default.processDistinct(ExecutorScope_default2.getMainPath(scope));
+            const unifiedOutputCount = await ConsumerExecutor_default.processDistinct(ExecutorScope_default.getMainPath(scope));
             tracker.measure("process-distinct:main", performance.now() - counter);
             postOperation.totalOutputCount = unifiedOutputCount;
             Logger_default.log(`[${usageId}] Distinct pass complete: ${unifiedOutputCount} unique rows in ${Math.round(performance.now() - counter)}ms`);
@@ -25283,24 +25533,31 @@ var ExecutorOrchestratorClass = class {
           if (consumer.options?.distinctOn) {
             Logger_default.log(`[${usageId}] Running unified distinctOn pass (${consumer.options.distinctOn})`);
             counter = performance.now();
-            const unifiedOutputCount = await ConsumerExecutor_default.processDistinctOn(consumer, ExecutorScope_default2.getMainPath(scope));
+            const unifiedOutputCount = await ConsumerExecutor_default.processDistinctOn(consumer, ExecutorScope_default.getMainPath(scope));
             tracker.measure("process-distinct-on:main", performance.now() - counter);
             postOperation.totalOutputCount = unifiedOutputCount;
             Logger_default.log(`[${usageId}] DistinctOn pass complete: ${unifiedOutputCount} rows in ${Math.round(performance.now() - counter)}ms`);
           }
-        }
-        if (consumer.options?.pivot) {
-          Logger_default.log(`[${usageId}] Running pivot operation`);
-          counter = performance.now();
-          const unifiedOutputCount = await ConsumerExecutor_default.processPivot(consumer, ExecutorScope_default2.getMainPath(scope));
-          tracker.measure("process-pivot:main", performance.now() - counter);
-          postOperation.totalOutputCount = unifiedOutputCount;
-          Logger_default.log(`[${usageId}] Pivot complete: ${unifiedOutputCount} rows in ${Math.round(performance.now() - counter)}ms`);
+          if (consumer.options?.pivot) {
+            Logger_default.log(`[${usageId}] Running pivot operation`);
+            counter = performance.now();
+            const unifiedOutputCount = await ConsumerExecutor_default.processPivot(consumer, ExecutorScope_default.getMainPath(scope));
+            tracker.measure("process-pivot:main", performance.now() - counter);
+            postOperation.totalOutputCount = unifiedOutputCount;
+            Logger_default.log(`[${usageId}] Pivot complete: ${unifiedOutputCount} rows in ${Math.round(performance.now() - counter)}ms`);
+          }
+          if (ConsumerManager_default.hasLineCountFields(consumer, true)) {
+            Logger_default.log(`[${usageId}] Running unified lineCount pass across merged workers`);
+            counter = performance.now();
+            const processedLineCount = await ConsumerExecutor_default.processLineCount(consumer, ExecutorScope_default.getMainPath(scope), executorResults);
+            tracker.measure("process-line-count:main", performance.now() - counter);
+            Logger_default.log(`[${usageId}] LineCount pass complete: normalized ${processedLineCount} rows in ${Math.round(performance.now() - counter)}ms`);
+          }
         }
         if (consumer.validate && consumer.validate.length > 0) {
           Logger_default.log(`[${usageId}] Running dataset-level validations`);
           counter = performance.now();
-          const validationResults = await ConsumerExecutor_default.processDatasetValidation(consumer, ExecutorScope_default2.getMainPath(scope));
+          const validationResults = await ConsumerExecutor_default.processDatasetValidation(consumer, ExecutorScope_default.getMainPath(scope));
           tracker.measure("dataset-validation", performance.now() - counter);
           for (const result of validationResults) {
             if (result.onFail === "fail") {
@@ -25317,7 +25574,7 @@ var ExecutorOrchestratorClass = class {
           Logger_default.log(`[${usageId}] Splitting output by size limit (${scope.limitFileSize})`);
           counter = performance.now();
           const writer = new ExecutorWriter_default();
-          await writer.splitBySize(scope, ExecutorScope_default2.getMainPath(scope));
+          await writer.splitBySize(scope, ExecutorScope_default.getMainPath(scope));
           tracker.measure("split-by-size", performance.now() - counter);
           Logger_default.log(`[${usageId}] Split complete in ${Math.round(performance.now() - counter)}ms`);
         }
@@ -25325,7 +25582,7 @@ var ExecutorOrchestratorClass = class {
         _progress.update({ phase: "Exporting result", progress: 0 });
         counter = performance.now();
         Logger_default.log(`[${usageId}] Exporting results to ${consumer.outputs.length} output(s)`);
-        const exportRes = await OutputExecutor_default.exportResult(consumer, ConsumerManager_default.getExpandedFields(consumer), scope);
+        const exportRes = await OutputExecutor_default.exportResult(consumer, expandedFields, scope);
         tracker.measure("export-result", performance.now() - counter);
         Logger_default.log(`[${usageId}] Export complete in ${Math.round(performance.now() - counter)}ms (key: ${exportRes.key})`);
         _progress.update({ phase: "Exporting result", progress: 1 });
@@ -25336,9 +25593,9 @@ var ExecutorOrchestratorClass = class {
           tracker.measure("on-success-actions", performance.now() - counter);
           Logger_default.log(`[${usageId}] On-success actions complete in ${Math.round(performance.now() - counter)}ms`);
         }
-        Logger_default.log(`[${usageId}] Starting cleanup operations`);
+        Logger_default.log(`[${usageId}] Starting cleanup operations | ${OrchestratorHelper_default.formatMemoryUsage()}`);
         await this.performCleanupOperations(scope, tracker);
-        const finalResult = this.computeFinalResult(tracker, executorResults, usageId, exportRes.key);
+        const finalResult = OrchestratorHelper_default.computeFinalResult(tracker, executorResults, usageId, exportRes.key);
         finalResult.elapsedMS = performance.now() - start;
         if (Algo_default.hasVal(postOperation.totalOutputCount))
           finalResult.outputCount = postOperation.totalOutputCount;
@@ -25347,9 +25604,10 @@ var ExecutorOrchestratorClass = class {
         await Logger_default.flush();
         return finalResult;
       } catch (error) {
-        Logger_default.log(`[${usageId}] Consumer "${consumer.name}" failed: ${Helper_default.asError(error).message}`);
+        Logger_default.log(`[${usageId}] Consumer "${consumer.name}" failed: ${Helper_default.asError(error).message} | ${OrchestratorHelper_default.formatMemoryUsage()}`);
         Logger_default.error(error);
-        await pool.terminate();
+        if (activePool)
+          await activePool.terminate();
         await ConsumerOnFinishManager_default.onConsumerError(consumer, usageId);
         Logger_default.log(`[${usageId}] Running cleanup after failure`);
         await this.performCleanupOperations(scope, tracker);
@@ -25364,18 +25622,17 @@ var ExecutorOrchestratorClass = class {
      * Returns a single chunk for small files where parallelism overhead isn't worth it.
      */
     this.scopeWork = (fileUri, numChunks) => {
-      const fileSize = import_fs13.default.statSync(fileUri).size;
+      const fileSize = import_fs12.default.statSync(fileUri).size;
       if (fileSize === 0) return [];
       if (fileSize < Constants_default.defaults.MIN_FILE_SIZE_FOR_PARALLEL) {
         return [{ start: 0, end: fileSize, isFirstChunk: true, fileUri, index: 0 }];
       }
-      const availableCores = Math.max(1, Math.floor(import_os.default.cpus().length * 0.7));
-      const cpus = numChunks ?? Math.min(availableCores, Constants_default.defaults.MAX_THREAD_COUNT);
+      const targetWorkers = numChunks ?? OrchestratorHelper_default.getParallelWorkerCount();
       const maxChunksBySize = Math.floor(fileSize / Constants_default.defaults.MIN_CHUNK_SIZE);
-      const effectiveChunks = Math.min(cpus, maxChunksBySize);
+      const effectiveChunks = Math.min(targetWorkers, maxChunksBySize);
       if (effectiveChunks <= 1) return [{ start: 0, end: fileSize, isFirstChunk: true, fileUri, index: 0 }];
       const targetChunkSize = Math.floor(fileSize / effectiveChunks);
-      const fd = import_fs13.default.openSync(fileUri, "r");
+      const fd = import_fs12.default.openSync(fileUri, "r");
       try {
         const offsets = [];
         let currentStart = 0;
@@ -25393,7 +25650,23 @@ var ExecutorOrchestratorClass = class {
         }
         return offsets;
       } finally {
-        import_fs13.default.closeSync(fd);
+        import_fs12.default.closeSync(fd);
+      }
+    };
+    this.assignChunkRowMetadata = (chunks, executorResults) => {
+      const validResults = executorResults.filter((result) => Algo_default.hasVal(result));
+      const resultsByChunkIndex = new Map(validResults.map((result) => [result.chunkIndex, result]));
+      let nextRowOffset = 0;
+      for (const chunk of chunks) {
+        const result = resultsByChunkIndex.get(chunk.index);
+        if (!result)
+          continue;
+        const rowCount = result.inputCount;
+        chunk.rowOffset = nextRowOffset;
+        chunk.rowCount = rowCount;
+        result.rowOffset = nextRowOffset;
+        result.rowCount = rowCount;
+        nextRowOffset += rowCount;
       }
     };
     /**
@@ -25406,7 +25679,7 @@ var ExecutorOrchestratorClass = class {
       let currentPos = position;
       while (currentPos < fileSize) {
         const bytesToRead = Math.min(BUFFER_SIZE, fileSize - currentPos);
-        const bytesRead = import_fs13.default.readSync(fd, buffer, 0, bytesToRead, currentPos);
+        const bytesRead = import_fs12.default.readSync(fd, buffer, 0, bytesToRead, currentPos);
         if (bytesRead === 0) break;
         for (let i = 0; i < bytesRead; i++) {
           if (buffer[i] === 10) {
@@ -25470,21 +25743,21 @@ var ExecutorOrchestratorClass = class {
               startRow: prod.settings.startRow,
               startColumn: prod.settings.startColumn
             });
-            await (0, import_promises11.pipeline)(
+            await (0, import_promises10.pipeline)(
               xlsCsvStream,
-              import_fs13.default.createWriteStream(decodedPath)
+              import_fs12.default.createWriteStream(decodedPath)
             );
-            const fileStats = await import_promises10.default.stat(decodedPath);
+            const fileStats = await import_promises9.default.stat(decodedPath);
             decodedFiles.push({ fullUri: decodedPath, fileSize: fileStats.size });
             decodedCount++;
             continue;
           }
           if (inferredType === "XML") {
-            const fileContent = await import_promises10.default.readFile(file.fullUri, "utf-8");
+            const fileContent = await import_promises9.default.readFile(file.fullUri, "utf-8");
             const jsonData = XMLParser_default.xmlToJson(fileContent);
             const records = normalizeXmlRows(jsonData);
             if (records.length === 0) {
-              await import_promises10.default.writeFile(decodedPath, "", "utf-8");
+              await import_promises9.default.writeFile(decodedPath, "", "utf-8");
             } else {
               const columns = [];
               for (const record of records) {
@@ -25500,9 +25773,9 @@ var ExecutorOrchestratorClass = class {
                 const row = columns.map((column) => csvSafeValue(record[column]));
                 lines.push(CSVParser_default.stringifyRow(row));
               }
-              await import_promises10.default.writeFile(decodedPath, lines.join("\n"), "utf-8");
+              await import_promises9.default.writeFile(decodedPath, lines.join("\n"), "utf-8");
             }
-            const fileStats = await import_promises10.default.stat(decodedPath);
+            const fileStats = await import_promises9.default.stat(decodedPath);
             decodedFiles.push({ fullUri: decodedPath, fileSize: fileStats.size });
             decodedCount++;
             continue;
@@ -25521,86 +25794,32 @@ var ExecutorOrchestratorClass = class {
       }));
       return decodedResults;
     };
-    this._getWorkerPath = () => {
-      const currentDir = __dirname;
-      if (ProcessENVManager_default.getEnvVariable("NODE_ENV") === "dev" || ProcessENVManager_default.getEnvVariable("NODE_ENV") === "development")
-        return import_path18.default.resolve("./.build/workers");
-      const forcedPath = ProcessENVManager_default.getEnvVariable("REMORA_WORKERS_PATH");
-      if (forcedPath && forcedPath.length > 0)
-        return import_path18.default.join(__dirname, forcedPath);
-      if (!currentDir.includes(".build")) {
-        return import_path18.default.join(__dirname, "../workers");
-      } else {
-        return import_path18.default.resolve("./.build/workers");
-      }
-    };
     this.reconcileExecutorThreadsResults = async (scope, executorResults, tracker) => {
-      const mainPath = ExecutorScope_default2.getMainPath(scope);
+      const mainPath = ExecutorScope_default.getMainPath(scope);
       ConsumerExecutor_default._ensurePath(mainPath);
       if (executorResults.length > 1) {
         Logger_default.log(`[${scope.id}] Merging ${executorResults.length} worker output files into ${mainPath}`);
         const perf = performance.now();
         for (const workerResult of executorResults) {
-          await (0, import_promises11.pipeline)(
-            import_fs13.default.createReadStream(workerResult.resultUri),
-            import_fs13.default.createWriteStream(mainPath, { flags: "a" })
+          await (0, import_promises10.pipeline)(
+            import_fs12.default.createReadStream(workerResult.resultUri),
+            import_fs12.default.createWriteStream(mainPath, { flags: "a" })
           );
-          await import_promises10.default.unlink(workerResult.resultUri);
+          await import_promises9.default.unlink(workerResult.resultUri);
         }
         tracker.measure("merge-workers", performance.now() - perf);
         Logger_default.log(`[${scope.id}] Merge complete in ${Math.round(performance.now() - perf)}ms`);
       } else if (executorResults.length === 1) {
         Logger_default.log(`[${scope.id}] Single worker \u2014 renaming output to ${mainPath}`);
-        await import_promises10.default.rename(executorResults[0].resultUri, mainPath);
+        await import_promises9.default.rename(executorResults[0].resultUri, mainPath);
       }
       return mainPath;
     };
     this.performCleanupOperations = async (scope, tracker) => {
       const start = performance.now();
-      await ExecutorScope_default2.clearScope(scope);
+      await ExecutorScope_default.clearScope(scope);
       tracker.measure("cleanup-operations", performance.now() - start);
     };
-    this.computeFinalResult = (tracker, executorResults, executionId, resultUri) => {
-      const result = {
-        cycles: Algo_default.max(executorResults.map((x) => x.cycles)),
-        elapsedMS: Algo_default.sum(executorResults.map((x) => x.elapsedMS)),
-        inputCount: Algo_default.sum(executorResults.map((x) => x.inputCount)),
-        outputCount: Algo_default.sum(executorResults.map((x) => x.outputCount)),
-        workerCount: executorResults.length,
-        executionId,
-        resultUri,
-        operations: {}
-      };
-      for (const res of executorResults) {
-        for (const opKey of Object.keys(res.operations)) {
-          const op = res.operations[opKey];
-          let label = result.operations[opKey];
-          if (!label) {
-            result.operations[opKey] = { avg: -1, max: -1, min: -1, elapsedMS: [] };
-            label = result.operations[opKey];
-          }
-          label.elapsedMS.push(op.elapsedMS);
-        }
-        for (const opKey of Object.keys(result.operations)) {
-          const operation = result.operations[opKey];
-          if (operation.elapsedMS.length > 0) {
-            operation.min = Math.min(...operation.elapsedMS);
-            operation.max = Math.max(...operation.elapsedMS);
-            operation.avg = Algo_default.mean(operation.elapsedMS);
-          }
-        }
-      }
-      const trackerOperations = tracker.getOperations();
-      for (const opKey of Object.keys(trackerOperations)) {
-        const trackerOp = trackerOperations[opKey];
-        const value = trackerOp.elapsedMS;
-        if (!result.operations[opKey]) {
-          result.operations[opKey] = { avg: value, max: value, min: value, elapsedMS: [] };
-        }
-        result.operations[opKey].elapsedMS.push(value);
-      }
-      return result;
-    };
     this.onWorkAdvanced = (packet, workerId, progress, totalBytesToProcess, bytesProcessedByWorker) => {
       const { processed } = packet;
       bytesProcessedByWorker[workerId] = processed;