@forzalabs/remora 1.2.1 → 1.2.2

package/workers/ExecutorWorker.js

@@ -146,7 +146,7 @@ var require_has_flag = __commonJS({
 var require_supports_colors = __commonJS({
   "../../node_modules/logform/node_modules/@colors/colors/lib/system/supports-colors.js"(exports2, module2) {
     "use strict";
-    var os2 = require("os");
+    var os3 = require("os");
     var hasFlag = require_has_flag();
     var env = process.env;
     var forceColor = void 0;
@@ -184,7 +184,7 @@ var require_supports_colors = __commonJS({
       }
       var min = forceColor ? 1 : 0;
       if (process.platform === "win32") {
-        var osRelease = os2.release().split(".");
+        var osRelease = os3.release().split(".");
         if (Number(process.versions.node.split(".")[0]) >= 8 && Number(osRelease[0]) >= 10 && Number(osRelease[2]) >= 10586) {
           return Number(osRelease[2]) >= 14931 ? 3 : 2;
         }
@@ -5428,7 +5428,7 @@ var require_winston_transport = __commonJS({
 var require_console = __commonJS({
   "../../packages/logger/node_modules/winston/lib/winston/transports/console.js"(exports2, module2) {
     "use strict";
-    var os2 = require("os");
+    var os3 = require("os");
     var { LEVEL, MESSAGE } = require_triple_beam();
     var TransportStream = require_winston_transport();
     module2.exports = class Console extends TransportStream {
@@ -5442,7 +5442,7 @@ var require_console = __commonJS({
         this.name = options.name || "console";
         this.stderrLevels = this._stringArrayToSet(options.stderrLevels);
         this.consoleWarnLevels = this._stringArrayToSet(options.consoleWarnLevels);
-        this.eol = typeof options.eol === "string" ? options.eol : os2.EOL;
+        this.eol = typeof options.eol === "string" ? options.eol : os3.EOL;
         this.forceConsole = options.forceConsole || false;
         this._consoleLog = console.log.bind(console);
         this._consoleWarn = console.warn.bind(console);
@@ -9654,15 +9654,15 @@ var require_index_cjs = __commonJS({
       };
     }
     function wrapConversion(toModel, graph) {
-      const path17 = [graph[toModel].parent, toModel];
+      const path18 = [graph[toModel].parent, toModel];
       let fn = convert$1[graph[toModel].parent][toModel];
       let cur = graph[toModel].parent;
       while (graph[cur].parent) {
-        path17.unshift(graph[cur].parent);
+        path18.unshift(graph[cur].parent);
         fn = link(convert$1[graph[cur].parent][cur], fn);
         cur = graph[cur].parent;
       }
-      fn.conversion = path17;
+      fn.conversion = path18;
       return fn;
     }
     function route(fromModel) {
@@ -10292,7 +10292,7 @@ var require_node2 = __commonJS({
 var require_tail_file = __commonJS({
   "../../packages/logger/node_modules/winston/lib/winston/tail-file.js"(exports2, module2) {
     "use strict";
-    var fs18 = require("fs");
+    var fs19 = require("fs");
     var { StringDecoder } = require("string_decoder");
     var { Stream } = require_readable();
     function noop() {
@@ -10313,7 +10313,7 @@ var require_tail_file = __commonJS({
         stream.emit("end");
         stream.emit("close");
       };
-      fs18.open(options.file, "a+", "0644", (err, fd) => {
+      fs19.open(options.file, "a+", "0644", (err, fd) => {
         if (err) {
           if (!iter) {
             stream.emit("error", err);
@@ -10325,10 +10325,10 @@ var require_tail_file = __commonJS({
         }
         (function read() {
           if (stream.destroyed) {
-            fs18.close(fd, noop);
+            fs19.close(fd, noop);
             return;
           }
-          return fs18.read(fd, buffer, 0, buffer.length, pos, (error, bytes) => {
+          return fs19.read(fd, buffer, 0, buffer.length, pos, (error, bytes) => {
             if (error) {
               if (!iter) {
                 stream.emit("error", error);
@@ -10387,15 +10387,15 @@ var require_tail_file = __commonJS({
 var require_file = __commonJS({
   "../../packages/logger/node_modules/winston/lib/winston/transports/file.js"(exports2, module2) {
     "use strict";
-    var fs18 = require("fs");
-    var path17 = require("path");
+    var fs19 = require("fs");
+    var path18 = require("path");
     var asyncSeries = require_series();
     var zlib2 = require("zlib");
     var { MESSAGE } = require_triple_beam();
     var { Stream, PassThrough } = require_readable();
     var TransportStream = require_winston_transport();
     var debug = require_node2()("winston:file");
-    var os2 = require("os");
+    var os3 = require("os");
     var tailFile = require_tail_file();
     module2.exports = class File extends TransportStream {
       /**
@@ -10418,14 +10418,14 @@ var require_file = __commonJS({
         this._onError = this._onError.bind(this);
         if (options.filename || options.dirname) {
           throwIf("filename or dirname", "stream");
-          this._basename = this.filename = options.filename ? path17.basename(options.filename) : "winston.log";
-          this.dirname = options.dirname || path17.dirname(options.filename);
+          this._basename = this.filename = options.filename ? path18.basename(options.filename) : "winston.log";
+          this.dirname = options.dirname || path18.dirname(options.filename);
           this.options = options.options || { flags: "a" };
         } else if (options.stream) {
           console.warn("options.stream will be removed in winston@4. Use winston.transports.Stream");
           throwIf("stream", "filename", "maxsize");
           this._dest = this._stream.pipe(this._setupStream(options.stream));
-          this.dirname = path17.dirname(this._dest.path);
+          this.dirname = path18.dirname(this._dest.path);
         } else {
           throw new Error("Cannot log to file without filename or stream.");
         }
@@ -10433,7 +10433,7 @@ var require_file = __commonJS({
         this.rotationFormat = options.rotationFormat || false;
         this.zippedArchive = options.zippedArchive || false;
         this.maxFiles = options.maxFiles || null;
-        this.eol = typeof options.eol === "string" ? options.eol : os2.EOL;
+        this.eol = typeof options.eol === "string" ? options.eol : os3.EOL;
         this.tailable = options.tailable || false;
         this.lazy = options.lazy || false;
         this._size = 0;
@@ -10588,11 +10588,11 @@ var require_file = __commonJS({
           options = {};
         }
         options = normalizeQuery(options);
-        const file = path17.join(this.dirname, this.filename);
+        const file = path18.join(this.dirname, this.filename);
         let buff = "";
         let results = [];
         let row = 0;
-        const stream = fs18.createReadStream(file, {
+        const stream = fs19.createReadStream(file, {
           encoding: "utf8"
         });
         stream.on("error", (err) => {
@@ -10693,7 +10693,7 @@ var require_file = __commonJS({
        * TODO: Refactor me.
        */
       stream(options = {}) {
-        const file = path17.join(this.dirname, this.filename);
+        const file = path18.join(this.dirname, this.filename);
         const stream = new Stream();
         const tail = {
           file,
@@ -10743,8 +10743,8 @@ var require_file = __commonJS({
        */
       stat(callback) {
         const target = this._getFile();
-        const fullpath = path17.join(this.dirname, target);
-        fs18.stat(fullpath, (err, stat) => {
+        const fullpath = path18.join(this.dirname, target);
+        fs19.stat(fullpath, (err, stat) => {
           if (err && err.code === "ENOENT") {
             debug("ENOENT\xA0ok", fullpath);
             this.filename = target;
@@ -10847,9 +10847,9 @@ var require_file = __commonJS({
        * @returns {WritableStream} Stream that writes to disk for the active file.
        */
       _createStream(source) {
-        const fullpath = path17.join(this.dirname, this.filename);
+        const fullpath = path18.join(this.dirname, this.filename);
         debug("create stream start", fullpath, this.options);
-        const dest = fs18.createWriteStream(fullpath, this.options).on("error", (err) => debug(err)).on("close", () => debug("close", dest.path, dest.bytesWritten)).on("open", () => {
+        const dest = fs19.createWriteStream(fullpath, this.options).on("error", (err) => debug(err)).on("close", () => debug("close", dest.path, dest.bytesWritten)).on("open", () => {
           debug("file open ok", fullpath);
           this.emit("open", fullpath);
           source.pipe(dest);
@@ -10872,16 +10872,16 @@ var require_file = __commonJS({
        */
       _incFile(callback) {
         debug("_incFile", this.filename);
-        const ext = path17.extname(this._basename);
-        const basename = path17.basename(this._basename, ext);
+        const ext = path18.extname(this._basename);
+        const basename = path18.basename(this._basename, ext);
         const tasks = [];
         if (this.zippedArchive) {
           tasks.push(
             function(cb) {
               const num = this._created > 0 && !this.tailable ? this._created : "";
               this._compressFile(
-                path17.join(this.dirname, `${basename}${num}${ext}`),
-                path17.join(this.dirname, `${basename}${num}${ext}.gz`),
+                path18.join(this.dirname, `${basename}${num}${ext}`),
+                path18.join(this.dirname, `${basename}${num}${ext}.gz`),
                 cb
               );
             }.bind(this)
@@ -10906,8 +10906,8 @@ var require_file = __commonJS({
        * @private
        */
       _getFile() {
-        const ext = path17.extname(this._basename);
-        const basename = path17.basename(this._basename, ext);
+        const ext = path18.extname(this._basename);
+        const basename = path18.basename(this._basename, ext);
         const isRotation = this.rotationFormat ? this.rotationFormat() : this._created;
         return !this.tailable && this._created ? `${basename}${isRotation}${ext}` : `${basename}${ext}`;
       }
@@ -10927,8 +10927,8 @@ var require_file = __commonJS({
         const isOldest = oldest !== 0 ? oldest : "";
         const isZipped = this.zippedArchive ? ".gz" : "";
         const filePath = `${basename}${isOldest}${ext}${isZipped}`;
-        const target = path17.join(this.dirname, filePath);
-        fs18.unlink(target, callback);
+        const target = path18.join(this.dirname, filePath);
+        fs19.unlink(target, callback);
       }
       /**
        * Roll files forward based on integer, up to maxFiles. e.g. if base if
@@ -10950,20 +10950,20 @@ var require_file = __commonJS({
         for (let x = this.maxFiles - 1; x > 1; x--) {
           tasks.push(function(i, cb) {
             let fileName = `${basename}${i - 1}${ext}${isZipped}`;
-            const tmppath = path17.join(this.dirname, fileName);
-            fs18.exists(tmppath, (exists) => {
+            const tmppath = path18.join(this.dirname, fileName);
+            fs19.exists(tmppath, (exists) => {
               if (!exists) {
                 return cb(null);
               }
               fileName = `${basename}${i}${ext}${isZipped}`;
-              fs18.rename(tmppath, path17.join(this.dirname, fileName), cb);
+              fs19.rename(tmppath, path18.join(this.dirname, fileName), cb);
             });
           }.bind(this, x));
         }
         asyncSeries(tasks, () => {
-          fs18.rename(
-            path17.join(this.dirname, `${basename}${ext}${isZipped}`),
-            path17.join(this.dirname, `${basename}1${ext}${isZipped}`),
+          fs19.rename(
+            path18.join(this.dirname, `${basename}${ext}${isZipped}`),
+            path18.join(this.dirname, `${basename}1${ext}${isZipped}`),
             callback
           );
         });
@@ -10977,22 +10977,22 @@ var require_file = __commonJS({
        * @private
        */
       _compressFile(src, dest, callback) {
-        fs18.access(src, fs18.F_OK, (err) => {
+        fs19.access(src, fs19.F_OK, (err) => {
           if (err) {
             return callback();
           }
           var gzip = zlib2.createGzip();
-          var inp = fs18.createReadStream(src);
-          var out = fs18.createWriteStream(dest);
+          var inp = fs19.createReadStream(src);
+          var out = fs19.createWriteStream(dest);
           out.on("finish", () => {
-            fs18.unlink(src, callback);
+            fs19.unlink(src, callback);
           });
           inp.pipe(gzip).pipe(out);
         });
       }
       _createLogDirIfNotExist(dirPath) {
-        if (!fs18.existsSync(dirPath)) {
-          fs18.mkdirSync(dirPath, { recursive: true });
+        if (!fs19.existsSync(dirPath)) {
+          fs19.mkdirSync(dirPath, { recursive: true });
         }
       }
     };
@@ -11076,9 +11076,9 @@ var require_http = __commonJS({
         };
         const auth = options.params.auth || null;
         delete options.params.auth;
-        const path17 = options.params.path || null;
+        const path18 = options.params.path || null;
         delete options.params.path;
-        this._request(options, auth, path17, (err, res, body) => {
+        this._request(options, auth, path18, (err, res, body) => {
           if (res && res.statusCode !== 200) {
             err = new Error(`Invalid HTTP Status Code: ${res.statusCode}`);
           }
@@ -11106,12 +11106,12 @@ var require_http = __commonJS({
           method: "stream",
           params: options
         };
-        const path17 = options.params.path || null;
+        const path18 = options.params.path || null;
         delete options.params.path;
         const auth = options.params.auth || null;
         delete options.params.auth;
         let buff = "";
-        const req = this._request(options, auth, path17);
+        const req = this._request(options, auth, path18);
         stream.destroy = () => req.destroy();
         req.on("data", (data) => {
           data = (buff + data).split(/\n+/);
@@ -11137,14 +11137,14 @@ var require_http = __commonJS({
        * @param {string} path - request path
        * @param {function} callback - Continuation to respond to when complete.
        */
-      _request(options, auth, path17, callback) {
+      _request(options, auth, path18, callback) {
         options = options || {};
         auth = auth || this.auth;
-        path17 = path17 || this.path || "";
+        path18 = path18 || this.path || "";
         if (this.batch) {
-          this._doBatch(options, callback, auth, path17);
+          this._doBatch(options, callback, auth, path18);
         } else {
-          this._doRequest(options, callback, auth, path17);
+          this._doRequest(options, callback, auth, path18);
         }
       }
       /**
@@ -11154,18 +11154,18 @@ var require_http = __commonJS({
        * @param {Object?} auth - authentication options
        * @param {string} path - request path
        */
-      _doBatch(options, callback, auth, path17) {
+      _doBatch(options, callback, auth, path18) {
         this.batchOptions.push(options);
         if (this.batchOptions.length === 1) {
           const me = this;
           this.batchCallback = callback;
           this.batchTimeoutID = setTimeout(function() {
             me.batchTimeoutID = -1;
-            me._doBatchRequest(me.batchCallback, auth, path17);
+            me._doBatchRequest(me.batchCallback, auth, path18);
           }, this.batchInterval);
         }
         if (this.batchOptions.length === this.batchCount) {
-          this._doBatchRequest(this.batchCallback, auth, path17);
+          this._doBatchRequest(this.batchCallback, auth, path18);
         }
       }
       /**
@@ -11174,14 +11174,14 @@ var require_http = __commonJS({
        * @param {Object?} auth - authentication options
        * @param {string} path - request path
        */
-      _doBatchRequest(callback, auth, path17) {
+      _doBatchRequest(callback, auth, path18) {
         if (this.batchTimeoutID > 0) {
           clearTimeout(this.batchTimeoutID);
           this.batchTimeoutID = -1;
         }
         const batchOptionsCopy = this.batchOptions.slice();
         this.batchOptions = [];
-        this._doRequest(batchOptionsCopy, callback, auth, path17);
+        this._doRequest(batchOptionsCopy, callback, auth, path18);
       }
       /**
        * Make a request to a winstond server or any http server which can
@@ -11191,7 +11191,7 @@ var require_http = __commonJS({
        * @param {Object?} auth - authentication options
        * @param {string} path - request path
        */
-      _doRequest(options, callback, auth, path17) {
+      _doRequest(options, callback, auth, path18) {
         const headers = Object.assign({}, this.headers);
         if (auth && auth.bearer) {
           headers.Authorization = `Bearer ${auth.bearer}`;
@@ -11201,7 +11201,7 @@ var require_http = __commonJS({
           method: "POST",
           host: this.host,
           port: this.port,
-          path: `/${path17.replace(/^\//, "")}`,
+          path: `/${path18.replace(/^\//, "")}`,
           headers,
           auth: auth && auth.username && auth.password ? `${auth.username}:${auth.password}` : "",
           agent: this.agent
@@ -11236,7 +11236,7 @@ var require_stream3 = __commonJS({
     "use strict";
     var isStream = require_is_stream();
     var { MESSAGE } = require_triple_beam();
-    var os2 = require("os");
+    var os3 = require("os");
     var TransportStream = require_winston_transport();
     module2.exports = class Stream extends TransportStream {
       /**
@@ -11252,7 +11252,7 @@ var require_stream3 = __commonJS({
         this._stream = options.stream;
         this._stream.setMaxListeners(Infinity);
         this.isObjectMode = options.stream._writableState.objectMode;
-        this.eol = typeof options.eol === "string" ? options.eol : os2.EOL;
+        this.eol = typeof options.eol === "string" ? options.eol : os3.EOL;
       }
       /**
        * Core logging method exposed to Winston.
@@ -11635,7 +11635,7 @@ var require_exception_stream = __commonJS({
 var require_exception_handler = __commonJS({
   "../../packages/logger/node_modules/winston/lib/winston/exception-handler.js"(exports2, module2) {
     "use strict";
-    var os2 = require("os");
+    var os3 = require("os");
     var asyncForEach = require_forEach();
     var debug = require_node2()("winston:exception");
     var once = require_one_time();
@@ -11730,8 +11730,8 @@ var require_exception_handler = __commonJS({
        */
       getOsInfo() {
         return {
-          loadavg: os2.loadavg(),
-          uptime: os2.uptime()
+          loadavg: os3.loadavg(),
+          uptime: os3.uptime()
         };
       }
       /**
@@ -11873,7 +11873,7 @@ var require_rejection_stream = __commonJS({
 var require_rejection_handler = __commonJS({
   "../../packages/logger/node_modules/winston/lib/winston/rejection-handler.js"(exports2, module2) {
     "use strict";
-    var os2 = require("os");
+    var os3 = require("os");
     var asyncForEach = require_forEach();
     var debug = require_node2()("winston:rejection");
     var once = require_one_time();
@@ -11970,8 +11970,8 @@ var require_rejection_handler = __commonJS({
        */
       getOsInfo() {
         return {
-          loadavg: os2.loadavg(),
-          uptime: os2.uptime()
+          loadavg: os3.loadavg(),
+          uptime: os3.uptime()
         };
       }
       /**
@@ -14221,7 +14221,7 @@ var Environment = new EnvironmentClass();
 var Environment_default = Environment;
 
 // ../../packages/executors/src/ConsumerExecutor.ts
-var import_path14 = __toESM(require("path"));
+var import_path15 = __toESM(require("path"));
 var import_fs9 = __toESM(require("fs"));
 var import_readline6 = __toESM(require("readline"));
 var import_promises8 = __toESM(require("fs/promises"));
@@ -14606,6 +14606,11 @@ var AutoMapperEngineClass = class {
 };
 var AutoMapperEngine = new AutoMapperEngineClass();
 
+// ../../packages/engines/src/producer/ProducerEngine.ts
+var fs11 = __toESM(require("fs"), 1);
+var os = __toESM(require("os"), 1);
+var import_path11 = __toESM(require("path"), 1);
+
 // ../../packages/drivers/src/DeltaShareDriver.ts
 var DeltaShareSourceDriver = class {
   constructor() {
@@ -15440,8 +15445,8 @@ var HttpClientClass = class {
         throw new Error(err.message ?? "An error occurred on GET.");
       }
     };
-    this.getWorkerUrl = (path17) => {
-      const cleanPath = path17.startsWith("/") ? path17.substring(1) : path17;
+    this.getWorkerUrl = (path18) => {
+      const cleanPath = path18.startsWith("/") ? path18.substring(1) : path18;
       return `${this.workerHost}/${cleanPath}`;
     };
     this.workerHost = ProcessENVManager_default.getEnvVariable("REMORA_WORKER_HOST") || "http://worker:5069";
@@ -16511,12 +16516,12 @@ var noopTracker = { _operations: {}, measure: () => {
 }, getOperations: () => ({}) };
 var ProducerEngineClass = class {
   constructor() {
-    this.readFile = async (producer, options) => {
+    this.readFile = async (producer, options, sourceOverride) => {
       Affirm_default(producer, "Invalid producer");
       Affirm_default(options, "Invalid options");
       if (options.readmode === "lines")
         Affirm_default(options.lines, "Invalid lines");
-      const source = Environment_default.getSource(producer.source);
+      const source = sourceOverride ?? Environment_default.getSource(producer.source);
       Affirm_default(source, `No source found for producer "${producer.name}" with name "${producer.source}"`);
       const driver = await DriverFactory_default.instantiateSource(source);
       Affirm_default(driver, `No driver found for producer "${producer.name}" with driver type "${source.engine}"`);
@@ -16590,8 +16595,22 @@ var ProducerEngineClass = class {
         case "local":
         case "aws-s3":
         case "delta-share": {
-          const fileData = await this.readFile(producer, { readmode: "lines", lines: { from: 0, to: sampleSize } });
-          rawData = fileData.data;
+          const { compressionType } = producer.settings;
+          let tempDir = null;
+          try {
+            let effectiveProducer = producer;
+            let effectiveSource = source;
+            if (compressionType) {
+              tempDir = fs11.mkdtempSync(import_path11.default.join(os.tmpdir(), "remora-sample-"));
+              const decompressedFileKey = await this._decompressForSampling(producer, source, tempDir);
+              effectiveProducer = { ...producer, settings: { ...producer.settings, fileKey: decompressedFileKey, compressionType: void 0 } };
+              effectiveSource = { ...source, authentication: { ...source.authentication, path: tempDir } };
+            }
+            const fileData = await this.readFile(effectiveProducer, { readmode: "lines", lines: { from: 0, to: sampleSize } }, effectiveSource);
+            rawData = fileData.data;
+          } finally {
+            if (tempDir) fs11.rmSync(tempDir, { recursive: true, force: true });
+          }
           break;
         }
         default:
@@ -16676,13 +16695,54 @@ var ProducerEngineClass = class {
       }
       return dimensions;
     };
+    this._decompressForSampling = async (producer, source, tempDir) => {
+      const { fileKey, compressionType } = producer.settings;
+      const sourcePath = source.authentication["path"];
+      switch (compressionType?.toUpperCase()) {
+        case "GZ": {
+          const decompressedName = fileKey.replace(/\.gz$/i, "");
+          const outputPath = import_path11.default.join(tempDir, decompressedName);
+          const outputDir = import_path11.default.dirname(outputPath);
+          if (!fs11.existsSync(outputDir)) fs11.mkdirSync(outputDir, { recursive: true });
+          await ParseCompression_default.decompressToFile(compressionType, fileKey, sourcePath, outputPath);
+          return decompressedName;
+        }
+        case "ZIP":
+        case "TAR": {
+          const extractDir = import_path11.default.join(tempDir, "extracted");
+          await ParseCompression_default.decompressToFile(compressionType, fileKey, sourcePath, extractDir);
+          const files = this._findFilesRecursive(extractDir);
+          Affirm_default(files.length > 0, `No files found after decompressing "${fileKey}"`);
+          const firstFile = files[0];
+          const relativePath = import_path11.default.relative(extractDir, firstFile);
+          const destPath = import_path11.default.join(tempDir, relativePath);
+          const destDir = import_path11.default.dirname(destPath);
+          if (!fs11.existsSync(destDir)) fs11.mkdirSync(destDir, { recursive: true });
+          fs11.renameSync(firstFile, destPath);
+          return relativePath;
+        }
+        default:
+          throw new Error(`Unsupported compression type "${compressionType}" for producer "${producer.name}"`);
+      }
+    };
+    this._findFilesRecursive = (dir) => {
+      const results = [];
+      for (const entry of fs11.readdirSync(dir)) {
+        const fullPath = import_path11.default.join(dir, entry);
+        if (fs11.statSync(fullPath).isDirectory())
+          results.push(...this._findFilesRecursive(fullPath));
+        else
+          results.push(fullPath);
+      }
+      return results;
+    };
   }
 };
 var ProducerEngine = new ProducerEngineClass();
 var ProducerEngine_default = ProducerEngine;
 
 // ../../packages/engines/src/ai/DeveloperEngine.ts
-var import_path11 = __toESM(require("path"), 1);
+var import_path12 = __toESM(require("path"), 1);
 var import_promises6 = __toESM(require("fs/promises"), 1);
 var import_dayjs2 = __toESM(require("dayjs"), 1);
 var import_customParseFormat2 = __toESM(require("dayjs/plugin/customParseFormat"), 1);
@@ -16709,7 +16769,7 @@ var DeveloperEngineClass = class {
         _version: producer._version ?? 1
       };
       mappedProducer["$schema"] = producer["$schema"];
-      const producerPath = import_path11.default.join(process.cwd(), "remora", "producers", `${producer.name}.json`);
+      const producerPath = import_path12.default.join(process.cwd(), "remora", "producers", `${producer.name}.json`);
       await import_promises6.default.writeFile(producerPath, JSON.stringify(mappedProducer, null, 4), "utf-8");
       return { producer: mappedProducer, fields: typeDefinitions };
     };
@@ -16968,8 +17028,8 @@ var DeveloperEngineClass = class {
       Affirm_default(fileType, "Producer must have a fileType setting for mock data generation");
       const mockRecords = this.generateMockRecords(producer.dimensions, records);
       const basePath = source.authentication.path || process.cwd();
-      const filePath = import_path11.default.join(basePath, fileKey.replace("%", "mock"));
-      await import_promises6.default.mkdir(import_path11.default.dirname(filePath), { recursive: true });
+      const filePath = import_path12.default.join(basePath, fileKey.replace("%", "mock"));
+      await import_promises6.default.mkdir(import_path12.default.dirname(filePath), { recursive: true });
       const content = this.formatMockData(mockRecords, fileType, delimiter);
       await import_promises6.default.writeFile(filePath, content, "utf-8");
       return { filePath, recordCount: records };
@@ -18419,10 +18479,10 @@ var UsageManager = new UsageManagerClass();
 var UsageManager_default = UsageManager;
 
 // ../../packages/executors/src/OutputExecutor.ts
-var fs13 = __toESM(require("fs"));
+var fs14 = __toESM(require("fs"));
 
 // ../../packages/executors/src/ExecutorScope.ts
-var import_path12 = __toESM(require("path"));
+var import_path13 = __toESM(require("path"));
 var import_fs8 = __toESM(require("fs"));
 var import_promises7 = __toESM(require("fs/promises"));
 var ExecutorScopeClass2 = class {
@@ -18430,7 +18490,7 @@ var ExecutorScopeClass2 = class {
     this.WORKERS_FOLDER = "workers";
     this.PRODUCERS_FOLDER = "producers";
     this.getWorkerPath = (scope, workerId) => {
-      return import_path12.default.join(
+      return import_path13.default.join(
         Constants_default.defaults.REMORA_PATH,
         Constants_default.defaults.PRODUCER_TEMP_FOLDER,
         // A specific execution sits entirely in this folder, so at the end it's safe to delete it entirely
@@ -18440,7 +18500,7 @@ var ExecutorScopeClass2 = class {
       );
     };
     this.getProducerPath = (scope, producer, sourceFileKey) => {
-      return import_path12.default.join(
+      return import_path13.default.join(
         Constants_default.defaults.REMORA_PATH,
         Constants_default.defaults.PRODUCER_TEMP_FOLDER,
         // A specific execution sits entirely in this folder, so at the end it's safe to delete it entirely
@@ -18451,7 +18511,7 @@ var ExecutorScopeClass2 = class {
       );
     };
     this.getMainPath = (scope) => {
-      return import_path12.default.join(
+      return import_path13.default.join(
         Constants_default.defaults.REMORA_PATH,
         Constants_default.defaults.PRODUCER_TEMP_FOLDER,
         scope.folder,
@@ -18459,7 +18519,7 @@ var ExecutorScopeClass2 = class {
       );
     };
     this.clearScope = async (scope) => {
-      const scopePath = import_path12.default.join(
+      const scopePath = import_path13.default.join(
         Constants_default.defaults.REMORA_PATH,
         Constants_default.defaults.PRODUCER_TEMP_FOLDER,
         scope.folder
@@ -18469,7 +18529,7 @@ var ExecutorScopeClass2 = class {
       }
     };
     this.ensurePath = (fileUri) => {
-      const dir = import_path12.default.dirname(fileUri);
+      const dir = import_path13.default.dirname(fileUri);
       if (!import_fs8.default.existsSync(dir))
         import_fs8.default.mkdirSync(dir, { recursive: true });
       if (!import_fs8.default.existsSync(fileUri))
@@ -18481,7 +18541,7 @@ var ExecutorScope2 = new ExecutorScopeClass2();
 var ExecutorScope_default2 = ExecutorScope2;
 
 // ../../packages/executors/src/OutputExecutor.ts
-var import_path13 = __toESM(require("path"));
+var import_path14 = __toESM(require("path"));
 var OutputExecutorClass = class {
   constructor() {
     this._getInternalRecordFormat = (consumer) => {
@@ -18527,13 +18587,13 @@ var OutputExecutorClass = class {
       for (const output of consumer.outputs) {
         const destination = Environment_default.getSource(output.exportDestination);
         const driver = await DriverFactory_default.instantiateDestination(destination);
-        const currentPath = import_path13.default.dirname(ExecutorScope_default2.getMainPath(scope));
+        const currentPath = import_path14.default.dirname(ExecutorScope_default2.getMainPath(scope));
         const destinationName = this._composeFileName(consumer, output, this._getExtension(output));
         Logger_default.log(`Exporting consumer "${consumer.name}" to "${output.exportDestination}" as ${output.format} (${destinationName})`);
-        const filenameArray = fs13.readdirSync(currentPath).filter((filename) => filename.includes(".dataset"));
+        const filenameArray = fs14.readdirSync(currentPath).filter((filename) => filename.includes(".dataset"));
         for (const filename in filenameArray) {
           const destinationPath = this.getCompletedPath(destinationName, filename);
-          const startingPath = import_path13.default.join(currentPath, filenameArray[filename]);
+          const startingPath = import_path14.default.join(currentPath, filenameArray[filename]);
           if (output.format === internalFormat) {
             results.push(await driver.move(startingPath, destinationPath));
           } else {
@@ -18593,8 +18653,8 @@ var OutputExecutor_default = OutputExecutor;
 var ConsumerExecutorClass = class {
   constructor() {
     this._getWorkPath = (consumer, executionId) => {
-      const execFolder = import_path14.default.join(consumer.name, executionId);
-      const workPath = import_path14.default.join("./remora", Constants_default.defaults.PRODUCER_TEMP_FOLDER, execFolder, ".dataset");
+      const execFolder = import_path15.default.join(consumer.name, executionId);
+      const workPath = import_path15.default.join("./remora", Constants_default.defaults.PRODUCER_TEMP_FOLDER, execFolder, ".dataset");
       return workPath;
     };
     this._clearWorkPath = async (workPath) => {
@@ -18605,7 +18665,7 @@ var ConsumerExecutorClass = class {
       } catch (error) {
       }
       try {
-        const dir = import_path14.default.dirname(workPath);
+        const dir = import_path15.default.dirname(workPath);
         if (import_fs9.default.existsSync(dir)) {
           await import_promises8.default.rmdir(dir);
         }
@@ -18613,7 +18673,7 @@ var ConsumerExecutorClass = class {
       }
     };
     this._ensurePath = (pathUri) => {
-      const dir = import_path14.default.dirname(pathUri);
+      const dir = import_path15.default.dirname(pathUri);
       if (!import_fs9.default.existsSync(dir))
         import_fs9.default.mkdirSync(dir, { recursive: true });
       if (!import_fs9.default.existsSync(pathUri))
@@ -19013,7 +19073,7 @@ var import_fs10 = __toESM(require("fs"));
 var import_readline7 = __toESM(require("readline"));
 
 // ../../packages/executors/src/ProducerExecutor.ts
-var import_path15 = __toESM(require("path"));
+var import_path16 = __toESM(require("path"));
 var ProducerExecutorClass = class {
   constructor() {
     this.ready = async (producer, scope) => {
@@ -19037,7 +19097,7 @@ var ProducerExecutorClass = class {
       counter = performance.now();
       for (const dimension of dimensions) {
         if (dimension.prodDimension.sourceFilename === true)
-          record[dimension.name] = import_path15.default.basename(chunk.fileUri);
+          record[dimension.name] = import_path16.default.basename(chunk.fileUri);
         const maskType = ProducerManager_default.getMask(dimension.prodDimension);
         if (Algo_default.hasVal(maskType))
           record[dimension.name] = CryptoEngine_default.hashValue(maskType, record[dimension.name]?.toString(), dimension.prodDimension.type);
@@ -19227,7 +19287,7 @@ var Executor_default = Executor;
 var import_os = __toESM(require("os"));
 var import_fs11 = __toESM(require("fs"));
 var import_promises9 = __toESM(require("fs/promises"));
-var import_path16 = __toESM(require("path"));
+var import_path17 = __toESM(require("path"));
 var import_workerpool = __toESM(require("workerpool"));
 
 // ../../packages/executors/src/ExecutorProgress.ts
@@ -19285,7 +19345,7 @@ var ExecutorProgress = class {
 var ExecutorProgress_default = ExecutorProgress;
 
 // ../../packages/executors/src/ExecutorWriter.ts
-var fs16 = __toESM(require("fs"));
+var fs17 = __toESM(require("fs"));
 var import_readline8 = __toESM(require("readline"));
 var ExecutorWriter = class {
   constructor() {
@@ -19302,11 +19362,11 @@ var ExecutorWriter = class {
     };
     this.splitBySize = async (scope, sourcePath) => {
       const maxOutputFileSize = scope.limitFileSize * this.FAKE_GB;
-      const readStream = fs16.createReadStream(sourcePath);
+      const readStream = fs17.createReadStream(sourcePath);
       const reader = import_readline8.default.createInterface({ input: readStream, crlfDelay: Infinity });
       let writerIndex = 0;
       let destPath = this.getCompletedPath(sourcePath, writerIndex);
-      let writeStream = fs16.createWriteStream(destPath, { flags: "a" });
+      let writeStream = fs17.createWriteStream(destPath, { flags: "a" });
       for await (const line of reader) {
         if (readStream.bytesRead > maxOutputFileSize * (writerIndex + 1)) {
           writeStream.end();
@@ -19316,7 +19376,7 @@ var ExecutorWriter = class {
           });
           writerIndex++;
           destPath = this.getCompletedPath(sourcePath, writerIndex);
-          writeStream = fs16.createWriteStream(destPath, { flags: "a" });
+          writeStream = fs17.createWriteStream(destPath, { flags: "a" });
         }
         writeStream.write(line + "\n");
       }
@@ -19325,7 +19385,7 @@ var ExecutorWriter = class {
         writeStream.on("finish", resolve);
         writeStream.on("error", reject);
       });
-      await fs16.promises.unlink(sourcePath);
+      await fs17.promises.unlink(sourcePath);
     };
     /**
      * Manage the Writestream for main.dataset
@@ -19395,7 +19455,7 @@ var ExecutorOrchestratorClass = class {
       };
       const workerPath = this._getWorkerPath();
       Logger_default.log(`Initializing worker pool from ${workerPath} (heap limit: ${Constants_default.defaults.MIN_RUNTIME_HEAP_MB}MB)`);
-      return import_workerpool.default.pool(import_path16.default.join(workerPath, "ExecutorWorker.js"), options);
+      return import_workerpool.default.pool(import_path17.default.join(workerPath, "ExecutorWorker.js"), options);
     };
     this.launch = async (request) => {
       Affirm_default(request, "Invalid options");
@@ -19516,6 +19576,14 @@ var ExecutorOrchestratorClass = class {
           }
           Logger_default.log(`[${usageId}] Dataset validations complete in ${Math.round(performance.now() - counter)}ms`);
         }
+        if (scope.limitFileSize) {
+          Logger_default.log(`[${usageId}] Splitting output by size limit (${scope.limitFileSize})`);
+          counter = performance.now();
+          const writer = new ExecutorWriter_default();
+          await writer.splitBySize(scope, ExecutorScope_default2.getMainPath(scope));
+          tracker.measure("split-by-size", performance.now() - counter);
+          Logger_default.log(`[${usageId}] Split complete in ${Math.round(performance.now() - counter)}ms`);
+        }
         counter = performance.now();
         Logger_default.log(`[${usageId}] Exporting results to ${consumer.outputs.length} output(s)`);
         const exportRes = await OutputExecutor_default.exportResult(consumer, ConsumerManager_default.getExpandedFields(consumer), scope);
@@ -19623,20 +19691,19 @@ var ExecutorOrchestratorClass = class {
     this._getWorkerPath = () => {
       const currentDir = __dirname;
       if (ProcessENVManager_default.getEnvVariable("NODE_ENV") === "dev" || ProcessENVManager_default.getEnvVariable("NODE_ENV") === "development")
-        return import_path16.default.resolve("./.build/workers");
+        return import_path17.default.resolve("./.build/workers");
       const forcedPath = ProcessENVManager_default.getEnvVariable("REMORA_WORKERS_PATH");
       if (forcedPath && forcedPath.length > 0)
-        return import_path16.default.join(__dirname, forcedPath);
+        return import_path17.default.join(__dirname, forcedPath);
       if (!currentDir.includes(".build")) {
-        return import_path16.default.join(__dirname, "../workers");
+        return import_path17.default.join(__dirname, "../workers");
       } else {
-        return import_path16.default.resolve("./.build/workers");
+        return import_path17.default.resolve("./.build/workers");
       }
     };
     this.reconcileExecutorThreadsResults = async (scope, executorResults, tracker) => {
       const mainPath = ExecutorScope_default2.getMainPath(scope);
       ConsumerExecutor_default._ensurePath(mainPath);
-      const writer = new ExecutorWriter_default();
       if (executorResults.length > 1) {
         Logger_default.log(`[${scope.id}] Merging ${executorResults.length} worker output files into ${mainPath}`);
         const perf = performance.now();
@@ -19652,10 +19719,6 @@ var ExecutorOrchestratorClass = class {
         Logger_default.log(`[${scope.id}] Single worker \u2014 renaming output to ${mainPath}`);
         await import_promises9.default.rename(executorResults[0].resultUri, mainPath);
       }
-      if (scope.limitFileSize) {
-        Logger_default.log(`[${scope.id}] Splitting output by size limit (${scope.limitFileSize})`);
-        await writer.splitBySize(scope, mainPath);
-      }
       return mainPath;
     };
     this.performCleanupOperations = async (scope, tracker) => {