@forzalabs/remora 1.2.1 → 1.2.2

package/index.js

@@ -147,7 +147,7 @@ var require_has_flag = __commonJS({
 var require_supports_colors = __commonJS({
   "../../node_modules/logform/node_modules/@colors/colors/lib/system/supports-colors.js"(exports2, module2) {
     "use strict";
-    var os2 = require("os");
+    var os3 = require("os");
     var hasFlag = require_has_flag();
     var env = process.env;
     var forceColor = void 0;
@@ -185,7 +185,7 @@ var require_supports_colors = __commonJS({
       }
       var min = forceColor ? 1 : 0;
       if (process.platform === "win32") {
-        var osRelease = os2.release().split(".");
+        var osRelease = os3.release().split(".");
         if (Number(process.versions.node.split(".")[0]) >= 8 && Number(osRelease[0]) >= 10 && Number(osRelease[2]) >= 10586) {
           return Number(osRelease[2]) >= 14931 ? 3 : 2;
         }
@@ -5429,7 +5429,7 @@ var require_winston_transport = __commonJS({
 var require_console = __commonJS({
   "../../packages/logger/node_modules/winston/lib/winston/transports/console.js"(exports2, module2) {
     "use strict";
-    var os2 = require("os");
+    var os3 = require("os");
     var { LEVEL, MESSAGE } = require_triple_beam();
     var TransportStream = require_winston_transport();
     module2.exports = class Console extends TransportStream {
@@ -5443,7 +5443,7 @@ var require_console = __commonJS({
         this.name = options.name || "console";
         this.stderrLevels = this._stringArrayToSet(options.stderrLevels);
         this.consoleWarnLevels = this._stringArrayToSet(options.consoleWarnLevels);
-        this.eol = typeof options.eol === "string" ? options.eol : os2.EOL;
+        this.eol = typeof options.eol === "string" ? options.eol : os3.EOL;
         this.forceConsole = options.forceConsole || false;
         this._consoleLog = console.log.bind(console);
         this._consoleWarn = console.warn.bind(console);
@@ -9655,15 +9655,15 @@ var require_index_cjs = __commonJS({
       };
     }
     function wrapConversion(toModel, graph) {
-      const path23 = [graph[toModel].parent, toModel];
+      const path24 = [graph[toModel].parent, toModel];
       let fn = convert$1[graph[toModel].parent][toModel];
       let cur = graph[toModel].parent;
       while (graph[cur].parent) {
-        path23.unshift(graph[cur].parent);
+        path24.unshift(graph[cur].parent);
         fn = link(convert$1[graph[cur].parent][cur], fn);
         cur = graph[cur].parent;
       }
-      fn.conversion = path23;
+      fn.conversion = path24;
       return fn;
     }
     function route(fromModel) {
@@ -10293,7 +10293,7 @@ var require_node2 = __commonJS({
 var require_tail_file = __commonJS({
   "../../packages/logger/node_modules/winston/lib/winston/tail-file.js"(exports2, module2) {
     "use strict";
-    var fs24 = require("fs");
+    var fs25 = require("fs");
     var { StringDecoder } = require("string_decoder");
     var { Stream } = require_readable();
     function noop() {
@@ -10314,7 +10314,7 @@ var require_tail_file = __commonJS({
         stream.emit("end");
         stream.emit("close");
       };
-      fs24.open(options.file, "a+", "0644", (err, fd) => {
+      fs25.open(options.file, "a+", "0644", (err, fd) => {
         if (err) {
           if (!iter) {
             stream.emit("error", err);
@@ -10326,10 +10326,10 @@ var require_tail_file = __commonJS({
         }
         (function read() {
           if (stream.destroyed) {
-            fs24.close(fd, noop);
+            fs25.close(fd, noop);
             return;
           }
-          return fs24.read(fd, buffer, 0, buffer.length, pos, (error, bytes) => {
+          return fs25.read(fd, buffer, 0, buffer.length, pos, (error, bytes) => {
             if (error) {
               if (!iter) {
                 stream.emit("error", error);
@@ -10388,15 +10388,15 @@ var require_tail_file = __commonJS({
 var require_file = __commonJS({
   "../../packages/logger/node_modules/winston/lib/winston/transports/file.js"(exports2, module2) {
     "use strict";
-    var fs24 = require("fs");
-    var path23 = require("path");
+    var fs25 = require("fs");
+    var path24 = require("path");
     var asyncSeries = require_series();
     var zlib2 = require("zlib");
     var { MESSAGE } = require_triple_beam();
     var { Stream, PassThrough } = require_readable();
     var TransportStream = require_winston_transport();
     var debug2 = require_node2()("winston:file");
-    var os2 = require("os");
+    var os3 = require("os");
     var tailFile = require_tail_file();
     module2.exports = class File extends TransportStream {
       /**
@@ -10419,14 +10419,14 @@ var require_file = __commonJS({
         this._onError = this._onError.bind(this);
         if (options.filename || options.dirname) {
           throwIf("filename or dirname", "stream");
-          this._basename = this.filename = options.filename ? path23.basename(options.filename) : "winston.log";
-          this.dirname = options.dirname || path23.dirname(options.filename);
+          this._basename = this.filename = options.filename ? path24.basename(options.filename) : "winston.log";
+          this.dirname = options.dirname || path24.dirname(options.filename);
           this.options = options.options || { flags: "a" };
         } else if (options.stream) {
           console.warn("options.stream will be removed in winston@4. Use winston.transports.Stream");
           throwIf("stream", "filename", "maxsize");
           this._dest = this._stream.pipe(this._setupStream(options.stream));
-          this.dirname = path23.dirname(this._dest.path);
+          this.dirname = path24.dirname(this._dest.path);
         } else {
           throw new Error("Cannot log to file without filename or stream.");
         }
@@ -10434,7 +10434,7 @@ var require_file = __commonJS({
         this.rotationFormat = options.rotationFormat || false;
         this.zippedArchive = options.zippedArchive || false;
         this.maxFiles = options.maxFiles || null;
-        this.eol = typeof options.eol === "string" ? options.eol : os2.EOL;
+        this.eol = typeof options.eol === "string" ? options.eol : os3.EOL;
         this.tailable = options.tailable || false;
         this.lazy = options.lazy || false;
         this._size = 0;
@@ -10589,11 +10589,11 @@ var require_file = __commonJS({
           options = {};
         }
         options = normalizeQuery(options);
-        const file = path23.join(this.dirname, this.filename);
+        const file = path24.join(this.dirname, this.filename);
         let buff = "";
         let results = [];
         let row = 0;
-        const stream = fs24.createReadStream(file, {
+        const stream = fs25.createReadStream(file, {
           encoding: "utf8"
         });
         stream.on("error", (err) => {
@@ -10694,7 +10694,7 @@ var require_file = __commonJS({
        * TODO: Refactor me.
        */
       stream(options = {}) {
-        const file = path23.join(this.dirname, this.filename);
+        const file = path24.join(this.dirname, this.filename);
         const stream = new Stream();
         const tail = {
           file,
@@ -10744,8 +10744,8 @@ var require_file = __commonJS({
        */
       stat(callback) {
         const target = this._getFile();
-        const fullpath = path23.join(this.dirname, target);
-        fs24.stat(fullpath, (err, stat) => {
+        const fullpath = path24.join(this.dirname, target);
+        fs25.stat(fullpath, (err, stat) => {
           if (err && err.code === "ENOENT") {
             debug2("ENOENT\xA0ok", fullpath);
             this.filename = target;
@@ -10848,9 +10848,9 @@ var require_file = __commonJS({
        * @returns {WritableStream} Stream that writes to disk for the active file.
        */
       _createStream(source) {
-        const fullpath = path23.join(this.dirname, this.filename);
+        const fullpath = path24.join(this.dirname, this.filename);
         debug2("create stream start", fullpath, this.options);
-        const dest = fs24.createWriteStream(fullpath, this.options).on("error", (err) => debug2(err)).on("close", () => debug2("close", dest.path, dest.bytesWritten)).on("open", () => {
+        const dest = fs25.createWriteStream(fullpath, this.options).on("error", (err) => debug2(err)).on("close", () => debug2("close", dest.path, dest.bytesWritten)).on("open", () => {
           debug2("file open ok", fullpath);
           this.emit("open", fullpath);
           source.pipe(dest);
@@ -10873,16 +10873,16 @@ var require_file = __commonJS({
        */
       _incFile(callback) {
         debug2("_incFile", this.filename);
-        const ext = path23.extname(this._basename);
-        const basename = path23.basename(this._basename, ext);
+        const ext = path24.extname(this._basename);
+        const basename = path24.basename(this._basename, ext);
         const tasks = [];
         if (this.zippedArchive) {
           tasks.push(
             function(cb) {
               const num = this._created > 0 && !this.tailable ? this._created : "";
               this._compressFile(
-                path23.join(this.dirname, `${basename}${num}${ext}`),
-                path23.join(this.dirname, `${basename}${num}${ext}.gz`),
+                path24.join(this.dirname, `${basename}${num}${ext}`),
+                path24.join(this.dirname, `${basename}${num}${ext}.gz`),
                 cb
               );
             }.bind(this)
@@ -10907,8 +10907,8 @@ var require_file = __commonJS({
        * @private
        */
       _getFile() {
-        const ext = path23.extname(this._basename);
-        const basename = path23.basename(this._basename, ext);
+        const ext = path24.extname(this._basename);
+        const basename = path24.basename(this._basename, ext);
         const isRotation = this.rotationFormat ? this.rotationFormat() : this._created;
         return !this.tailable && this._created ? `${basename}${isRotation}${ext}` : `${basename}${ext}`;
       }
@@ -10928,8 +10928,8 @@ var require_file = __commonJS({
         const isOldest = oldest !== 0 ? oldest : "";
         const isZipped = this.zippedArchive ? ".gz" : "";
         const filePath = `${basename}${isOldest}${ext}${isZipped}`;
-        const target = path23.join(this.dirname, filePath);
-        fs24.unlink(target, callback);
+        const target = path24.join(this.dirname, filePath);
+        fs25.unlink(target, callback);
       }
       /**
        * Roll files forward based on integer, up to maxFiles. e.g. if base if
@@ -10951,20 +10951,20 @@ var require_file = __commonJS({
         for (let x = this.maxFiles - 1; x > 1; x--) {
           tasks.push(function(i, cb) {
             let fileName = `${basename}${i - 1}${ext}${isZipped}`;
-            const tmppath = path23.join(this.dirname, fileName);
-            fs24.exists(tmppath, (exists) => {
+            const tmppath = path24.join(this.dirname, fileName);
+            fs25.exists(tmppath, (exists) => {
               if (!exists) {
                 return cb(null);
               }
               fileName = `${basename}${i}${ext}${isZipped}`;
-              fs24.rename(tmppath, path23.join(this.dirname, fileName), cb);
+              fs25.rename(tmppath, path24.join(this.dirname, fileName), cb);
             });
           }.bind(this, x));
         }
         asyncSeries(tasks, () => {
-          fs24.rename(
-            path23.join(this.dirname, `${basename}${ext}${isZipped}`),
-            path23.join(this.dirname, `${basename}1${ext}${isZipped}`),
+          fs25.rename(
+            path24.join(this.dirname, `${basename}${ext}${isZipped}`),
+            path24.join(this.dirname, `${basename}1${ext}${isZipped}`),
             callback
           );
         });
@@ -10978,22 +10978,22 @@ var require_file = __commonJS({
        * @private
        */
       _compressFile(src, dest, callback) {
-        fs24.access(src, fs24.F_OK, (err) => {
+        fs25.access(src, fs25.F_OK, (err) => {
           if (err) {
             return callback();
           }
           var gzip = zlib2.createGzip();
-          var inp = fs24.createReadStream(src);
-          var out = fs24.createWriteStream(dest);
+          var inp = fs25.createReadStream(src);
+          var out = fs25.createWriteStream(dest);
           out.on("finish", () => {
-            fs24.unlink(src, callback);
+            fs25.unlink(src, callback);
           });
           inp.pipe(gzip).pipe(out);
         });
       }
       _createLogDirIfNotExist(dirPath) {
-        if (!fs24.existsSync(dirPath)) {
-          fs24.mkdirSync(dirPath, { recursive: true });
+        if (!fs25.existsSync(dirPath)) {
+          fs25.mkdirSync(dirPath, { recursive: true });
         }
       }
     };
@@ -11077,9 +11077,9 @@ var require_http = __commonJS({
         };
         const auth = options.params.auth || null;
         delete options.params.auth;
-        const path23 = options.params.path || null;
+        const path24 = options.params.path || null;
         delete options.params.path;
-        this._request(options, auth, path23, (err, res, body) => {
+        this._request(options, auth, path24, (err, res, body) => {
           if (res && res.statusCode !== 200) {
             err = new Error(`Invalid HTTP Status Code: ${res.statusCode}`);
           }
@@ -11107,12 +11107,12 @@ var require_http = __commonJS({
           method: "stream",
           params: options
         };
-        const path23 = options.params.path || null;
+        const path24 = options.params.path || null;
         delete options.params.path;
         const auth = options.params.auth || null;
         delete options.params.auth;
         let buff = "";
-        const req = this._request(options, auth, path23);
+        const req = this._request(options, auth, path24);
         stream.destroy = () => req.destroy();
         req.on("data", (data) => {
           data = (buff + data).split(/\n+/);
@@ -11138,14 +11138,14 @@ var require_http = __commonJS({
        * @param {string} path - request path
        * @param {function} callback - Continuation to respond to when complete.
        */
-      _request(options, auth, path23, callback) {
+      _request(options, auth, path24, callback) {
         options = options || {};
         auth = auth || this.auth;
-        path23 = path23 || this.path || "";
+        path24 = path24 || this.path || "";
         if (this.batch) {
-          this._doBatch(options, callback, auth, path23);
+          this._doBatch(options, callback, auth, path24);
         } else {
-          this._doRequest(options, callback, auth, path23);
+          this._doRequest(options, callback, auth, path24);
         }
       }
       /**
@@ -11155,18 +11155,18 @@ var require_http = __commonJS({
        * @param {Object?} auth - authentication options
        * @param {string} path - request path
        */
-      _doBatch(options, callback, auth, path23) {
+      _doBatch(options, callback, auth, path24) {
         this.batchOptions.push(options);
         if (this.batchOptions.length === 1) {
           const me = this;
           this.batchCallback = callback;
           this.batchTimeoutID = setTimeout(function() {
             me.batchTimeoutID = -1;
-            me._doBatchRequest(me.batchCallback, auth, path23);
+            me._doBatchRequest(me.batchCallback, auth, path24);
           }, this.batchInterval);
         }
         if (this.batchOptions.length === this.batchCount) {
-          this._doBatchRequest(this.batchCallback, auth, path23);
+          this._doBatchRequest(this.batchCallback, auth, path24);
         }
       }
       /**
@@ -11175,14 +11175,14 @@ var require_http = __commonJS({
        * @param {Object?} auth - authentication options
        * @param {string} path - request path
        */
-      _doBatchRequest(callback, auth, path23) {
+      _doBatchRequest(callback, auth, path24) {
         if (this.batchTimeoutID > 0) {
           clearTimeout(this.batchTimeoutID);
           this.batchTimeoutID = -1;
         }
         const batchOptionsCopy = this.batchOptions.slice();
         this.batchOptions = [];
-        this._doRequest(batchOptionsCopy, callback, auth, path23);
+        this._doRequest(batchOptionsCopy, callback, auth, path24);
       }
       /**
        * Make a request to a winstond server or any http server which can
@@ -11192,7 +11192,7 @@ var require_http = __commonJS({
        * @param {Object?} auth - authentication options
        * @param {string} path - request path
        */
-      _doRequest(options, callback, auth, path23) {
+      _doRequest(options, callback, auth, path24) {
         const headers = Object.assign({}, this.headers);
         if (auth && auth.bearer) {
           headers.Authorization = `Bearer ${auth.bearer}`;
@@ -11202,7 +11202,7 @@ var require_http = __commonJS({
           method: "POST",
           host: this.host,
           port: this.port,
-          path: `/${path23.replace(/^\//, "")}`,
+          path: `/${path24.replace(/^\//, "")}`,
           headers,
           auth: auth && auth.username && auth.password ? `${auth.username}:${auth.password}` : "",
           agent: this.agent
@@ -11237,7 +11237,7 @@ var require_stream3 = __commonJS({
     "use strict";
     var isStream = require_is_stream();
     var { MESSAGE } = require_triple_beam();
-    var os2 = require("os");
+    var os3 = require("os");
     var TransportStream = require_winston_transport();
     module2.exports = class Stream extends TransportStream {
       /**
@@ -11253,7 +11253,7 @@ var require_stream3 = __commonJS({
         this._stream = options.stream;
         this._stream.setMaxListeners(Infinity);
         this.isObjectMode = options.stream._writableState.objectMode;
-        this.eol = typeof options.eol === "string" ? options.eol : os2.EOL;
+        this.eol = typeof options.eol === "string" ? options.eol : os3.EOL;
       }
       /**
        * Core logging method exposed to Winston.
@@ -11636,7 +11636,7 @@ var require_exception_stream = __commonJS({
 var require_exception_handler = __commonJS({
   "../../packages/logger/node_modules/winston/lib/winston/exception-handler.js"(exports2, module2) {
     "use strict";
-    var os2 = require("os");
+    var os3 = require("os");
     var asyncForEach = require_forEach();
     var debug2 = require_node2()("winston:exception");
     var once = require_one_time();
@@ -11731,8 +11731,8 @@ var require_exception_handler = __commonJS({
        */
       getOsInfo() {
         return {
-          loadavg: os2.loadavg(),
-          uptime: os2.uptime()
+          loadavg: os3.loadavg(),
+          uptime: os3.uptime()
         };
       }
       /**
@@ -11874,7 +11874,7 @@ var require_rejection_stream = __commonJS({
 var require_rejection_handler = __commonJS({
   "../../packages/logger/node_modules/winston/lib/winston/rejection-handler.js"(exports2, module2) {
     "use strict";
-    var os2 = require("os");
+    var os3 = require("os");
     var asyncForEach = require_forEach();
     var debug2 = require_node2()("winston:rejection");
     var once = require_one_time();
@@ -11971,8 +11971,8 @@ var require_rejection_handler = __commonJS({
        */
       getOsInfo() {
         return {
-          loadavg: os2.loadavg(),
-          uptime: os2.uptime()
+          loadavg: os3.loadavg(),
+          uptime: os3.uptime()
         };
       }
       /**
@@ -15102,8 +15102,8 @@ var HttpClientClass = class {
         throw new Error(err.message ?? "An error occurred on GET.");
       }
     };
-    this.getWorkerUrl = (path23) => {
-      const cleanPath = path23.startsWith("/") ? path23.substring(1) : path23;
+    this.getWorkerUrl = (path24) => {
+      const cleanPath = path24.startsWith("/") ? path24.substring(1) : path24;
       return `${this.workerHost}/${cleanPath}`;
     };
     this.workerHost = ProcessENVManager_default.getEnvVariable("REMORA_WORKER_HOST") || "http://worker:5069";
@@ -16676,7 +16676,7 @@ var LicenceManager = new LicenceManagerClass();
 var LicenceManager_default = LicenceManager;
 
 // ../../packages/executors/src/ConsumerExecutor.ts
-var import_path17 = __toESM(require("path"));
+var import_path18 = __toESM(require("path"));
 var import_fs11 = __toESM(require("fs"));
 var import_readline6 = __toESM(require("readline"));
 var import_promises8 = __toESM(require("fs/promises"));
@@ -17062,6 +17062,11 @@ var AutoMapperEngineClass = class {
 var AutoMapperEngine = new AutoMapperEngineClass();
 var AutoMapperEngine_default = AutoMapperEngine;
 
+// ../../packages/engines/src/producer/ProducerEngine.ts
+var fs15 = __toESM(require("fs"), 1);
+var os = __toESM(require("os"), 1);
+var import_path14 = __toESM(require("path"), 1);
+
 // ../../packages/engines/src/transform/TypeCaster.ts
 var import_dayjs = __toESM(require("dayjs"), 1);
 var import_customParseFormat = __toESM(require("dayjs/plugin/customParseFormat"), 1);
@@ -17181,12 +17186,12 @@ var noopTracker = { _operations: {}, measure: () => {
 }, getOperations: () => ({}) };
 var ProducerEngineClass = class {
   constructor() {
-    this.readFile = async (producer, options) => {
+    this.readFile = async (producer, options, sourceOverride) => {
       Affirm_default(producer, "Invalid producer");
       Affirm_default(options, "Invalid options");
       if (options.readmode === "lines")
         Affirm_default(options.lines, "Invalid lines");
-      const source = Environment_default.getSource(producer.source);
+      const source = sourceOverride ?? Environment_default.getSource(producer.source);
       Affirm_default(source, `No source found for producer "${producer.name}" with name "${producer.source}"`);
       const driver = await DriverFactory_default.instantiateSource(source);
       Affirm_default(driver, `No driver found for producer "${producer.name}" with driver type "${source.engine}"`);
@@ -17260,8 +17265,22 @@ var ProducerEngineClass = class {
         case "local":
         case "aws-s3":
         case "delta-share": {
-          const fileData = await this.readFile(producer, { readmode: "lines", lines: { from: 0, to: sampleSize } });
-          rawData = fileData.data;
+          const { compressionType } = producer.settings;
+          let tempDir = null;
+          try {
+            let effectiveProducer = producer;
+            let effectiveSource = source;
+            if (compressionType) {
+              tempDir = fs15.mkdtempSync(import_path14.default.join(os.tmpdir(), "remora-sample-"));
+              const decompressedFileKey = await this._decompressForSampling(producer, source, tempDir);
+              effectiveProducer = { ...producer, settings: { ...producer.settings, fileKey: decompressedFileKey, compressionType: void 0 } };
+              effectiveSource = { ...source, authentication: { ...source.authentication, path: tempDir } };
+            }
+            const fileData = await this.readFile(effectiveProducer, { readmode: "lines", lines: { from: 0, to: sampleSize } }, effectiveSource);
+            rawData = fileData.data;
+          } finally {
+            if (tempDir) fs15.rmSync(tempDir, { recursive: true, force: true });
+          }
           break;
         }
         default:
@@ -17346,13 +17365,54 @@ var ProducerEngineClass = class {
       }
       return dimensions;
     };
+    this._decompressForSampling = async (producer, source, tempDir) => {
+      const { fileKey, compressionType } = producer.settings;
+      const sourcePath = source.authentication["path"];
+      switch (compressionType?.toUpperCase()) {
+        case "GZ": {
+          const decompressedName = fileKey.replace(/\.gz$/i, "");
+          const outputPath = import_path14.default.join(tempDir, decompressedName);
+          const outputDir = import_path14.default.dirname(outputPath);
+          if (!fs15.existsSync(outputDir)) fs15.mkdirSync(outputDir, { recursive: true });
+          await ParseCompression_default.decompressToFile(compressionType, fileKey, sourcePath, outputPath);
+          return decompressedName;
+        }
+        case "ZIP":
+        case "TAR": {
+          const extractDir = import_path14.default.join(tempDir, "extracted");
+          await ParseCompression_default.decompressToFile(compressionType, fileKey, sourcePath, extractDir);
+          const files = this._findFilesRecursive(extractDir);
+          Affirm_default(files.length > 0, `No files found after decompressing "${fileKey}"`);
+          const firstFile = files[0];
+          const relativePath = import_path14.default.relative(extractDir, firstFile);
+          const destPath = import_path14.default.join(tempDir, relativePath);
+          const destDir = import_path14.default.dirname(destPath);
+          if (!fs15.existsSync(destDir)) fs15.mkdirSync(destDir, { recursive: true });
+          fs15.renameSync(firstFile, destPath);
+          return relativePath;
+        }
+        default:
+          throw new Error(`Unsupported compression type "${compressionType}" for producer "${producer.name}"`);
+      }
+    };
+    this._findFilesRecursive = (dir) => {
+      const results = [];
+      for (const entry of fs15.readdirSync(dir)) {
+        const fullPath = import_path14.default.join(dir, entry);
+        if (fs15.statSync(fullPath).isDirectory())
+          results.push(...this._findFilesRecursive(fullPath));
+        else
+          results.push(fullPath);
+      }
+      return results;
+    };
   }
 };
 var ProducerEngine = new ProducerEngineClass();
 var ProducerEngine_default = ProducerEngine;
 
 // ../../packages/engines/src/ai/DeveloperEngine.ts
-var import_path14 = __toESM(require("path"), 1);
+var import_path15 = __toESM(require("path"), 1);
 var import_promises6 = __toESM(require("fs/promises"), 1);
 var import_dayjs2 = __toESM(require("dayjs"), 1);
 var import_customParseFormat2 = __toESM(require("dayjs/plugin/customParseFormat"), 1);
@@ -17379,7 +17439,7 @@ var DeveloperEngineClass = class {
         _version: producer._version ?? 1
       };
       mappedProducer["$schema"] = producer["$schema"];
-      const producerPath = import_path14.default.join(process.cwd(), "remora", "producers", `${producer.name}.json`);
+      const producerPath = import_path15.default.join(process.cwd(), "remora", "producers", `${producer.name}.json`);
       await import_promises6.default.writeFile(producerPath, JSON.stringify(mappedProducer, null, 4), "utf-8");
       return { producer: mappedProducer, fields: typeDefinitions };
     };
@@ -17638,8 +17698,8 @@ var DeveloperEngineClass = class {
       Affirm_default(fileType, "Producer must have a fileType setting for mock data generation");
       const mockRecords = this.generateMockRecords(producer.dimensions, records);
       const basePath = source.authentication.path || process.cwd();
-      const filePath = import_path14.default.join(basePath, fileKey.replace("%", "mock"));
-      await import_promises6.default.mkdir(import_path14.default.dirname(filePath), { recursive: true });
+      const filePath = import_path15.default.join(basePath, fileKey.replace("%", "mock"));
+      await import_promises6.default.mkdir(import_path15.default.dirname(filePath), { recursive: true });
       const content = this.formatMockData(mockRecords, fileType, delimiter);
       await import_promises6.default.writeFile(filePath, content, "utf-8");
       return { filePath, recordCount: records };
@@ -18820,10 +18880,10 @@ var UsageManager = new UsageManagerClass();
 var UsageManager_default = UsageManager;
 
 // ../../packages/executors/src/OutputExecutor.ts
-var fs17 = __toESM(require("fs"));
+var fs18 = __toESM(require("fs"));
 
 // ../../packages/executors/src/ExecutorScope.ts
-var import_path15 = __toESM(require("path"));
+var import_path16 = __toESM(require("path"));
 var import_fs10 = __toESM(require("fs"));
 var import_promises7 = __toESM(require("fs/promises"));
 var ExecutorScopeClass2 = class {
@@ -18831,7 +18891,7 @@ var ExecutorScopeClass2 = class {
     this.WORKERS_FOLDER = "workers";
     this.PRODUCERS_FOLDER = "producers";
     this.getWorkerPath = (scope, workerId) => {
-      return import_path15.default.join(
+      return import_path16.default.join(
         Constants_default.defaults.REMORA_PATH,
         Constants_default.defaults.PRODUCER_TEMP_FOLDER,
         // A specific execution sits entirely in this folder, so at the end it's safe to delete it entirely
@@ -18841,7 +18901,7 @@ var ExecutorScopeClass2 = class {
       );
     };
     this.getProducerPath = (scope, producer, sourceFileKey) => {
-      return import_path15.default.join(
+      return import_path16.default.join(
         Constants_default.defaults.REMORA_PATH,
         Constants_default.defaults.PRODUCER_TEMP_FOLDER,
         // A specific execution sits entirely in this folder, so at the end it's safe to delete it entirely
@@ -18852,7 +18912,7 @@ var ExecutorScopeClass2 = class {
       );
     };
     this.getMainPath = (scope) => {
-      return import_path15.default.join(
+      return import_path16.default.join(
         Constants_default.defaults.REMORA_PATH,
         Constants_default.defaults.PRODUCER_TEMP_FOLDER,
         scope.folder,
@@ -18860,7 +18920,7 @@ var ExecutorScopeClass2 = class {
       );
     };
     this.clearScope = async (scope) => {
-      const scopePath = import_path15.default.join(
+      const scopePath = import_path16.default.join(
         Constants_default.defaults.REMORA_PATH,
         Constants_default.defaults.PRODUCER_TEMP_FOLDER,
         scope.folder
@@ -18870,7 +18930,7 @@ var ExecutorScopeClass2 = class {
       }
     };
     this.ensurePath = (fileUri) => {
-      const dir = import_path15.default.dirname(fileUri);
+      const dir = import_path16.default.dirname(fileUri);
       if (!import_fs10.default.existsSync(dir))
         import_fs10.default.mkdirSync(dir, { recursive: true });
       if (!import_fs10.default.existsSync(fileUri))
@@ -18882,7 +18942,7 @@ var ExecutorScope2 = new ExecutorScopeClass2();
 var ExecutorScope_default2 = ExecutorScope2;
 
 // ../../packages/executors/src/OutputExecutor.ts
-var import_path16 = __toESM(require("path"));
+var import_path17 = __toESM(require("path"));
 var OutputExecutorClass = class {
   constructor() {
     this._getInternalRecordFormat = (consumer) => {
@@ -18928,13 +18988,13 @@ var OutputExecutorClass = class {
       for (const output of consumer.outputs) {
         const destination = Environment_default.getSource(output.exportDestination);
         const driver = await DriverFactory_default.instantiateDestination(destination);
-        const currentPath = import_path16.default.dirname(ExecutorScope_default2.getMainPath(scope));
+        const currentPath = import_path17.default.dirname(ExecutorScope_default2.getMainPath(scope));
         const destinationName = this._composeFileName(consumer, output, this._getExtension(output));
         Logger_default.log(`Exporting consumer "${consumer.name}" to "${output.exportDestination}" as ${output.format} (${destinationName})`);
-        const filenameArray = fs17.readdirSync(currentPath).filter((filename) => filename.includes(".dataset"));
+        const filenameArray = fs18.readdirSync(currentPath).filter((filename) => filename.includes(".dataset"));
         for (const filename in filenameArray) {
           const destinationPath = this.getCompletedPath(destinationName, filename);
-          const startingPath = import_path16.default.join(currentPath, filenameArray[filename]);
+          const startingPath = import_path17.default.join(currentPath, filenameArray[filename]);
           if (output.format === internalFormat) {
             results.push(await driver.move(startingPath, destinationPath));
           } else {
@@ -18994,8 +19054,8 @@ var OutputExecutor_default = OutputExecutor;
 var ConsumerExecutorClass = class {
   constructor() {
     this._getWorkPath = (consumer, executionId) => {
-      const execFolder = import_path17.default.join(consumer.name, executionId);
-      const workPath = import_path17.default.join("./remora", Constants_default.defaults.PRODUCER_TEMP_FOLDER, execFolder, ".dataset");
+      const execFolder = import_path18.default.join(consumer.name, executionId);
+      const workPath = import_path18.default.join("./remora", Constants_default.defaults.PRODUCER_TEMP_FOLDER, execFolder, ".dataset");
       return workPath;
     };
     this._clearWorkPath = async (workPath) => {
@@ -19006,7 +19066,7 @@ var ConsumerExecutorClass = class {
       } catch (error) {
       }
       try {
-        const dir = import_path17.default.dirname(workPath);
+        const dir = import_path18.default.dirname(workPath);
         if (import_fs11.default.existsSync(dir)) {
           await import_promises8.default.rmdir(dir);
         }
@@ -19014,7 +19074,7 @@ var ConsumerExecutorClass = class {
       }
     };
     this._ensurePath = (pathUri) => {
-      const dir = import_path17.default.dirname(pathUri);
+      const dir = import_path18.default.dirname(pathUri);
       if (!import_fs11.default.existsSync(dir))
         import_fs11.default.mkdirSync(dir, { recursive: true });
       if (!import_fs11.default.existsSync(pathUri))
@@ -19410,7 +19470,7 @@ var ConsumerExecutor = new ConsumerExecutorClass();
 var ConsumerExecutor_default = ConsumerExecutor;
 
 // ../../packages/executors/src/ProducerExecutor.ts
-var import_path18 = __toESM(require("path"));
+var import_path19 = __toESM(require("path"));
 var ProducerExecutorClass = class {
   constructor() {
     this.ready = async (producer, scope) => {
@@ -19434,7 +19494,7 @@ var ProducerExecutorClass = class {
       counter = performance.now();
       for (const dimension of dimensions) {
         if (dimension.prodDimension.sourceFilename === true)
-          record[dimension.name] = import_path18.default.basename(chunk.fileUri);
+          record[dimension.name] = import_path19.default.basename(chunk.fileUri);
         const maskType = ProducerManager_default.getMask(dimension.prodDimension);
         if (Algo_default.hasVal(maskType))
           record[dimension.name] = CryptoEngine_default.hashValue(maskType, record[dimension.name]?.toString(), dimension.prodDimension.type);
@@ -19468,7 +19528,7 @@ var ExecutorPerformance_default = ExecutorPerformance;
 var import_os = __toESM(require("os"));
 var import_fs12 = __toESM(require("fs"));
 var import_promises9 = __toESM(require("fs/promises"));
-var import_path19 = __toESM(require("path"));
+var import_path20 = __toESM(require("path"));
 var import_workerpool = __toESM(require("workerpool"));
 
 // ../../packages/executors/src/ExecutorProgress.ts
@@ -19526,7 +19586,7 @@ var ExecutorProgress = class {
 var ExecutorProgress_default = ExecutorProgress;
 
 // ../../packages/executors/src/ExecutorWriter.ts
-var fs19 = __toESM(require("fs"));
+var fs20 = __toESM(require("fs"));
 var import_readline7 = __toESM(require("readline"));
 var ExecutorWriter = class {
   constructor() {
@@ -19543,11 +19603,11 @@ var ExecutorWriter = class {
     };
     this.splitBySize = async (scope, sourcePath) => {
       const maxOutputFileSize = scope.limitFileSize * this.FAKE_GB;
-      const readStream = fs19.createReadStream(sourcePath);
+      const readStream = fs20.createReadStream(sourcePath);
       const reader = import_readline7.default.createInterface({ input: readStream, crlfDelay: Infinity });
       let writerIndex = 0;
       let destPath = this.getCompletedPath(sourcePath, writerIndex);
-      let writeStream = fs19.createWriteStream(destPath, { flags: "a" });
+      let writeStream = fs20.createWriteStream(destPath, { flags: "a" });
       for await (const line of reader) {
         if (readStream.bytesRead > maxOutputFileSize * (writerIndex + 1)) {
           writeStream.end();
@@ -19557,7 +19617,7 @@ var ExecutorWriter = class {
           });
           writerIndex++;
           destPath = this.getCompletedPath(sourcePath, writerIndex);
-          writeStream = fs19.createWriteStream(destPath, { flags: "a" });
+          writeStream = fs20.createWriteStream(destPath, { flags: "a" });
         }
         writeStream.write(line + "\n");
       }
@@ -19566,7 +19626,7 @@ var ExecutorWriter = class {
         writeStream.on("finish", resolve);
         writeStream.on("error", reject);
       });
-      await fs19.promises.unlink(sourcePath);
+      await fs20.promises.unlink(sourcePath);
     };
     /**
      * Manage the Writestream for main.dataset
@@ -19636,7 +19696,7 @@ var ExecutorOrchestratorClass = class {
       };
       const workerPath = this._getWorkerPath();
       Logger_default.log(`Initializing worker pool from ${workerPath} (heap limit: ${Constants_default.defaults.MIN_RUNTIME_HEAP_MB}MB)`);
-      return import_workerpool.default.pool(import_path19.default.join(workerPath, "ExecutorWorker.js"), options);
+      return import_workerpool.default.pool(import_path20.default.join(workerPath, "ExecutorWorker.js"), options);
     };
     this.launch = async (request) => {
       Affirm_default(request, "Invalid options");
@@ -19757,6 +19817,14 @@ var ExecutorOrchestratorClass = class {
           }
           Logger_default.log(`[${usageId}] Dataset validations complete in ${Math.round(performance.now() - counter)}ms`);
         }
+        if (scope.limitFileSize) {
+          Logger_default.log(`[${usageId}] Splitting output by size limit (${scope.limitFileSize})`);
+          counter = performance.now();
+          const writer = new ExecutorWriter_default();
+          await writer.splitBySize(scope, ExecutorScope_default2.getMainPath(scope));
+          tracker.measure("split-by-size", performance.now() - counter);
+          Logger_default.log(`[${usageId}] Split complete in ${Math.round(performance.now() - counter)}ms`);
+        }
         counter = performance.now();
         Logger_default.log(`[${usageId}] Exporting results to ${consumer.outputs.length} output(s)`);
         const exportRes = await OutputExecutor_default.exportResult(consumer, ConsumerManager_default.getExpandedFields(consumer), scope);
@@ -19864,20 +19932,19 @@ var ExecutorOrchestratorClass = class {
     this._getWorkerPath = () => {
       const currentDir = __dirname;
       if (ProcessENVManager_default.getEnvVariable("NODE_ENV") === "dev" || ProcessENVManager_default.getEnvVariable("NODE_ENV") === "development")
-        return import_path19.default.resolve("./.build/workers");
+        return import_path20.default.resolve("./.build/workers");
       const forcedPath = ProcessENVManager_default.getEnvVariable("REMORA_WORKERS_PATH");
       if (forcedPath && forcedPath.length > 0)
-        return import_path19.default.join(__dirname, forcedPath);
+        return import_path20.default.join(__dirname, forcedPath);
       if (!currentDir.includes(".build")) {
-        return import_path19.default.join(__dirname, "../workers");
+        return import_path20.default.join(__dirname, "../workers");
       } else {
-        return import_path19.default.resolve("./.build/workers");
+        return import_path20.default.resolve("./.build/workers");
       }
     };
     this.reconcileExecutorThreadsResults = async (scope, executorResults, tracker) => {
       const mainPath = ExecutorScope_default2.getMainPath(scope);
       ConsumerExecutor_default._ensurePath(mainPath);
-      const writer = new ExecutorWriter_default();
       if (executorResults.length > 1) {
         Logger_default.log(`[${scope.id}] Merging ${executorResults.length} worker output files into ${mainPath}`);
         const perf = performance.now();
@@ -19893,10 +19960,6 @@ var ExecutorOrchestratorClass = class {
         Logger_default.log(`[${scope.id}] Single worker \u2014 renaming output to ${mainPath}`);
         await import_promises9.default.rename(executorResults[0].resultUri, mainPath);
       }
-      if (scope.limitFileSize) {
-        Logger_default.log(`[${scope.id}] Splitting output by size limit (${scope.limitFileSize})`);
-        await writer.splitBySize(scope, mainPath);
-      }
       return mainPath;
     };
     this.performCleanupOperations = async (scope, tracker) => {
@@ -20083,19 +20146,19 @@ var discover = async (producerName) => {
 // src/actions/create_producer.ts
 var import_chalk8 = __toESM(require("chalk"));
 var import_fs13 = __toESM(require("fs"));
-var import_path20 = __toESM(require("path"));
+var import_path21 = __toESM(require("path"));
 var create_producer = async (name) => {
   try {
     if (!import_fs13.default.existsSync("./remora/producers")) {
       throw new Error(import_chalk8.default.red("Missing directory: ") + import_chalk8.default.yellow("./remora/producers"));
     }
     const defaultProducerTemplate = import_fs13.default.readFileSync(
-      import_path20.default.join(DOCUMENTATION_DIR, "default_resources/producer.json"),
+      import_path21.default.join(DOCUMENTATION_DIR, "default_resources/producer.json"),
       "utf-8"
     );
     const defaultProducer = JSON.parse(defaultProducerTemplate);
     defaultProducer.name = name;
-    const producerPath = import_path20.default.join("remora/producers", `${name}.json`);
+    const producerPath = import_path21.default.join("remora/producers", `${name}.json`);
     import_fs13.default.writeFileSync(producerPath, JSON.stringify(defaultProducer, null, 4));
     console.log(import_chalk8.default.green(`\u2705 Created producer config at ${producerPath}`));
     console.log(import_chalk8.default.blue("Remember to:"));
@@ -20112,20 +20175,20 @@ var create_producer = async (name) => {
 // src/actions/create_consumer.ts
 var import_chalk9 = __toESM(require("chalk"));
 var import_fs14 = __toESM(require("fs"));
-var import_path21 = __toESM(require("path"));
+var import_path22 = __toESM(require("path"));
 var create_consumer = async (name, producerName) => {
   try {
     if (!import_fs14.default.existsSync("./remora/consumers")) {
       throw new Error(import_chalk9.default.red("Missing directory: ") + import_chalk9.default.yellow("./remora/consumers"));
     }
     const defaultConsumerTemplate = import_fs14.default.readFileSync(
-      import_path21.default.join(DOCUMENTATION_DIR, "default_resources/consumer.json"),
+      import_path22.default.join(DOCUMENTATION_DIR, "default_resources/consumer.json"),
       "utf-8"
     );
     const defaultConsumer = JSON.parse(defaultConsumerTemplate);
     defaultConsumer.name = name;
     if (producerName) {
-      const producerPath = import_path21.default.join("remora/producers", `${producerName}.json`);
+      const producerPath = import_path22.default.join("remora/producers", `${producerName}.json`);
       if (!import_fs14.default.existsSync(producerPath)) {
         throw new Error(import_chalk9.default.red("Producer not found: ") + import_chalk9.default.yellow(producerPath));
       }
@@ -20146,7 +20209,7 @@ var create_consumer = async (name, producerName) => {
       defaultConsumer.filters = void 0;
       defaultConsumer.metadata = void 0;
     }
-    const consumerPath = import_path21.default.join("remora/consumers", `${name}.json`);
+    const consumerPath = import_path22.default.join("remora/consumers", `${name}.json`);
     import_fs14.default.writeFileSync(consumerPath, JSON.stringify(defaultConsumer, null, 4));
     console.log(import_chalk9.default.green(`\u2705 Created consumer config at ${consumerPath}`));
     if (!producerName) {
@@ -20171,7 +20234,7 @@ var create_consumer = async (name, producerName) => {
 var import_chalk10 = __toESM(require("chalk"));
 var import_ora5 = __toESM(require("ora"));
 var import_fs15 = __toESM(require("fs"));
-var import_path22 = __toESM(require("path"));
+var import_path23 = __toESM(require("path"));
 var automap = async (producerName, schemaNames) => {
   try {
     compile();
@@ -20196,12 +20259,12 @@ var automap = async (producerName, schemaNames) => {
     const sampleStrings = sampleData.map((item) => JSON.stringify(item));
     const mapResult = await AutoMapperEngine_default.map(sampleStrings, schemas, producer.settings.fileKey, [source]);
     for (const producer2 of mapResult.producers) {
-      const producerPath = import_path22.default.join("remora/producers", `${producer2.name}.json`);
+      const producerPath = import_path23.default.join("remora/producers", `${producer2.name}.json`);
       import_fs15.default.writeFileSync(producerPath, JSON.stringify(producer2, null, 4));
       console.log(import_chalk10.default.blue(`Created producer: ${producer2.name}`));
     }
     for (const consumer of mapResult.consumers) {
-      const consumerPath = import_path22.default.join("remora/consumers", `${consumer.name}.json`);
+      const consumerPath = import_path23.default.join("remora/consumers", `${consumer.name}.json`);
       import_fs15.default.writeFileSync(consumerPath, JSON.stringify(consumer, null, 4));
       console.log(import_chalk10.default.blue(`Created consumer: ${consumer.name}`));
     }