@forzalabs/remora 1.2.1 → 1.2.3

package/CHANGELOG.md

@@ -6,6 +6,8 @@ The format is based on Keep a Changelog, and this project adheres to Semantic Ve
 
 ## Unreleased
 
+## V 1.2.2 - 2026-04-10
+
 ### Added
 - Added field-level consumer validations with support for multiple rules per field and per-rule failure actions: `fail`, `skip`, `warn`, and `set_default`
 - Added dataset-level consumer validations for `unique_fields`, `min_rows`, `max_rows`, `no_duplicates`, and `not_empty`
@@ -14,6 +16,8 @@ The format is based on Keep a Changelog, and this project adheres to Semantic Ve
 - Added `warn()` logging support for non-fatal validation outcomes
 - Added canary consumer coverage for field-level and dataset-level validations with passing, warning, skipped, defaulted, and failing scenarios
 - Added `verify:local` to the canary package to build the local CLI and run the canary suite against it instead of the published package
+- Added canary coverage for `sample` and `discover` against gzipped local producer inputs
+- Added compressed CSV and JSONL canary fixtures plus matching producer definitions for gzip-based CLI verification
 
 ### Changed
 - Updated consumer field validation configuration from a single flat validation object to an ordered array of validation rules with explicit `onFail` behavior
@@ -21,6 +25,8 @@ The format is based on Keep a Changelog, and this project adheres to Semantic Ve
 ### Fixed
 - Fixed the consumer JSON schema to support the new field-level and dataset-level validation configuration
 - Fixed AJV strict-mode compatibility for validation `in` and `not_in` rule arrays by replacing union `type` declarations with `oneOf`
+- Fixed CLI `sample` and `discover` commands so they can read producers configured with compressed local files by reusing the existing decompression logic during sampling
+- Fixed conflict in export with unique values and split by max size
 
 ## V 1.1.15 - 2026-03-26
 

package/index.js

@@ -9655,15 +9655,15 @@ var require_index_cjs = __commonJS({
       };
     }
     function wrapConversion(toModel, graph) {
-      const path23 = [graph[toModel].parent, toModel];
+      const path24 = [graph[toModel].parent, toModel];
       let fn = convert$1[graph[toModel].parent][toModel];
       let cur = graph[toModel].parent;
       while (graph[cur].parent) {
-        path23.unshift(graph[cur].parent);
+        path24.unshift(graph[cur].parent);
         fn = link(convert$1[graph[cur].parent][cur], fn);
         cur = graph[cur].parent;
       }
-      fn.conversion = path23;
+      fn.conversion = path24;
       return fn;
     }
     function route(fromModel) {
@@ -10389,7 +10389,7 @@ var require_file = __commonJS({
   "../../packages/logger/node_modules/winston/lib/winston/transports/file.js"(exports2, module2) {
     "use strict";
     var fs24 = require("fs");
-    var path23 = require("path");
+    var path24 = require("path");
     var asyncSeries = require_series();
     var zlib2 = require("zlib");
     var { MESSAGE } = require_triple_beam();
@@ -10419,14 +10419,14 @@ var require_file = __commonJS({
         this._onError = this._onError.bind(this);
         if (options.filename || options.dirname) {
           throwIf("filename or dirname", "stream");
-          this._basename = this.filename = options.filename ? path23.basename(options.filename) : "winston.log";
-          this.dirname = options.dirname || path23.dirname(options.filename);
+          this._basename = this.filename = options.filename ? path24.basename(options.filename) : "winston.log";
+          this.dirname = options.dirname || path24.dirname(options.filename);
           this.options = options.options || { flags: "a" };
         } else if (options.stream) {
           console.warn("options.stream will be removed in winston@4. Use winston.transports.Stream");
           throwIf("stream", "filename", "maxsize");
           this._dest = this._stream.pipe(this._setupStream(options.stream));
-          this.dirname = path23.dirname(this._dest.path);
+          this.dirname = path24.dirname(this._dest.path);
         } else {
           throw new Error("Cannot log to file without filename or stream.");
         }
@@ -10589,7 +10589,7 @@ var require_file = __commonJS({
           options = {};
         }
         options = normalizeQuery(options);
-        const file = path23.join(this.dirname, this.filename);
+        const file = path24.join(this.dirname, this.filename);
         let buff = "";
         let results = [];
         let row = 0;
@@ -10694,7 +10694,7 @@ var require_file = __commonJS({
        * TODO: Refactor me.
        */
       stream(options = {}) {
-        const file = path23.join(this.dirname, this.filename);
+        const file = path24.join(this.dirname, this.filename);
         const stream = new Stream();
         const tail = {
           file,
@@ -10744,7 +10744,7 @@ var require_file = __commonJS({
        */
       stat(callback) {
         const target = this._getFile();
-        const fullpath = path23.join(this.dirname, target);
+        const fullpath = path24.join(this.dirname, target);
         fs24.stat(fullpath, (err, stat) => {
           if (err && err.code === "ENOENT") {
             debug2("ENOENT\xA0ok", fullpath);
@@ -10848,7 +10848,7 @@ var require_file = __commonJS({
        * @returns {WritableStream} Stream that writes to disk for the active file.
        */
       _createStream(source) {
-        const fullpath = path23.join(this.dirname, this.filename);
+        const fullpath = path24.join(this.dirname, this.filename);
         debug2("create stream start", fullpath, this.options);
         const dest = fs24.createWriteStream(fullpath, this.options).on("error", (err) => debug2(err)).on("close", () => debug2("close", dest.path, dest.bytesWritten)).on("open", () => {
           debug2("file open ok", fullpath);
@@ -10873,16 +10873,16 @@ var require_file = __commonJS({
        */
       _incFile(callback) {
         debug2("_incFile", this.filename);
-        const ext = path23.extname(this._basename);
-        const basename = path23.basename(this._basename, ext);
+        const ext = path24.extname(this._basename);
+        const basename = path24.basename(this._basename, ext);
         const tasks = [];
         if (this.zippedArchive) {
           tasks.push(
             function(cb) {
               const num = this._created > 0 && !this.tailable ? this._created : "";
               this._compressFile(
-                path23.join(this.dirname, `${basename}${num}${ext}`),
-                path23.join(this.dirname, `${basename}${num}${ext}.gz`),
+                path24.join(this.dirname, `${basename}${num}${ext}`),
+                path24.join(this.dirname, `${basename}${num}${ext}.gz`),
                 cb
               );
             }.bind(this)
@@ -10907,8 +10907,8 @@ var require_file = __commonJS({
        * @private
        */
       _getFile() {
-        const ext = path23.extname(this._basename);
-        const basename = path23.basename(this._basename, ext);
+        const ext = path24.extname(this._basename);
+        const basename = path24.basename(this._basename, ext);
         const isRotation = this.rotationFormat ? this.rotationFormat() : this._created;
         return !this.tailable && this._created ? `${basename}${isRotation}${ext}` : `${basename}${ext}`;
       }
@@ -10928,7 +10928,7 @@ var require_file = __commonJS({
         const isOldest = oldest !== 0 ? oldest : "";
         const isZipped = this.zippedArchive ? ".gz" : "";
         const filePath = `${basename}${isOldest}${ext}${isZipped}`;
-        const target = path23.join(this.dirname, filePath);
+        const target = path24.join(this.dirname, filePath);
         fs24.unlink(target, callback);
       }
       /**
@@ -10951,20 +10951,20 @@ var require_file = __commonJS({
         for (let x = this.maxFiles - 1; x > 1; x--) {
           tasks.push(function(i, cb) {
             let fileName = `${basename}${i - 1}${ext}${isZipped}`;
-            const tmppath = path23.join(this.dirname, fileName);
+            const tmppath = path24.join(this.dirname, fileName);
             fs24.exists(tmppath, (exists) => {
               if (!exists) {
                 return cb(null);
               }
               fileName = `${basename}${i}${ext}${isZipped}`;
-              fs24.rename(tmppath, path23.join(this.dirname, fileName), cb);
+              fs24.rename(tmppath, path24.join(this.dirname, fileName), cb);
             });
           }.bind(this, x));
         }
         asyncSeries(tasks, () => {
           fs24.rename(
-            path23.join(this.dirname, `${basename}${ext}${isZipped}`),
-            path23.join(this.dirname, `${basename}1${ext}${isZipped}`),
+            path24.join(this.dirname, `${basename}${ext}${isZipped}`),
+            path24.join(this.dirname, `${basename}1${ext}${isZipped}`),
             callback
           );
         });
@@ -11077,9 +11077,9 @@ var require_http = __commonJS({
         };
         const auth = options.params.auth || null;
         delete options.params.auth;
-        const path23 = options.params.path || null;
+        const path24 = options.params.path || null;
         delete options.params.path;
-        this._request(options, auth, path23, (err, res, body) => {
+        this._request(options, auth, path24, (err, res, body) => {
           if (res && res.statusCode !== 200) {
             err = new Error(`Invalid HTTP Status Code: ${res.statusCode}`);
           }
@@ -11107,12 +11107,12 @@ var require_http = __commonJS({
           method: "stream",
           params: options
         };
-        const path23 = options.params.path || null;
+        const path24 = options.params.path || null;
         delete options.params.path;
         const auth = options.params.auth || null;
         delete options.params.auth;
         let buff = "";
-        const req = this._request(options, auth, path23);
+        const req = this._request(options, auth, path24);
         stream.destroy = () => req.destroy();
         req.on("data", (data) => {
           data = (buff + data).split(/\n+/);
@@ -11138,14 +11138,14 @@ var require_http = __commonJS({
        * @param {string} path - request path
        * @param {function} callback - Continuation to respond to when complete.
        */
-      _request(options, auth, path23, callback) {
+      _request(options, auth, path24, callback) {
         options = options || {};
         auth = auth || this.auth;
-        path23 = path23 || this.path || "";
+        path24 = path24 || this.path || "";
         if (this.batch) {
-          this._doBatch(options, callback, auth, path23);
+          this._doBatch(options, callback, auth, path24);
         } else {
-          this._doRequest(options, callback, auth, path23);
+          this._doRequest(options, callback, auth, path24);
         }
       }
       /**
@@ -11155,18 +11155,18 @@ var require_http = __commonJS({
        * @param {Object?} auth - authentication options
        * @param {string} path - request path
        */
-      _doBatch(options, callback, auth, path23) {
+      _doBatch(options, callback, auth, path24) {
         this.batchOptions.push(options);
         if (this.batchOptions.length === 1) {
           const me = this;
           this.batchCallback = callback;
           this.batchTimeoutID = setTimeout(function() {
             me.batchTimeoutID = -1;
-            me._doBatchRequest(me.batchCallback, auth, path23);
+            me._doBatchRequest(me.batchCallback, auth, path24);
           }, this.batchInterval);
         }
         if (this.batchOptions.length === this.batchCount) {
-          this._doBatchRequest(this.batchCallback, auth, path23);
+          this._doBatchRequest(this.batchCallback, auth, path24);
         }
       }
       /**
@@ -11175,14 +11175,14 @@ var require_http = __commonJS({
        * @param {Object?} auth - authentication options
        * @param {string} path - request path
        */
-      _doBatchRequest(callback, auth, path23) {
+      _doBatchRequest(callback, auth, path24) {
         if (this.batchTimeoutID > 0) {
           clearTimeout(this.batchTimeoutID);
           this.batchTimeoutID = -1;
         }
         const batchOptionsCopy = this.batchOptions.slice();
         this.batchOptions = [];
-        this._doRequest(batchOptionsCopy, callback, auth, path23);
+        this._doRequest(batchOptionsCopy, callback, auth, path24);
       }
       /**
        * Make a request to a winstond server or any http server which can
@@ -11192,7 +11192,7 @@ var require_http = __commonJS({
        * @param {Object?} auth - authentication options
        * @param {string} path - request path
        */
-      _doRequest(options, callback, auth, path23) {
+      _doRequest(options, callback, auth, path24) {
         const headers = Object.assign({}, this.headers);
         if (auth && auth.bearer) {
           headers.Authorization = `Bearer ${auth.bearer}`;
@@ -11202,7 +11202,7 @@ var require_http = __commonJS({
           method: "POST",
           host: this.host,
           port: this.port,
-          path: `/${path23.replace(/^\//, "")}`,
+          path: `/${path24.replace(/^\//, "")}`,
           headers,
           auth: auth && auth.username && auth.password ? `${auth.username}:${auth.password}` : "",
           agent: this.agent
@@ -15102,8 +15102,8 @@ var HttpClientClass = class {
         throw new Error(err.message ?? "An error occurred on GET.");
       }
     };
-    this.getWorkerUrl = (path23) => {
-      const cleanPath = path23.startsWith("/") ? path23.substring(1) : path23;
+    this.getWorkerUrl = (path24) => {
+      const cleanPath = path24.startsWith("/") ? path24.substring(1) : path24;
       return `${this.workerHost}/${cleanPath}`;
     };
     this.workerHost = ProcessENVManager_default.getEnvVariable("REMORA_WORKER_HOST") || "http://worker:5069";
@@ -16676,7 +16676,7 @@ var LicenceManager = new LicenceManagerClass();
 var LicenceManager_default = LicenceManager;
 
 // ../../packages/executors/src/ConsumerExecutor.ts
-var import_path17 = __toESM(require("path"));
+var import_path18 = __toESM(require("path"));
 var import_fs11 = __toESM(require("fs"));
 var import_readline6 = __toESM(require("readline"));
 var import_promises8 = __toESM(require("fs/promises"));
@@ -17062,6 +17062,10 @@ var AutoMapperEngineClass = class {
 var AutoMapperEngine = new AutoMapperEngineClass();
 var AutoMapperEngine_default = AutoMapperEngine;
 
+// ../../packages/engines/src/producer/ProducerEngine.ts
+var import_path14 = __toESM(require("path"), 1);
+var import_crypto3 = require("crypto");
+
 // ../../packages/engines/src/transform/TypeCaster.ts
 var import_dayjs = __toESM(require("dayjs"), 1);
 var import_customParseFormat = __toESM(require("dayjs/plugin/customParseFormat"), 1);
@@ -17181,12 +17185,12 @@ var noopTracker = { _operations: {}, measure: () => {
 }, getOperations: () => ({}) };
 var ProducerEngineClass = class {
   constructor() {
-    this.readFile = async (producer, options) => {
+    this.readFile = async (producer, options, sourceOverride) => {
       Affirm_default(producer, "Invalid producer");
       Affirm_default(options, "Invalid options");
       if (options.readmode === "lines")
         Affirm_default(options.lines, "Invalid lines");
-      const source = Environment_default.getSource(producer.source);
+      const source = sourceOverride ?? Environment_default.getSource(producer.source);
       Affirm_default(source, `No source found for producer "${producer.name}" with name "${producer.source}"`);
       const driver = await DriverFactory_default.instantiateSource(source);
       Affirm_default(driver, `No driver found for producer "${producer.name}" with driver type "${source.engine}"`);
@@ -17260,8 +17264,32 @@ var ProducerEngineClass = class {
         case "local":
         case "aws-s3":
         case "delta-share": {
-          const fileData = await this.readFile(producer, { readmode: "lines", lines: { from: 0, to: sampleSize } });
-          rawData = fileData.data;
+          const { compressionType } = producer.settings;
+          let scope = null;
+          try {
+            let effectiveProducer = producer;
+            let effectiveSource = source;
+            if (compressionType) {
+              scope = { id: (0, import_crypto3.randomUUID)(), folder: `sample-${(0, import_crypto3.randomUUID)()}`, workersId: [] };
+              const driver = await DriverFactory_default.instantiateSource(source);
+              const readyResult = await driver.ready({ producer, scope });
+              const firstFile = readyResult.files[0].fullUri;
+              const decompressedDir = import_path14.default.dirname(firstFile);
+              effectiveSource = { ...source, engine: "local", authentication: { ...source.authentication, path: decompressedDir } };
+              effectiveProducer = {
+                ...producer,
+                settings: {
+                  ...producer.settings,
+                  fileKey: readyResult.files.length > 1 ? "%.dataset" : import_path14.default.basename(firstFile),
+                  compressionType: void 0
+                }
+              };
+            }
+            const fileData = await this.readFile(effectiveProducer, { readmode: "lines", lines: { from: 0, to: sampleSize } }, effectiveSource);
+            rawData = fileData.data;
+          } finally {
+            if (scope) await ExecutorScope_default.clearScope(scope);
+          }
           break;
         }
         default:
@@ -17352,7 +17380,7 @@ var ProducerEngine = new ProducerEngineClass();
 var ProducerEngine_default = ProducerEngine;
 
 // ../../packages/engines/src/ai/DeveloperEngine.ts
-var import_path14 = __toESM(require("path"), 1);
+var import_path15 = __toESM(require("path"), 1);
 var import_promises6 = __toESM(require("fs/promises"), 1);
 var import_dayjs2 = __toESM(require("dayjs"), 1);
 var import_customParseFormat2 = __toESM(require("dayjs/plugin/customParseFormat"), 1);
@@ -17379,7 +17407,7 @@ var DeveloperEngineClass = class {
         _version: producer._version ?? 1
       };
       mappedProducer["$schema"] = producer["$schema"];
-      const producerPath = import_path14.default.join(process.cwd(), "remora", "producers", `${producer.name}.json`);
+      const producerPath = import_path15.default.join(process.cwd(), "remora", "producers", `${producer.name}.json`);
       await import_promises6.default.writeFile(producerPath, JSON.stringify(mappedProducer, null, 4), "utf-8");
       return { producer: mappedProducer, fields: typeDefinitions };
     };
@@ -17638,8 +17666,8 @@ var DeveloperEngineClass = class {
       Affirm_default(fileType, "Producer must have a fileType setting for mock data generation");
       const mockRecords = this.generateMockRecords(producer.dimensions, records);
       const basePath = source.authentication.path || process.cwd();
-      const filePath = import_path14.default.join(basePath, fileKey.replace("%", "mock"));
-      await import_promises6.default.mkdir(import_path14.default.dirname(filePath), { recursive: true });
+      const filePath = import_path15.default.join(basePath, fileKey.replace("%", "mock"));
+      await import_promises6.default.mkdir(import_path15.default.dirname(filePath), { recursive: true });
       const content = this.formatMockData(mockRecords, fileType, delimiter);
       await import_promises6.default.writeFile(filePath, content, "utf-8");
       return { filePath, recordCount: records };
@@ -18823,7 +18851,7 @@ var UsageManager_default = UsageManager;
 var fs17 = __toESM(require("fs"));
 
 // ../../packages/executors/src/ExecutorScope.ts
-var import_path15 = __toESM(require("path"));
+var import_path16 = __toESM(require("path"));
 var import_fs10 = __toESM(require("fs"));
 var import_promises7 = __toESM(require("fs/promises"));
 var ExecutorScopeClass2 = class {
@@ -18831,7 +18859,7 @@ var ExecutorScopeClass2 = class {
     this.WORKERS_FOLDER = "workers";
     this.PRODUCERS_FOLDER = "producers";
     this.getWorkerPath = (scope, workerId) => {
-      return import_path15.default.join(
+      return import_path16.default.join(
         Constants_default.defaults.REMORA_PATH,
         Constants_default.defaults.PRODUCER_TEMP_FOLDER,
         // A specific execution sits entirely in this folder, so at the end it's safe to delete it entirely
@@ -18841,7 +18869,7 @@ var ExecutorScopeClass2 = class {
       );
     };
     this.getProducerPath = (scope, producer, sourceFileKey) => {
-      return import_path15.default.join(
+      return import_path16.default.join(
         Constants_default.defaults.REMORA_PATH,
         Constants_default.defaults.PRODUCER_TEMP_FOLDER,
         // A specific execution sits entirely in this folder, so at the end it's safe to delete it entirely
@@ -18852,7 +18880,7 @@ var ExecutorScopeClass2 = class {
       );
     };
     this.getMainPath = (scope) => {
-      return import_path15.default.join(
+      return import_path16.default.join(
         Constants_default.defaults.REMORA_PATH,
         Constants_default.defaults.PRODUCER_TEMP_FOLDER,
         scope.folder,
@@ -18860,7 +18888,7 @@ var ExecutorScopeClass2 = class {
       );
     };
     this.clearScope = async (scope) => {
-      const scopePath = import_path15.default.join(
+      const scopePath = import_path16.default.join(
         Constants_default.defaults.REMORA_PATH,
         Constants_default.defaults.PRODUCER_TEMP_FOLDER,
         scope.folder
@@ -18870,7 +18898,7 @@ var ExecutorScopeClass2 = class {
       }
     };
     this.ensurePath = (fileUri) => {
-      const dir = import_path15.default.dirname(fileUri);
+      const dir = import_path16.default.dirname(fileUri);
       if (!import_fs10.default.existsSync(dir))
         import_fs10.default.mkdirSync(dir, { recursive: true });
       if (!import_fs10.default.existsSync(fileUri))
@@ -18882,7 +18910,7 @@ var ExecutorScope2 = new ExecutorScopeClass2();
 var ExecutorScope_default2 = ExecutorScope2;
 
 // ../../packages/executors/src/OutputExecutor.ts
-var import_path16 = __toESM(require("path"));
+var import_path17 = __toESM(require("path"));
 var OutputExecutorClass = class {
   constructor() {
     this._getInternalRecordFormat = (consumer) => {
@@ -18928,13 +18956,13 @@ var OutputExecutorClass = class {
       for (const output of consumer.outputs) {
         const destination = Environment_default.getSource(output.exportDestination);
         const driver = await DriverFactory_default.instantiateDestination(destination);
-        const currentPath = import_path16.default.dirname(ExecutorScope_default2.getMainPath(scope));
+        const currentPath = import_path17.default.dirname(ExecutorScope_default2.getMainPath(scope));
         const destinationName = this._composeFileName(consumer, output, this._getExtension(output));
         Logger_default.log(`Exporting consumer "${consumer.name}" to "${output.exportDestination}" as ${output.format} (${destinationName})`);
         const filenameArray = fs17.readdirSync(currentPath).filter((filename) => filename.includes(".dataset"));
         for (const filename in filenameArray) {
           const destinationPath = this.getCompletedPath(destinationName, filename);
-          const startingPath = import_path16.default.join(currentPath, filenameArray[filename]);
+          const startingPath = import_path17.default.join(currentPath, filenameArray[filename]);
           if (output.format === internalFormat) {
             results.push(await driver.move(startingPath, destinationPath));
           } else {
@@ -18994,8 +19022,8 @@ var OutputExecutor_default = OutputExecutor;
 var ConsumerExecutorClass = class {
   constructor() {
     this._getWorkPath = (consumer, executionId) => {
-      const execFolder = import_path17.default.join(consumer.name, executionId);
-      const workPath = import_path17.default.join("./remora", Constants_default.defaults.PRODUCER_TEMP_FOLDER, execFolder, ".dataset");
+      const execFolder = import_path18.default.join(consumer.name, executionId);
+      const workPath = import_path18.default.join("./remora", Constants_default.defaults.PRODUCER_TEMP_FOLDER, execFolder, ".dataset");
       return workPath;
     };
     this._clearWorkPath = async (workPath) => {
@@ -19006,7 +19034,7 @@ var ConsumerExecutorClass = class {
       } catch (error) {
       }
       try {
-        const dir = import_path17.default.dirname(workPath);
+        const dir = import_path18.default.dirname(workPath);
         if (import_fs11.default.existsSync(dir)) {
           await import_promises8.default.rmdir(dir);
         }
@@ -19014,7 +19042,7 @@ var ConsumerExecutorClass = class {
       }
     };
     this._ensurePath = (pathUri) => {
-      const dir = import_path17.default.dirname(pathUri);
+      const dir = import_path18.default.dirname(pathUri);
       if (!import_fs11.default.existsSync(dir))
         import_fs11.default.mkdirSync(dir, { recursive: true });
       if (!import_fs11.default.existsSync(pathUri))
@@ -19410,7 +19438,7 @@ var ConsumerExecutor = new ConsumerExecutorClass();
 var ConsumerExecutor_default = ConsumerExecutor;
 
 // ../../packages/executors/src/ProducerExecutor.ts
-var import_path18 = __toESM(require("path"));
+var import_path19 = __toESM(require("path"));
 var ProducerExecutorClass = class {
   constructor() {
     this.ready = async (producer, scope) => {
@@ -19434,7 +19462,7 @@ var ProducerExecutorClass = class {
       counter = performance.now();
       for (const dimension of dimensions) {
         if (dimension.prodDimension.sourceFilename === true)
-          record[dimension.name] = import_path18.default.basename(chunk.fileUri);
+          record[dimension.name] = import_path19.default.basename(chunk.fileUri);
         const maskType = ProducerManager_default.getMask(dimension.prodDimension);
         if (Algo_default.hasVal(maskType))
           record[dimension.name] = CryptoEngine_default.hashValue(maskType, record[dimension.name]?.toString(), dimension.prodDimension.type);
@@ -19468,7 +19496,7 @@ var ExecutorPerformance_default = ExecutorPerformance;
 var import_os = __toESM(require("os"));
 var import_fs12 = __toESM(require("fs"));
 var import_promises9 = __toESM(require("fs/promises"));
-var import_path19 = __toESM(require("path"));
+var import_path20 = __toESM(require("path"));
 var import_workerpool = __toESM(require("workerpool"));
 
 // ../../packages/executors/src/ExecutorProgress.ts
@@ -19636,7 +19664,7 @@ var ExecutorOrchestratorClass = class {
       };
       const workerPath = this._getWorkerPath();
       Logger_default.log(`Initializing worker pool from ${workerPath} (heap limit: ${Constants_default.defaults.MIN_RUNTIME_HEAP_MB}MB)`);
-      return import_workerpool.default.pool(import_path19.default.join(workerPath, "ExecutorWorker.js"), options);
+      return import_workerpool.default.pool(import_path20.default.join(workerPath, "ExecutorWorker.js"), options);
     };
     this.launch = async (request) => {
       Affirm_default(request, "Invalid options");
@@ -19757,6 +19785,14 @@ var ExecutorOrchestratorClass = class {
           }
           Logger_default.log(`[${usageId}] Dataset validations complete in ${Math.round(performance.now() - counter)}ms`);
         }
+        if (scope.limitFileSize) {
+          Logger_default.log(`[${usageId}] Splitting output by size limit (${scope.limitFileSize})`);
+          counter = performance.now();
+          const writer = new ExecutorWriter_default();
+          await writer.splitBySize(scope, ExecutorScope_default2.getMainPath(scope));
+          tracker.measure("split-by-size", performance.now() - counter);
+          Logger_default.log(`[${usageId}] Split complete in ${Math.round(performance.now() - counter)}ms`);
+        }
         counter = performance.now();
         Logger_default.log(`[${usageId}] Exporting results to ${consumer.outputs.length} output(s)`);
         const exportRes = await OutputExecutor_default.exportResult(consumer, ConsumerManager_default.getExpandedFields(consumer), scope);
@@ -19864,20 +19900,19 @@ var ExecutorOrchestratorClass = class {
     this._getWorkerPath = () => {
       const currentDir = __dirname;
       if (ProcessENVManager_default.getEnvVariable("NODE_ENV") === "dev" || ProcessENVManager_default.getEnvVariable("NODE_ENV") === "development")
-        return import_path19.default.resolve("./.build/workers");
+        return import_path20.default.resolve("./.build/workers");
       const forcedPath = ProcessENVManager_default.getEnvVariable("REMORA_WORKERS_PATH");
       if (forcedPath && forcedPath.length > 0)
-        return import_path19.default.join(__dirname, forcedPath);
+        return import_path20.default.join(__dirname, forcedPath);
       if (!currentDir.includes(".build")) {
-        return import_path19.default.join(__dirname, "../workers");
+        return import_path20.default.join(__dirname, "../workers");
       } else {
-        return import_path19.default.resolve("./.build/workers");
+        return import_path20.default.resolve("./.build/workers");
       }
     };
     this.reconcileExecutorThreadsResults = async (scope, executorResults, tracker) => {
       const mainPath = ExecutorScope_default2.getMainPath(scope);
       ConsumerExecutor_default._ensurePath(mainPath);
-      const writer = new ExecutorWriter_default();
       if (executorResults.length > 1) {
         Logger_default.log(`[${scope.id}] Merging ${executorResults.length} worker output files into ${mainPath}`);
         const perf = performance.now();
@@ -19893,10 +19928,6 @@ var ExecutorOrchestratorClass = class {
         Logger_default.log(`[${scope.id}] Single worker \u2014 renaming output to ${mainPath}`);
         await import_promises9.default.rename(executorResults[0].resultUri, mainPath);
       }
-      if (scope.limitFileSize) {
-        Logger_default.log(`[${scope.id}] Splitting output by size limit (${scope.limitFileSize})`);
-        await writer.splitBySize(scope, mainPath);
-      }
       return mainPath;
     };
     this.performCleanupOperations = async (scope, tracker) => {
@@ -20083,19 +20114,19 @@ var discover = async (producerName) => {
 // src/actions/create_producer.ts
 var import_chalk8 = __toESM(require("chalk"));
 var import_fs13 = __toESM(require("fs"));
-var import_path20 = __toESM(require("path"));
+var import_path21 = __toESM(require("path"));
 var create_producer = async (name) => {
   try {
     if (!import_fs13.default.existsSync("./remora/producers")) {
       throw new Error(import_chalk8.default.red("Missing directory: ") + import_chalk8.default.yellow("./remora/producers"));
     }
     const defaultProducerTemplate = import_fs13.default.readFileSync(
-      import_path20.default.join(DOCUMENTATION_DIR, "default_resources/producer.json"),
+      import_path21.default.join(DOCUMENTATION_DIR, "default_resources/producer.json"),
       "utf-8"
     );
     const defaultProducer = JSON.parse(defaultProducerTemplate);
     defaultProducer.name = name;
-    const producerPath = import_path20.default.join("remora/producers", `${name}.json`);
+    const producerPath = import_path21.default.join("remora/producers", `${name}.json`);
     import_fs13.default.writeFileSync(producerPath, JSON.stringify(defaultProducer, null, 4));
     console.log(import_chalk8.default.green(`\u2705 Created producer config at ${producerPath}`));
     console.log(import_chalk8.default.blue("Remember to:"));
@@ -20112,20 +20143,20 @@ var create_producer = async (name) => {
 // src/actions/create_consumer.ts
 var import_chalk9 = __toESM(require("chalk"));
 var import_fs14 = __toESM(require("fs"));
-var import_path21 = __toESM(require("path"));
+var import_path22 = __toESM(require("path"));
 var create_consumer = async (name, producerName) => {
   try {
     if (!import_fs14.default.existsSync("./remora/consumers")) {
       throw new Error(import_chalk9.default.red("Missing directory: ") + import_chalk9.default.yellow("./remora/consumers"));
     }
     const defaultConsumerTemplate = import_fs14.default.readFileSync(
-      import_path21.default.join(DOCUMENTATION_DIR, "default_resources/consumer.json"),
+      import_path22.default.join(DOCUMENTATION_DIR, "default_resources/consumer.json"),
       "utf-8"
     );
     const defaultConsumer = JSON.parse(defaultConsumerTemplate);
     defaultConsumer.name = name;
     if (producerName) {
-      const producerPath = import_path21.default.join("remora/producers", `${producerName}.json`);
+      const producerPath = import_path22.default.join("remora/producers", `${producerName}.json`);
       if (!import_fs14.default.existsSync(producerPath)) {
         throw new Error(import_chalk9.default.red("Producer not found: ") + import_chalk9.default.yellow(producerPath));
       }
@@ -20146,7 +20177,7 @@ var create_consumer = async (name, producerName) => {
       defaultConsumer.filters = void 0;
       defaultConsumer.metadata = void 0;
     }
-    const consumerPath = import_path21.default.join("remora/consumers", `${name}.json`);
+    const consumerPath = import_path22.default.join("remora/consumers", `${name}.json`);
     import_fs14.default.writeFileSync(consumerPath, JSON.stringify(defaultConsumer, null, 4));
     console.log(import_chalk9.default.green(`\u2705 Created consumer config at ${consumerPath}`));
     if (!producerName) {
@@ -20171,7 +20202,7 @@ var create_consumer = async (name, producerName) => {
 var import_chalk10 = __toESM(require("chalk"));
 var import_ora5 = __toESM(require("ora"));
 var import_fs15 = __toESM(require("fs"));
-var import_path22 = __toESM(require("path"));
+var import_path23 = __toESM(require("path"));
 var automap = async (producerName, schemaNames) => {
   try {
     compile();
@@ -20196,12 +20227,12 @@ var automap = async (producerName, schemaNames) => {
     const sampleStrings = sampleData.map((item) => JSON.stringify(item));
     const mapResult = await AutoMapperEngine_default.map(sampleStrings, schemas, producer.settings.fileKey, [source]);
     for (const producer2 of mapResult.producers) {
-      const producerPath = import_path22.default.join("remora/producers", `${producer2.name}.json`);
+      const producerPath = import_path23.default.join("remora/producers", `${producer2.name}.json`);
       import_fs15.default.writeFileSync(producerPath, JSON.stringify(producer2, null, 4));
       console.log(import_chalk10.default.blue(`Created producer: ${producer2.name}`));
     }
     for (const consumer of mapResult.consumers) {
-      const consumerPath = import_path22.default.join("remora/consumers", `${consumer.name}.json`);
+      const consumerPath = import_path23.default.join("remora/consumers", `${consumer.name}.json`);
       import_fs15.default.writeFileSync(consumerPath, JSON.stringify(consumer, null, 4));
       console.log(import_chalk10.default.blue(`Created consumer: ${consumer.name}`));
     }

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@forzalabs/remora",
-    "version": "1.2.1",
+    "version": "1.2.3",
     "description": "A powerful CLI tool for seamless data translation.",
     "main": "index.js",
     "private": false,

package/workers/ExecutorWorker.js

@@ -9654,15 +9654,15 @@ var require_index_cjs = __commonJS({
       };
     }
     function wrapConversion(toModel, graph) {
-      const path17 = [graph[toModel].parent, toModel];
+      const path18 = [graph[toModel].parent, toModel];
       let fn = convert$1[graph[toModel].parent][toModel];
       let cur = graph[toModel].parent;
       while (graph[cur].parent) {
-        path17.unshift(graph[cur].parent);
+        path18.unshift(graph[cur].parent);
         fn = link(convert$1[graph[cur].parent][cur], fn);
         cur = graph[cur].parent;
       }
-      fn.conversion = path17;
+      fn.conversion = path18;
       return fn;
     }
     function route(fromModel) {
@@ -10388,7 +10388,7 @@ var require_file = __commonJS({
   "../../packages/logger/node_modules/winston/lib/winston/transports/file.js"(exports2, module2) {
     "use strict";
     var fs18 = require("fs");
-    var path17 = require("path");
+    var path18 = require("path");
     var asyncSeries = require_series();
     var zlib2 = require("zlib");
     var { MESSAGE } = require_triple_beam();
@@ -10418,14 +10418,14 @@ var require_file = __commonJS({
         this._onError = this._onError.bind(this);
         if (options.filename || options.dirname) {
           throwIf("filename or dirname", "stream");
-          this._basename = this.filename = options.filename ? path17.basename(options.filename) : "winston.log";
-          this.dirname = options.dirname || path17.dirname(options.filename);
+          this._basename = this.filename = options.filename ? path18.basename(options.filename) : "winston.log";
+          this.dirname = options.dirname || path18.dirname(options.filename);
           this.options = options.options || { flags: "a" };
         } else if (options.stream) {
           console.warn("options.stream will be removed in winston@4. Use winston.transports.Stream");
           throwIf("stream", "filename", "maxsize");
           this._dest = this._stream.pipe(this._setupStream(options.stream));
-          this.dirname = path17.dirname(this._dest.path);
+          this.dirname = path18.dirname(this._dest.path);
         } else {
           throw new Error("Cannot log to file without filename or stream.");
         }
@@ -10588,7 +10588,7 @@ var require_file = __commonJS({
           options = {};
         }
         options = normalizeQuery(options);
-        const file = path17.join(this.dirname, this.filename);
+        const file = path18.join(this.dirname, this.filename);
         let buff = "";
         let results = [];
         let row = 0;
@@ -10693,7 +10693,7 @@ var require_file = __commonJS({
        * TODO: Refactor me.
        */
       stream(options = {}) {
-        const file = path17.join(this.dirname, this.filename);
+        const file = path18.join(this.dirname, this.filename);
         const stream = new Stream();
         const tail = {
           file,
@@ -10743,7 +10743,7 @@ var require_file = __commonJS({
        */
       stat(callback) {
         const target = this._getFile();
-        const fullpath = path17.join(this.dirname, target);
+        const fullpath = path18.join(this.dirname, target);
         fs18.stat(fullpath, (err, stat) => {
           if (err && err.code === "ENOENT") {
             debug("ENOENT\xA0ok", fullpath);
@@ -10847,7 +10847,7 @@ var require_file = __commonJS({
        * @returns {WritableStream} Stream that writes to disk for the active file.
        */
       _createStream(source) {
-        const fullpath = path17.join(this.dirname, this.filename);
+        const fullpath = path18.join(this.dirname, this.filename);
         debug("create stream start", fullpath, this.options);
         const dest = fs18.createWriteStream(fullpath, this.options).on("error", (err) => debug(err)).on("close", () => debug("close", dest.path, dest.bytesWritten)).on("open", () => {
           debug("file open ok", fullpath);
@@ -10872,16 +10872,16 @@ var require_file = __commonJS({
        */
       _incFile(callback) {
         debug("_incFile", this.filename);
-        const ext = path17.extname(this._basename);
-        const basename = path17.basename(this._basename, ext);
+        const ext = path18.extname(this._basename);
+        const basename = path18.basename(this._basename, ext);
         const tasks = [];
         if (this.zippedArchive) {
           tasks.push(
             function(cb) {
               const num = this._created > 0 && !this.tailable ? this._created : "";
               this._compressFile(
-                path17.join(this.dirname, `${basename}${num}${ext}`),
-                path17.join(this.dirname, `${basename}${num}${ext}.gz`),
+                path18.join(this.dirname, `${basename}${num}${ext}`),
+                path18.join(this.dirname, `${basename}${num}${ext}.gz`),
                 cb
               );
             }.bind(this)
@@ -10906,8 +10906,8 @@ var require_file = __commonJS({
        * @private
        */
       _getFile() {
-        const ext = path17.extname(this._basename);
-        const basename = path17.basename(this._basename, ext);
+        const ext = path18.extname(this._basename);
+        const basename = path18.basename(this._basename, ext);
         const isRotation = this.rotationFormat ? this.rotationFormat() : this._created;
         return !this.tailable && this._created ? `${basename}${isRotation}${ext}` : `${basename}${ext}`;
       }
@@ -10927,7 +10927,7 @@ var require_file = __commonJS({
         const isOldest = oldest !== 0 ? oldest : "";
         const isZipped = this.zippedArchive ? ".gz" : "";
         const filePath = `${basename}${isOldest}${ext}${isZipped}`;
-        const target = path17.join(this.dirname, filePath);
+        const target = path18.join(this.dirname, filePath);
         fs18.unlink(target, callback);
       }
       /**
@@ -10950,20 +10950,20 @@ var require_file = __commonJS({
         for (let x = this.maxFiles - 1; x > 1; x--) {
           tasks.push(function(i, cb) {
             let fileName = `${basename}${i - 1}${ext}${isZipped}`;
-            const tmppath = path17.join(this.dirname, fileName);
+            const tmppath = path18.join(this.dirname, fileName);
             fs18.exists(tmppath, (exists) => {
               if (!exists) {
                 return cb(null);
               }
               fileName = `${basename}${i}${ext}${isZipped}`;
-              fs18.rename(tmppath, path17.join(this.dirname, fileName), cb);
+              fs18.rename(tmppath, path18.join(this.dirname, fileName), cb);
             });
           }.bind(this, x));
         }
         asyncSeries(tasks, () => {
           fs18.rename(
-            path17.join(this.dirname, `${basename}${ext}${isZipped}`),
-            path17.join(this.dirname, `${basename}1${ext}${isZipped}`),
+            path18.join(this.dirname, `${basename}${ext}${isZipped}`),
+            path18.join(this.dirname, `${basename}1${ext}${isZipped}`),
             callback
           );
         });
@@ -11076,9 +11076,9 @@ var require_http = __commonJS({
         };
         const auth = options.params.auth || null;
         delete options.params.auth;
-        const path17 = options.params.path || null;
+        const path18 = options.params.path || null;
         delete options.params.path;
-        this._request(options, auth, path17, (err, res, body) => {
+        this._request(options, auth, path18, (err, res, body) => {
           if (res && res.statusCode !== 200) {
             err = new Error(`Invalid HTTP Status Code: ${res.statusCode}`);
           }
@@ -11106,12 +11106,12 @@ var require_http = __commonJS({
           method: "stream",
           params: options
         };
-        const path17 = options.params.path || null;
+        const path18 = options.params.path || null;
         delete options.params.path;
         const auth = options.params.auth || null;
         delete options.params.auth;
         let buff = "";
-        const req = this._request(options, auth, path17);
+        const req = this._request(options, auth, path18);
         stream.destroy = () => req.destroy();
         req.on("data", (data) => {
           data = (buff + data).split(/\n+/);
@@ -11137,14 +11137,14 @@ var require_http = __commonJS({
        * @param {string} path - request path
        * @param {function} callback - Continuation to respond to when complete.
        */
-      _request(options, auth, path17, callback) {
+      _request(options, auth, path18, callback) {
         options = options || {};
         auth = auth || this.auth;
-        path17 = path17 || this.path || "";
+        path18 = path18 || this.path || "";
         if (this.batch) {
-          this._doBatch(options, callback, auth, path17);
+          this._doBatch(options, callback, auth, path18);
         } else {
-          this._doRequest(options, callback, auth, path17);
+          this._doRequest(options, callback, auth, path18);
         }
       }
       /**
@@ -11154,18 +11154,18 @@ var require_http = __commonJS({
        * @param {Object?} auth - authentication options
        * @param {string} path - request path
        */
-      _doBatch(options, callback, auth, path17) {
+      _doBatch(options, callback, auth, path18) {
         this.batchOptions.push(options);
         if (this.batchOptions.length === 1) {
           const me = this;
           this.batchCallback = callback;
           this.batchTimeoutID = setTimeout(function() {
             me.batchTimeoutID = -1;
-            me._doBatchRequest(me.batchCallback, auth, path17);
+            me._doBatchRequest(me.batchCallback, auth, path18);
           }, this.batchInterval);
         }
         if (this.batchOptions.length === this.batchCount) {
-          this._doBatchRequest(this.batchCallback, auth, path17);
+          this._doBatchRequest(this.batchCallback, auth, path18);
         }
       }
       /**
@@ -11174,14 +11174,14 @@ var require_http = __commonJS({
        * @param {Object?} auth - authentication options
        * @param {string} path - request path
        */
-      _doBatchRequest(callback, auth, path17) {
+      _doBatchRequest(callback, auth, path18) {
         if (this.batchTimeoutID > 0) {
           clearTimeout(this.batchTimeoutID);
           this.batchTimeoutID = -1;
         }
         const batchOptionsCopy = this.batchOptions.slice();
         this.batchOptions = [];
-        this._doRequest(batchOptionsCopy, callback, auth, path17);
+        this._doRequest(batchOptionsCopy, callback, auth, path18);
       }
       /**
        * Make a request to a winstond server or any http server which can
@@ -11191,7 +11191,7 @@ var require_http = __commonJS({
        * @param {Object?} auth - authentication options
        * @param {string} path - request path
        */
-      _doRequest(options, callback, auth, path17) {
+      _doRequest(options, callback, auth, path18) {
         const headers = Object.assign({}, this.headers);
         if (auth && auth.bearer) {
           headers.Authorization = `Bearer ${auth.bearer}`;
@@ -11201,7 +11201,7 @@ var require_http = __commonJS({
           method: "POST",
           host: this.host,
           port: this.port,
-          path: `/${path17.replace(/^\//, "")}`,
+          path: `/${path18.replace(/^\//, "")}`,
           headers,
           auth: auth && auth.username && auth.password ? `${auth.username}:${auth.password}` : "",
           agent: this.agent
@@ -14221,7 +14221,7 @@ var Environment = new EnvironmentClass();
 var Environment_default = Environment;
 
 // ../../packages/executors/src/ConsumerExecutor.ts
-var import_path14 = __toESM(require("path"));
+var import_path15 = __toESM(require("path"));
 var import_fs9 = __toESM(require("fs"));
 var import_readline6 = __toESM(require("readline"));
 var import_promises8 = __toESM(require("fs/promises"));
@@ -14606,6 +14606,10 @@ var AutoMapperEngineClass = class {
 };
 var AutoMapperEngine = new AutoMapperEngineClass();
 
+// ../../packages/engines/src/producer/ProducerEngine.ts
+var import_path11 = __toESM(require("path"), 1);
+var import_crypto2 = require("crypto");
+
 // ../../packages/drivers/src/DeltaShareDriver.ts
 var DeltaShareSourceDriver = class {
   constructor() {
@@ -15440,8 +15444,8 @@ var HttpClientClass = class {
         throw new Error(err.message ?? "An error occurred on GET.");
       }
     };
-    this.getWorkerUrl = (path17) => {
-      const cleanPath = path17.startsWith("/") ? path17.substring(1) : path17;
+    this.getWorkerUrl = (path18) => {
+      const cleanPath = path18.startsWith("/") ? path18.substring(1) : path18;
       return `${this.workerHost}/${cleanPath}`;
     };
     this.workerHost = ProcessENVManager_default.getEnvVariable("REMORA_WORKER_HOST") || "http://worker:5069";
@@ -16511,12 +16515,12 @@ var noopTracker = { _operations: {}, measure: () => {
 }, getOperations: () => ({}) };
 var ProducerEngineClass = class {
   constructor() {
-    this.readFile = async (producer, options) => {
+    this.readFile = async (producer, options, sourceOverride) => {
       Affirm_default(producer, "Invalid producer");
       Affirm_default(options, "Invalid options");
       if (options.readmode === "lines")
         Affirm_default(options.lines, "Invalid lines");
-      const source = Environment_default.getSource(producer.source);
+      const source = sourceOverride ?? Environment_default.getSource(producer.source);
       Affirm_default(source, `No source found for producer "${producer.name}" with name "${producer.source}"`);
       const driver = await DriverFactory_default.instantiateSource(source);
       Affirm_default(driver, `No driver found for producer "${producer.name}" with driver type "${source.engine}"`);
@@ -16590,8 +16594,32 @@ var ProducerEngineClass = class {
         case "local":
         case "aws-s3":
         case "delta-share": {
-          const fileData = await this.readFile(producer, { readmode: "lines", lines: { from: 0, to: sampleSize } });
-          rawData = fileData.data;
+          const { compressionType } = producer.settings;
+          let scope = null;
+          try {
+            let effectiveProducer = producer;
+            let effectiveSource = source;
+            if (compressionType) {
+              scope = { id: (0, import_crypto2.randomUUID)(), folder: `sample-${(0, import_crypto2.randomUUID)()}`, workersId: [] };
+              const driver = await DriverFactory_default.instantiateSource(source);
+              const readyResult = await driver.ready({ producer, scope });
+              const firstFile = readyResult.files[0].fullUri;
+              const decompressedDir = import_path11.default.dirname(firstFile);
+              effectiveSource = { ...source, engine: "local", authentication: { ...source.authentication, path: decompressedDir } };
+              effectiveProducer = {
+                ...producer,
+                settings: {
+                  ...producer.settings,
+                  fileKey: readyResult.files.length > 1 ? "%.dataset" : import_path11.default.basename(firstFile),
+                  compressionType: void 0
+                }
+              };
+            }
+            const fileData = await this.readFile(effectiveProducer, { readmode: "lines", lines: { from: 0, to: sampleSize } }, effectiveSource);
+            rawData = fileData.data;
+          } finally {
+            if (scope) await ExecutorScope_default.clearScope(scope);
+          }
           break;
         }
         default:
@@ -16682,7 +16710,7 @@ var ProducerEngine = new ProducerEngineClass();
 var ProducerEngine_default = ProducerEngine;
 
 // ../../packages/engines/src/ai/DeveloperEngine.ts
-var import_path11 = __toESM(require("path"), 1);
+var import_path12 = __toESM(require("path"), 1);
 var import_promises6 = __toESM(require("fs/promises"), 1);
 var import_dayjs2 = __toESM(require("dayjs"), 1);
 var import_customParseFormat2 = __toESM(require("dayjs/plugin/customParseFormat"), 1);
@@ -16709,7 +16737,7 @@ var DeveloperEngineClass = class {
         _version: producer._version ?? 1
       };
       mappedProducer["$schema"] = producer["$schema"];
-      const producerPath = import_path11.default.join(process.cwd(), "remora", "producers", `${producer.name}.json`);
+      const producerPath = import_path12.default.join(process.cwd(), "remora", "producers", `${producer.name}.json`);
       await import_promises6.default.writeFile(producerPath, JSON.stringify(mappedProducer, null, 4), "utf-8");
       return { producer: mappedProducer, fields: typeDefinitions };
     };
@@ -16968,8 +16996,8 @@ var DeveloperEngineClass = class {
       Affirm_default(fileType, "Producer must have a fileType setting for mock data generation");
       const mockRecords = this.generateMockRecords(producer.dimensions, records);
       const basePath = source.authentication.path || process.cwd();
-      const filePath = import_path11.default.join(basePath, fileKey.replace("%", "mock"));
-      await import_promises6.default.mkdir(import_path11.default.dirname(filePath), { recursive: true });
+      const filePath = import_path12.default.join(basePath, fileKey.replace("%", "mock"));
+      await import_promises6.default.mkdir(import_path12.default.dirname(filePath), { recursive: true });
       const content = this.formatMockData(mockRecords, fileType, delimiter);
       await import_promises6.default.writeFile(filePath, content, "utf-8");
       return { filePath, recordCount: records };
@@ -18422,7 +18450,7 @@ var UsageManager_default = UsageManager;
 var fs13 = __toESM(require("fs"));
 
 // ../../packages/executors/src/ExecutorScope.ts
-var import_path12 = __toESM(require("path"));
+var import_path13 = __toESM(require("path"));
 var import_fs8 = __toESM(require("fs"));
 var import_promises7 = __toESM(require("fs/promises"));
 var ExecutorScopeClass2 = class {
@@ -18430,7 +18458,7 @@ var ExecutorScopeClass2 = class {
     this.WORKERS_FOLDER = "workers";
     this.PRODUCERS_FOLDER = "producers";
     this.getWorkerPath = (scope, workerId) => {
-      return import_path12.default.join(
+      return import_path13.default.join(
         Constants_default.defaults.REMORA_PATH,
         Constants_default.defaults.PRODUCER_TEMP_FOLDER,
         // A specific execution sits entirely in this folder, so at the end it's safe to delete it entirely
@@ -18440,7 +18468,7 @@ var ExecutorScopeClass2 = class {
       );
     };
     this.getProducerPath = (scope, producer, sourceFileKey) => {
-      return import_path12.default.join(
+      return import_path13.default.join(
         Constants_default.defaults.REMORA_PATH,
         Constants_default.defaults.PRODUCER_TEMP_FOLDER,
         // A specific execution sits entirely in this folder, so at the end it's safe to delete it entirely
@@ -18451,7 +18479,7 @@ var ExecutorScopeClass2 = class {
       );
     };
     this.getMainPath = (scope) => {
-      return import_path12.default.join(
+      return import_path13.default.join(
         Constants_default.defaults.REMORA_PATH,
         Constants_default.defaults.PRODUCER_TEMP_FOLDER,
         scope.folder,
@@ -18459,7 +18487,7 @@ var ExecutorScopeClass2 = class {
       );
     };
     this.clearScope = async (scope) => {
-      const scopePath = import_path12.default.join(
+      const scopePath = import_path13.default.join(
         Constants_default.defaults.REMORA_PATH,
         Constants_default.defaults.PRODUCER_TEMP_FOLDER,
         scope.folder
@@ -18469,7 +18497,7 @@ var ExecutorScopeClass2 = class {
       }
     };
     this.ensurePath = (fileUri) => {
-      const dir = import_path12.default.dirname(fileUri);
+      const dir = import_path13.default.dirname(fileUri);
       if (!import_fs8.default.existsSync(dir))
         import_fs8.default.mkdirSync(dir, { recursive: true });
       if (!import_fs8.default.existsSync(fileUri))
@@ -18481,7 +18509,7 @@ var ExecutorScope2 = new ExecutorScopeClass2();
 var ExecutorScope_default2 = ExecutorScope2;
 
 // ../../packages/executors/src/OutputExecutor.ts
-var import_path13 = __toESM(require("path"));
+var import_path14 = __toESM(require("path"));
 var OutputExecutorClass = class {
   constructor() {
     this._getInternalRecordFormat = (consumer) => {
@@ -18527,13 +18555,13 @@ var OutputExecutorClass = class {
       for (const output of consumer.outputs) {
         const destination = Environment_default.getSource(output.exportDestination);
         const driver = await DriverFactory_default.instantiateDestination(destination);
-        const currentPath = import_path13.default.dirname(ExecutorScope_default2.getMainPath(scope));
+        const currentPath = import_path14.default.dirname(ExecutorScope_default2.getMainPath(scope));
         const destinationName = this._composeFileName(consumer, output, this._getExtension(output));
         Logger_default.log(`Exporting consumer "${consumer.name}" to "${output.exportDestination}" as ${output.format} (${destinationName})`);
         const filenameArray = fs13.readdirSync(currentPath).filter((filename) => filename.includes(".dataset"));
         for (const filename in filenameArray) {
           const destinationPath = this.getCompletedPath(destinationName, filename);
-          const startingPath = import_path13.default.join(currentPath, filenameArray[filename]);
+          const startingPath = import_path14.default.join(currentPath, filenameArray[filename]);
           if (output.format === internalFormat) {
             results.push(await driver.move(startingPath, destinationPath));
           } else {
@@ -18593,8 +18621,8 @@ var OutputExecutor_default = OutputExecutor;
 var ConsumerExecutorClass = class {
   constructor() {
     this._getWorkPath = (consumer, executionId) => {
-      const execFolder = import_path14.default.join(consumer.name, executionId);
-      const workPath = import_path14.default.join("./remora", Constants_default.defaults.PRODUCER_TEMP_FOLDER, execFolder, ".dataset");
+      const execFolder = import_path15.default.join(consumer.name, executionId);
+      const workPath = import_path15.default.join("./remora", Constants_default.defaults.PRODUCER_TEMP_FOLDER, execFolder, ".dataset");
       return workPath;
     };
     this._clearWorkPath = async (workPath) => {
@@ -18605,7 +18633,7 @@ var ConsumerExecutorClass = class {
       } catch (error) {
       }
       try {
-        const dir = import_path14.default.dirname(workPath);
+        const dir = import_path15.default.dirname(workPath);
         if (import_fs9.default.existsSync(dir)) {
           await import_promises8.default.rmdir(dir);
         }
@@ -18613,7 +18641,7 @@ var ConsumerExecutorClass = class {
       }
     };
     this._ensurePath = (pathUri) => {
-      const dir = import_path14.default.dirname(pathUri);
+      const dir = import_path15.default.dirname(pathUri);
       if (!import_fs9.default.existsSync(dir))
         import_fs9.default.mkdirSync(dir, { recursive: true });
       if (!import_fs9.default.existsSync(pathUri))
@@ -19013,7 +19041,7 @@ var import_fs10 = __toESM(require("fs"));
 var import_readline7 = __toESM(require("readline"));
 
 // ../../packages/executors/src/ProducerExecutor.ts
-var import_path15 = __toESM(require("path"));
+var import_path16 = __toESM(require("path"));
 var ProducerExecutorClass = class {
   constructor() {
     this.ready = async (producer, scope) => {
@@ -19037,7 +19065,7 @@ var ProducerExecutorClass = class {
       counter = performance.now();
       for (const dimension of dimensions) {
         if (dimension.prodDimension.sourceFilename === true)
-          record[dimension.name] = import_path15.default.basename(chunk.fileUri);
+          record[dimension.name] = import_path16.default.basename(chunk.fileUri);
         const maskType = ProducerManager_default.getMask(dimension.prodDimension);
         if (Algo_default.hasVal(maskType))
           record[dimension.name] = CryptoEngine_default.hashValue(maskType, record[dimension.name]?.toString(), dimension.prodDimension.type);
@@ -19227,7 +19255,7 @@ var Executor_default = Executor;
 var import_os = __toESM(require("os"));
 var import_fs11 = __toESM(require("fs"));
 var import_promises9 = __toESM(require("fs/promises"));
-var import_path16 = __toESM(require("path"));
+var import_path17 = __toESM(require("path"));
 var import_workerpool = __toESM(require("workerpool"));
 
 // ../../packages/executors/src/ExecutorProgress.ts
@@ -19395,7 +19423,7 @@ var ExecutorOrchestratorClass = class {
       };
       const workerPath = this._getWorkerPath();
       Logger_default.log(`Initializing worker pool from ${workerPath} (heap limit: ${Constants_default.defaults.MIN_RUNTIME_HEAP_MB}MB)`);
-      return import_workerpool.default.pool(import_path16.default.join(workerPath, "ExecutorWorker.js"), options);
+      return import_workerpool.default.pool(import_path17.default.join(workerPath, "ExecutorWorker.js"), options);
     };
     this.launch = async (request) => {
       Affirm_default(request, "Invalid options");
@@ -19516,6 +19544,14 @@ var ExecutorOrchestratorClass = class {
           }
           Logger_default.log(`[${usageId}] Dataset validations complete in ${Math.round(performance.now() - counter)}ms`);
         }
+        if (scope.limitFileSize) {
+          Logger_default.log(`[${usageId}] Splitting output by size limit (${scope.limitFileSize})`);
+          counter = performance.now();
+          const writer = new ExecutorWriter_default();
+          await writer.splitBySize(scope, ExecutorScope_default2.getMainPath(scope));
+          tracker.measure("split-by-size", performance.now() - counter);
+          Logger_default.log(`[${usageId}] Split complete in ${Math.round(performance.now() - counter)}ms`);
+        }
         counter = performance.now();
         Logger_default.log(`[${usageId}] Exporting results to ${consumer.outputs.length} output(s)`);
         const exportRes = await OutputExecutor_default.exportResult(consumer, ConsumerManager_default.getExpandedFields(consumer), scope);
@@ -19623,20 +19659,19 @@ var ExecutorOrchestratorClass = class {
     this._getWorkerPath = () => {
       const currentDir = __dirname;
       if (ProcessENVManager_default.getEnvVariable("NODE_ENV") === "dev" || ProcessENVManager_default.getEnvVariable("NODE_ENV") === "development")
-        return import_path16.default.resolve("./.build/workers");
+        return import_path17.default.resolve("./.build/workers");
       const forcedPath = ProcessENVManager_default.getEnvVariable("REMORA_WORKERS_PATH");
       if (forcedPath && forcedPath.length > 0)
-        return import_path16.default.join(__dirname, forcedPath);
+        return import_path17.default.join(__dirname, forcedPath);
       if (!currentDir.includes(".build")) {
-        return import_path16.default.join(__dirname, "../workers");
+        return import_path17.default.join(__dirname, "../workers");
       } else {
-        return import_path16.default.resolve("./.build/workers");
+        return import_path17.default.resolve("./.build/workers");
       }
     };
     this.reconcileExecutorThreadsResults = async (scope, executorResults, tracker) => {
       const mainPath = ExecutorScope_default2.getMainPath(scope);
       ConsumerExecutor_default._ensurePath(mainPath);
-      const writer = new ExecutorWriter_default();
       if (executorResults.length > 1) {
         Logger_default.log(`[${scope.id}] Merging ${executorResults.length} worker output files into ${mainPath}`);
         const perf = performance.now();
@@ -19652,10 +19687,6 @@ var ExecutorOrchestratorClass = class {
         Logger_default.log(`[${scope.id}] Single worker \u2014 renaming output to ${mainPath}`);
         await import_promises9.default.rename(executorResults[0].resultUri, mainPath);
       }
-      if (scope.limitFileSize) {
-        Logger_default.log(`[${scope.id}] Splitting output by size limit (${scope.limitFileSize})`);
-        await writer.splitBySize(scope, mainPath);
-      }
       return mainPath;
     };
     this.performCleanupOperations = async (scope, tracker) => {