@forzalabs/remora 1.2.2 → 1.2.3

package/index.js

@@ -147,7 +147,7 @@ var require_has_flag = __commonJS({
 var require_supports_colors = __commonJS({
   "../../node_modules/logform/node_modules/@colors/colors/lib/system/supports-colors.js"(exports2, module2) {
     "use strict";
-    var os3 = require("os");
+    var os2 = require("os");
     var hasFlag = require_has_flag();
     var env = process.env;
     var forceColor = void 0;
@@ -185,7 +185,7 @@ var require_supports_colors = __commonJS({
       }
       var min = forceColor ? 1 : 0;
       if (process.platform === "win32") {
-        var osRelease = os3.release().split(".");
+        var osRelease = os2.release().split(".");
         if (Number(process.versions.node.split(".")[0]) >= 8 && Number(osRelease[0]) >= 10 && Number(osRelease[2]) >= 10586) {
           return Number(osRelease[2]) >= 14931 ? 3 : 2;
         }
@@ -5429,7 +5429,7 @@ var require_winston_transport = __commonJS({
 var require_console = __commonJS({
   "../../packages/logger/node_modules/winston/lib/winston/transports/console.js"(exports2, module2) {
     "use strict";
-    var os3 = require("os");
+    var os2 = require("os");
     var { LEVEL, MESSAGE } = require_triple_beam();
     var TransportStream = require_winston_transport();
     module2.exports = class Console extends TransportStream {
@@ -5443,7 +5443,7 @@ var require_console = __commonJS({
         this.name = options.name || "console";
         this.stderrLevels = this._stringArrayToSet(options.stderrLevels);
         this.consoleWarnLevels = this._stringArrayToSet(options.consoleWarnLevels);
-        this.eol = typeof options.eol === "string" ? options.eol : os3.EOL;
+        this.eol = typeof options.eol === "string" ? options.eol : os2.EOL;
         this.forceConsole = options.forceConsole || false;
         this._consoleLog = console.log.bind(console);
         this._consoleWarn = console.warn.bind(console);
@@ -10293,7 +10293,7 @@ var require_node2 = __commonJS({
 var require_tail_file = __commonJS({
   "../../packages/logger/node_modules/winston/lib/winston/tail-file.js"(exports2, module2) {
     "use strict";
-    var fs25 = require("fs");
+    var fs24 = require("fs");
     var { StringDecoder } = require("string_decoder");
     var { Stream } = require_readable();
     function noop() {
@@ -10314,7 +10314,7 @@ var require_tail_file = __commonJS({
         stream.emit("end");
         stream.emit("close");
       };
-      fs25.open(options.file, "a+", "0644", (err, fd) => {
+      fs24.open(options.file, "a+", "0644", (err, fd) => {
         if (err) {
           if (!iter) {
             stream.emit("error", err);
@@ -10326,10 +10326,10 @@ var require_tail_file = __commonJS({
         }
         (function read() {
           if (stream.destroyed) {
-            fs25.close(fd, noop);
+            fs24.close(fd, noop);
             return;
           }
-          return fs25.read(fd, buffer, 0, buffer.length, pos, (error, bytes) => {
+          return fs24.read(fd, buffer, 0, buffer.length, pos, (error, bytes) => {
             if (error) {
               if (!iter) {
                 stream.emit("error", error);
@@ -10388,7 +10388,7 @@ var require_tail_file = __commonJS({
 var require_file = __commonJS({
   "../../packages/logger/node_modules/winston/lib/winston/transports/file.js"(exports2, module2) {
     "use strict";
-    var fs25 = require("fs");
+    var fs24 = require("fs");
     var path24 = require("path");
     var asyncSeries = require_series();
     var zlib2 = require("zlib");
@@ -10396,7 +10396,7 @@ var require_file = __commonJS({
     var { Stream, PassThrough } = require_readable();
     var TransportStream = require_winston_transport();
     var debug2 = require_node2()("winston:file");
-    var os3 = require("os");
+    var os2 = require("os");
     var tailFile = require_tail_file();
     module2.exports = class File extends TransportStream {
       /**
@@ -10434,7 +10434,7 @@ var require_file = __commonJS({
         this.rotationFormat = options.rotationFormat || false;
         this.zippedArchive = options.zippedArchive || false;
         this.maxFiles = options.maxFiles || null;
-        this.eol = typeof options.eol === "string" ? options.eol : os3.EOL;
+        this.eol = typeof options.eol === "string" ? options.eol : os2.EOL;
         this.tailable = options.tailable || false;
         this.lazy = options.lazy || false;
         this._size = 0;
@@ -10593,7 +10593,7 @@ var require_file = __commonJS({
         let buff = "";
         let results = [];
         let row = 0;
-        const stream = fs25.createReadStream(file, {
+        const stream = fs24.createReadStream(file, {
           encoding: "utf8"
         });
         stream.on("error", (err) => {
@@ -10745,7 +10745,7 @@ var require_file = __commonJS({
       stat(callback) {
         const target = this._getFile();
         const fullpath = path24.join(this.dirname, target);
-        fs25.stat(fullpath, (err, stat) => {
+        fs24.stat(fullpath, (err, stat) => {
           if (err && err.code === "ENOENT") {
             debug2("ENOENT\xA0ok", fullpath);
             this.filename = target;
@@ -10850,7 +10850,7 @@ var require_file = __commonJS({
       _createStream(source) {
         const fullpath = path24.join(this.dirname, this.filename);
         debug2("create stream start", fullpath, this.options);
-        const dest = fs25.createWriteStream(fullpath, this.options).on("error", (err) => debug2(err)).on("close", () => debug2("close", dest.path, dest.bytesWritten)).on("open", () => {
+        const dest = fs24.createWriteStream(fullpath, this.options).on("error", (err) => debug2(err)).on("close", () => debug2("close", dest.path, dest.bytesWritten)).on("open", () => {
           debug2("file open ok", fullpath);
           this.emit("open", fullpath);
           source.pipe(dest);
@@ -10929,7 +10929,7 @@ var require_file = __commonJS({
         const isZipped = this.zippedArchive ? ".gz" : "";
         const filePath = `${basename}${isOldest}${ext}${isZipped}`;
         const target = path24.join(this.dirname, filePath);
-        fs25.unlink(target, callback);
+        fs24.unlink(target, callback);
       }
       /**
        * Roll files forward based on integer, up to maxFiles. e.g. if base if
@@ -10952,17 +10952,17 @@ var require_file = __commonJS({
           tasks.push(function(i, cb) {
             let fileName = `${basename}${i - 1}${ext}${isZipped}`;
             const tmppath = path24.join(this.dirname, fileName);
-            fs25.exists(tmppath, (exists) => {
+            fs24.exists(tmppath, (exists) => {
               if (!exists) {
                 return cb(null);
               }
               fileName = `${basename}${i}${ext}${isZipped}`;
-              fs25.rename(tmppath, path24.join(this.dirname, fileName), cb);
+              fs24.rename(tmppath, path24.join(this.dirname, fileName), cb);
             });
           }.bind(this, x));
         }
         asyncSeries(tasks, () => {
-          fs25.rename(
+          fs24.rename(
             path24.join(this.dirname, `${basename}${ext}${isZipped}`),
             path24.join(this.dirname, `${basename}1${ext}${isZipped}`),
             callback
@@ -10978,22 +10978,22 @@ var require_file = __commonJS({
        * @private
        */
       _compressFile(src, dest, callback) {
-        fs25.access(src, fs25.F_OK, (err) => {
+        fs24.access(src, fs24.F_OK, (err) => {
           if (err) {
             return callback();
           }
           var gzip = zlib2.createGzip();
-          var inp = fs25.createReadStream(src);
-          var out = fs25.createWriteStream(dest);
+          var inp = fs24.createReadStream(src);
+          var out = fs24.createWriteStream(dest);
           out.on("finish", () => {
-            fs25.unlink(src, callback);
+            fs24.unlink(src, callback);
           });
           inp.pipe(gzip).pipe(out);
         });
       }
       _createLogDirIfNotExist(dirPath) {
-        if (!fs25.existsSync(dirPath)) {
-          fs25.mkdirSync(dirPath, { recursive: true });
+        if (!fs24.existsSync(dirPath)) {
+          fs24.mkdirSync(dirPath, { recursive: true });
         }
       }
     };
@@ -11237,7 +11237,7 @@ var require_stream3 = __commonJS({
     "use strict";
     var isStream = require_is_stream();
     var { MESSAGE } = require_triple_beam();
-    var os3 = require("os");
+    var os2 = require("os");
     var TransportStream = require_winston_transport();
     module2.exports = class Stream extends TransportStream {
       /**
@@ -11253,7 +11253,7 @@ var require_stream3 = __commonJS({
         this._stream = options.stream;
         this._stream.setMaxListeners(Infinity);
         this.isObjectMode = options.stream._writableState.objectMode;
-        this.eol = typeof options.eol === "string" ? options.eol : os3.EOL;
+        this.eol = typeof options.eol === "string" ? options.eol : os2.EOL;
       }
       /**
        * Core logging method exposed to Winston.
@@ -11636,7 +11636,7 @@ var require_exception_stream = __commonJS({
 var require_exception_handler = __commonJS({
   "../../packages/logger/node_modules/winston/lib/winston/exception-handler.js"(exports2, module2) {
     "use strict";
-    var os3 = require("os");
+    var os2 = require("os");
     var asyncForEach = require_forEach();
     var debug2 = require_node2()("winston:exception");
     var once = require_one_time();
@@ -11731,8 +11731,8 @@ var require_exception_handler = __commonJS({
        */
       getOsInfo() {
         return {
-          loadavg: os3.loadavg(),
-          uptime: os3.uptime()
+          loadavg: os2.loadavg(),
+          uptime: os2.uptime()
         };
       }
       /**
@@ -11874,7 +11874,7 @@ var require_rejection_stream = __commonJS({
 var require_rejection_handler = __commonJS({
   "../../packages/logger/node_modules/winston/lib/winston/rejection-handler.js"(exports2, module2) {
     "use strict";
-    var os3 = require("os");
+    var os2 = require("os");
     var asyncForEach = require_forEach();
     var debug2 = require_node2()("winston:rejection");
     var once = require_one_time();
@@ -11971,8 +11971,8 @@ var require_rejection_handler = __commonJS({
        */
       getOsInfo() {
         return {
-          loadavg: os3.loadavg(),
-          uptime: os3.uptime()
+          loadavg: os2.loadavg(),
+          uptime: os2.uptime()
         };
       }
       /**
@@ -17063,9 +17063,8 @@ var AutoMapperEngine = new AutoMapperEngineClass();
 var AutoMapperEngine_default = AutoMapperEngine;
 
 // ../../packages/engines/src/producer/ProducerEngine.ts
-var fs15 = __toESM(require("fs"), 1);
-var os = __toESM(require("os"), 1);
 var import_path14 = __toESM(require("path"), 1);
+var import_crypto3 = require("crypto");
 
 // ../../packages/engines/src/transform/TypeCaster.ts
 var import_dayjs = __toESM(require("dayjs"), 1);
@@ -17266,20 +17265,30 @@ var ProducerEngineClass = class {
         case "aws-s3":
         case "delta-share": {
           const { compressionType } = producer.settings;
-          let tempDir = null;
+          let scope = null;
           try {
             let effectiveProducer = producer;
             let effectiveSource = source;
             if (compressionType) {
-              tempDir = fs15.mkdtempSync(import_path14.default.join(os.tmpdir(), "remora-sample-"));
-              const decompressedFileKey = await this._decompressForSampling(producer, source, tempDir);
-              effectiveProducer = { ...producer, settings: { ...producer.settings, fileKey: decompressedFileKey, compressionType: void 0 } };
-              effectiveSource = { ...source, authentication: { ...source.authentication, path: tempDir } };
+              scope = { id: (0, import_crypto3.randomUUID)(), folder: `sample-${(0, import_crypto3.randomUUID)()}`, workersId: [] };
+              const driver = await DriverFactory_default.instantiateSource(source);
+              const readyResult = await driver.ready({ producer, scope });
+              const firstFile = readyResult.files[0].fullUri;
+              const decompressedDir = import_path14.default.dirname(firstFile);
+              effectiveSource = { ...source, engine: "local", authentication: { ...source.authentication, path: decompressedDir } };
+              effectiveProducer = {
+                ...producer,
+                settings: {
+                  ...producer.settings,
+                  fileKey: readyResult.files.length > 1 ? "%.dataset" : import_path14.default.basename(firstFile),
+                  compressionType: void 0
+                }
+              };
             }
             const fileData = await this.readFile(effectiveProducer, { readmode: "lines", lines: { from: 0, to: sampleSize } }, effectiveSource);
             rawData = fileData.data;
           } finally {
-            if (tempDir) fs15.rmSync(tempDir, { recursive: true, force: true });
+            if (scope) await ExecutorScope_default.clearScope(scope);
           }
           break;
         }
@@ -17365,47 +17374,6 @@ var ProducerEngineClass = class {
       }
       return dimensions;
     };
-    this._decompressForSampling = async (producer, source, tempDir) => {
-      const { fileKey, compressionType } = producer.settings;
-      const sourcePath = source.authentication["path"];
-      switch (compressionType?.toUpperCase()) {
-        case "GZ": {
-          const decompressedName = fileKey.replace(/\.gz$/i, "");
-          const outputPath = import_path14.default.join(tempDir, decompressedName);
-          const outputDir = import_path14.default.dirname(outputPath);
-          if (!fs15.existsSync(outputDir)) fs15.mkdirSync(outputDir, { recursive: true });
-          await ParseCompression_default.decompressToFile(compressionType, fileKey, sourcePath, outputPath);
-          return decompressedName;
-        }
-        case "ZIP":
-        case "TAR": {
-          const extractDir = import_path14.default.join(tempDir, "extracted");
-          await ParseCompression_default.decompressToFile(compressionType, fileKey, sourcePath, extractDir);
-          const files = this._findFilesRecursive(extractDir);
-          Affirm_default(files.length > 0, `No files found after decompressing "${fileKey}"`);
-          const firstFile = files[0];
-          const relativePath = import_path14.default.relative(extractDir, firstFile);
-          const destPath = import_path14.default.join(tempDir, relativePath);
-          const destDir = import_path14.default.dirname(destPath);
-          if (!fs15.existsSync(destDir)) fs15.mkdirSync(destDir, { recursive: true });
-          fs15.renameSync(firstFile, destPath);
-          return relativePath;
-        }
-        default:
-          throw new Error(`Unsupported compression type "${compressionType}" for producer "${producer.name}"`);
-      }
-    };
-    this._findFilesRecursive = (dir) => {
-      const results = [];
-      for (const entry of fs15.readdirSync(dir)) {
-        const fullPath = import_path14.default.join(dir, entry);
-        if (fs15.statSync(fullPath).isDirectory())
-          results.push(...this._findFilesRecursive(fullPath));
-        else
-          results.push(fullPath);
-      }
-      return results;
-    };
   }
 };
 var ProducerEngine = new ProducerEngineClass();
@@ -18880,7 +18848,7 @@ var UsageManager = new UsageManagerClass();
 var UsageManager_default = UsageManager;
 
 // ../../packages/executors/src/OutputExecutor.ts
-var fs18 = __toESM(require("fs"));
+var fs17 = __toESM(require("fs"));
 
 // ../../packages/executors/src/ExecutorScope.ts
 var import_path16 = __toESM(require("path"));
@@ -18991,7 +18959,7 @@ var OutputExecutorClass = class {
         const currentPath = import_path17.default.dirname(ExecutorScope_default2.getMainPath(scope));
         const destinationName = this._composeFileName(consumer, output, this._getExtension(output));
         Logger_default.log(`Exporting consumer "${consumer.name}" to "${output.exportDestination}" as ${output.format} (${destinationName})`);
-        const filenameArray = fs18.readdirSync(currentPath).filter((filename) => filename.includes(".dataset"));
+        const filenameArray = fs17.readdirSync(currentPath).filter((filename) => filename.includes(".dataset"));
         for (const filename in filenameArray) {
           const destinationPath = this.getCompletedPath(destinationName, filename);
           const startingPath = import_path17.default.join(currentPath, filenameArray[filename]);
@@ -19586,7 +19554,7 @@ var ExecutorProgress = class {
 var ExecutorProgress_default = ExecutorProgress;
 
 // ../../packages/executors/src/ExecutorWriter.ts
-var fs20 = __toESM(require("fs"));
+var fs19 = __toESM(require("fs"));
 var import_readline7 = __toESM(require("readline"));
 var ExecutorWriter = class {
   constructor() {
@@ -19603,11 +19571,11 @@ var ExecutorWriter = class {
     };
     this.splitBySize = async (scope, sourcePath) => {
       const maxOutputFileSize = scope.limitFileSize * this.FAKE_GB;
-      const readStream = fs20.createReadStream(sourcePath);
+      const readStream = fs19.createReadStream(sourcePath);
       const reader = import_readline7.default.createInterface({ input: readStream, crlfDelay: Infinity });
       let writerIndex = 0;
       let destPath = this.getCompletedPath(sourcePath, writerIndex);
-      let writeStream = fs20.createWriteStream(destPath, { flags: "a" });
+      let writeStream = fs19.createWriteStream(destPath, { flags: "a" });
       for await (const line of reader) {
         if (readStream.bytesRead > maxOutputFileSize * (writerIndex + 1)) {
           writeStream.end();
@@ -19617,7 +19585,7 @@ var ExecutorWriter = class {
           });
           writerIndex++;
           destPath = this.getCompletedPath(sourcePath, writerIndex);
-          writeStream = fs20.createWriteStream(destPath, { flags: "a" });
+          writeStream = fs19.createWriteStream(destPath, { flags: "a" });
         }
         writeStream.write(line + "\n");
       }
@@ -19626,7 +19594,7 @@ var ExecutorWriter = class {
         writeStream.on("finish", resolve);
         writeStream.on("error", reject);
       });
-      await fs20.promises.unlink(sourcePath);
+      await fs19.promises.unlink(sourcePath);
     };
     /**
      * Manage the Writestream for main.dataset

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@forzalabs/remora",
-    "version": "1.2.2",
+    "version": "1.2.3",
     "description": "A powerful CLI tool for seamless data translation.",
     "main": "index.js",
     "private": false,

package/workers/ExecutorWorker.js

@@ -146,7 +146,7 @@ var require_has_flag = __commonJS({
 var require_supports_colors = __commonJS({
   "../../node_modules/logform/node_modules/@colors/colors/lib/system/supports-colors.js"(exports2, module2) {
     "use strict";
-    var os3 = require("os");
+    var os2 = require("os");
     var hasFlag = require_has_flag();
     var env = process.env;
     var forceColor = void 0;
@@ -184,7 +184,7 @@ var require_supports_colors = __commonJS({
       }
       var min = forceColor ? 1 : 0;
       if (process.platform === "win32") {
-        var osRelease = os3.release().split(".");
+        var osRelease = os2.release().split(".");
         if (Number(process.versions.node.split(".")[0]) >= 8 && Number(osRelease[0]) >= 10 && Number(osRelease[2]) >= 10586) {
           return Number(osRelease[2]) >= 14931 ? 3 : 2;
         }
@@ -5428,7 +5428,7 @@ var require_winston_transport = __commonJS({
 var require_console = __commonJS({
   "../../packages/logger/node_modules/winston/lib/winston/transports/console.js"(exports2, module2) {
     "use strict";
-    var os3 = require("os");
+    var os2 = require("os");
     var { LEVEL, MESSAGE } = require_triple_beam();
     var TransportStream = require_winston_transport();
     module2.exports = class Console extends TransportStream {
@@ -5442,7 +5442,7 @@ var require_console = __commonJS({
         this.name = options.name || "console";
         this.stderrLevels = this._stringArrayToSet(options.stderrLevels);
         this.consoleWarnLevels = this._stringArrayToSet(options.consoleWarnLevels);
-        this.eol = typeof options.eol === "string" ? options.eol : os3.EOL;
+        this.eol = typeof options.eol === "string" ? options.eol : os2.EOL;
         this.forceConsole = options.forceConsole || false;
         this._consoleLog = console.log.bind(console);
         this._consoleWarn = console.warn.bind(console);
@@ -10292,7 +10292,7 @@ var require_node2 = __commonJS({
 var require_tail_file = __commonJS({
   "../../packages/logger/node_modules/winston/lib/winston/tail-file.js"(exports2, module2) {
     "use strict";
-    var fs19 = require("fs");
+    var fs18 = require("fs");
     var { StringDecoder } = require("string_decoder");
     var { Stream } = require_readable();
     function noop() {
@@ -10313,7 +10313,7 @@ var require_tail_file = __commonJS({
         stream.emit("end");
         stream.emit("close");
       };
-      fs19.open(options.file, "a+", "0644", (err, fd) => {
+      fs18.open(options.file, "a+", "0644", (err, fd) => {
         if (err) {
           if (!iter) {
             stream.emit("error", err);
@@ -10325,10 +10325,10 @@ var require_tail_file = __commonJS({
         }
         (function read() {
           if (stream.destroyed) {
-            fs19.close(fd, noop);
+            fs18.close(fd, noop);
             return;
           }
-          return fs19.read(fd, buffer, 0, buffer.length, pos, (error, bytes) => {
+          return fs18.read(fd, buffer, 0, buffer.length, pos, (error, bytes) => {
             if (error) {
               if (!iter) {
                 stream.emit("error", error);
@@ -10387,7 +10387,7 @@ var require_tail_file = __commonJS({
 var require_file = __commonJS({
   "../../packages/logger/node_modules/winston/lib/winston/transports/file.js"(exports2, module2) {
     "use strict";
-    var fs19 = require("fs");
+    var fs18 = require("fs");
     var path18 = require("path");
     var asyncSeries = require_series();
     var zlib2 = require("zlib");
@@ -10395,7 +10395,7 @@ var require_file = __commonJS({
     var { Stream, PassThrough } = require_readable();
     var TransportStream = require_winston_transport();
     var debug = require_node2()("winston:file");
-    var os3 = require("os");
+    var os2 = require("os");
     var tailFile = require_tail_file();
     module2.exports = class File extends TransportStream {
       /**
@@ -10433,7 +10433,7 @@ var require_file = __commonJS({
         this.rotationFormat = options.rotationFormat || false;
         this.zippedArchive = options.zippedArchive || false;
         this.maxFiles = options.maxFiles || null;
-        this.eol = typeof options.eol === "string" ? options.eol : os3.EOL;
+        this.eol = typeof options.eol === "string" ? options.eol : os2.EOL;
         this.tailable = options.tailable || false;
         this.lazy = options.lazy || false;
         this._size = 0;
@@ -10592,7 +10592,7 @@ var require_file = __commonJS({
         let buff = "";
         let results = [];
         let row = 0;
-        const stream = fs19.createReadStream(file, {
+        const stream = fs18.createReadStream(file, {
           encoding: "utf8"
         });
         stream.on("error", (err) => {
@@ -10744,7 +10744,7 @@ var require_file = __commonJS({
       stat(callback) {
         const target = this._getFile();
         const fullpath = path18.join(this.dirname, target);
-        fs19.stat(fullpath, (err, stat) => {
+        fs18.stat(fullpath, (err, stat) => {
           if (err && err.code === "ENOENT") {
             debug("ENOENT\xA0ok", fullpath);
             this.filename = target;
@@ -10849,7 +10849,7 @@ var require_file = __commonJS({
       _createStream(source) {
         const fullpath = path18.join(this.dirname, this.filename);
         debug("create stream start", fullpath, this.options);
-        const dest = fs19.createWriteStream(fullpath, this.options).on("error", (err) => debug(err)).on("close", () => debug("close", dest.path, dest.bytesWritten)).on("open", () => {
+        const dest = fs18.createWriteStream(fullpath, this.options).on("error", (err) => debug(err)).on("close", () => debug("close", dest.path, dest.bytesWritten)).on("open", () => {
           debug("file open ok", fullpath);
           this.emit("open", fullpath);
           source.pipe(dest);
@@ -10928,7 +10928,7 @@ var require_file = __commonJS({
         const isZipped = this.zippedArchive ? ".gz" : "";
         const filePath = `${basename}${isOldest}${ext}${isZipped}`;
         const target = path18.join(this.dirname, filePath);
-        fs19.unlink(target, callback);
+        fs18.unlink(target, callback);
       }
       /**
        * Roll files forward based on integer, up to maxFiles. e.g. if base if
@@ -10951,17 +10951,17 @@ var require_file = __commonJS({
           tasks.push(function(i, cb) {
             let fileName = `${basename}${i - 1}${ext}${isZipped}`;
             const tmppath = path18.join(this.dirname, fileName);
-            fs19.exists(tmppath, (exists) => {
+            fs18.exists(tmppath, (exists) => {
               if (!exists) {
                 return cb(null);
               }
               fileName = `${basename}${i}${ext}${isZipped}`;
-              fs19.rename(tmppath, path18.join(this.dirname, fileName), cb);
+              fs18.rename(tmppath, path18.join(this.dirname, fileName), cb);
             });
           }.bind(this, x));
         }
         asyncSeries(tasks, () => {
-          fs19.rename(
+          fs18.rename(
             path18.join(this.dirname, `${basename}${ext}${isZipped}`),
             path18.join(this.dirname, `${basename}1${ext}${isZipped}`),
             callback
@@ -10977,22 +10977,22 @@ var require_file = __commonJS({
        * @private
        */
       _compressFile(src, dest, callback) {
-        fs19.access(src, fs19.F_OK, (err) => {
+        fs18.access(src, fs18.F_OK, (err) => {
           if (err) {
             return callback();
           }
           var gzip = zlib2.createGzip();
-          var inp = fs19.createReadStream(src);
-          var out = fs19.createWriteStream(dest);
+          var inp = fs18.createReadStream(src);
+          var out = fs18.createWriteStream(dest);
           out.on("finish", () => {
-            fs19.unlink(src, callback);
+            fs18.unlink(src, callback);
           });
           inp.pipe(gzip).pipe(out);
         });
       }
       _createLogDirIfNotExist(dirPath) {
-        if (!fs19.existsSync(dirPath)) {
-          fs19.mkdirSync(dirPath, { recursive: true });
+        if (!fs18.existsSync(dirPath)) {
+          fs18.mkdirSync(dirPath, { recursive: true });
         }
       }
     };
@@ -11236,7 +11236,7 @@ var require_stream3 = __commonJS({
     "use strict";
     var isStream = require_is_stream();
     var { MESSAGE } = require_triple_beam();
-    var os3 = require("os");
+    var os2 = require("os");
     var TransportStream = require_winston_transport();
     module2.exports = class Stream extends TransportStream {
       /**
@@ -11252,7 +11252,7 @@ var require_stream3 = __commonJS({
         this._stream = options.stream;
         this._stream.setMaxListeners(Infinity);
         this.isObjectMode = options.stream._writableState.objectMode;
-        this.eol = typeof options.eol === "string" ? options.eol : os3.EOL;
+        this.eol = typeof options.eol === "string" ? options.eol : os2.EOL;
       }
       /**
        * Core logging method exposed to Winston.
@@ -11635,7 +11635,7 @@ var require_exception_stream = __commonJS({
 var require_exception_handler = __commonJS({
   "../../packages/logger/node_modules/winston/lib/winston/exception-handler.js"(exports2, module2) {
     "use strict";
-    var os3 = require("os");
+    var os2 = require("os");
     var asyncForEach = require_forEach();
     var debug = require_node2()("winston:exception");
     var once = require_one_time();
@@ -11730,8 +11730,8 @@ var require_exception_handler = __commonJS({
        */
       getOsInfo() {
         return {
-          loadavg: os3.loadavg(),
-          uptime: os3.uptime()
+          loadavg: os2.loadavg(),
+          uptime: os2.uptime()
         };
       }
       /**
@@ -11873,7 +11873,7 @@ var require_rejection_stream = __commonJS({
 var require_rejection_handler = __commonJS({
   "../../packages/logger/node_modules/winston/lib/winston/rejection-handler.js"(exports2, module2) {
     "use strict";
-    var os3 = require("os");
+    var os2 = require("os");
     var asyncForEach = require_forEach();
     var debug = require_node2()("winston:rejection");
     var once = require_one_time();
@@ -11970,8 +11970,8 @@ var require_rejection_handler = __commonJS({
        */
       getOsInfo() {
         return {
-          loadavg: os3.loadavg(),
-          uptime: os3.uptime()
+          loadavg: os2.loadavg(),
+          uptime: os2.uptime()
         };
       }
       /**
@@ -14607,9 +14607,8 @@ var AutoMapperEngineClass = class {
 var AutoMapperEngine = new AutoMapperEngineClass();
 
 // ../../packages/engines/src/producer/ProducerEngine.ts
-var fs11 = __toESM(require("fs"), 1);
-var os = __toESM(require("os"), 1);
 var import_path11 = __toESM(require("path"), 1);
+var import_crypto2 = require("crypto");
 
 // ../../packages/drivers/src/DeltaShareDriver.ts
 var DeltaShareSourceDriver = class {
@@ -16596,20 +16595,30 @@ var ProducerEngineClass = class {
         case "aws-s3":
         case "delta-share": {
           const { compressionType } = producer.settings;
-          let tempDir = null;
+          let scope = null;
           try {
             let effectiveProducer = producer;
             let effectiveSource = source;
             if (compressionType) {
-              tempDir = fs11.mkdtempSync(import_path11.default.join(os.tmpdir(), "remora-sample-"));
-              const decompressedFileKey = await this._decompressForSampling(producer, source, tempDir);
-              effectiveProducer = { ...producer, settings: { ...producer.settings, fileKey: decompressedFileKey, compressionType: void 0 } };
-              effectiveSource = { ...source, authentication: { ...source.authentication, path: tempDir } };
+              scope = { id: (0, import_crypto2.randomUUID)(), folder: `sample-${(0, import_crypto2.randomUUID)()}`, workersId: [] };
+              const driver = await DriverFactory_default.instantiateSource(source);
+              const readyResult = await driver.ready({ producer, scope });
+              const firstFile = readyResult.files[0].fullUri;
+              const decompressedDir = import_path11.default.dirname(firstFile);
+              effectiveSource = { ...source, engine: "local", authentication: { ...source.authentication, path: decompressedDir } };
+              effectiveProducer = {
+                ...producer,
+                settings: {
+                  ...producer.settings,
+                  fileKey: readyResult.files.length > 1 ? "%.dataset" : import_path11.default.basename(firstFile),
+                  compressionType: void 0
+                }
+              };
             }
             const fileData = await this.readFile(effectiveProducer, { readmode: "lines", lines: { from: 0, to: sampleSize } }, effectiveSource);
             rawData = fileData.data;
           } finally {
-            if (tempDir) fs11.rmSync(tempDir, { recursive: true, force: true });
+            if (scope) await ExecutorScope_default.clearScope(scope);
           }
           break;
         }
@@ -16695,47 +16704,6 @@ var ProducerEngineClass = class {
       }
       return dimensions;
     };
-    this._decompressForSampling = async (producer, source, tempDir) => {
-      const { fileKey, compressionType } = producer.settings;
-      const sourcePath = source.authentication["path"];
-      switch (compressionType?.toUpperCase()) {
-        case "GZ": {
-          const decompressedName = fileKey.replace(/\.gz$/i, "");
-          const outputPath = import_path11.default.join(tempDir, decompressedName);
-          const outputDir = import_path11.default.dirname(outputPath);
-          if (!fs11.existsSync(outputDir)) fs11.mkdirSync(outputDir, { recursive: true });
-          await ParseCompression_default.decompressToFile(compressionType, fileKey, sourcePath, outputPath);
-          return decompressedName;
-        }
-        case "ZIP":
-        case "TAR": {
-          const extractDir = import_path11.default.join(tempDir, "extracted");
-          await ParseCompression_default.decompressToFile(compressionType, fileKey, sourcePath, extractDir);
-          const files = this._findFilesRecursive(extractDir);
-          Affirm_default(files.length > 0, `No files found after decompressing "${fileKey}"`);
-          const firstFile = files[0];
-          const relativePath = import_path11.default.relative(extractDir, firstFile);
-          const destPath = import_path11.default.join(tempDir, relativePath);
-          const destDir = import_path11.default.dirname(destPath);
-          if (!fs11.existsSync(destDir)) fs11.mkdirSync(destDir, { recursive: true });
-          fs11.renameSync(firstFile, destPath);
-          return relativePath;
-        }
-        default:
-          throw new Error(`Unsupported compression type "${compressionType}" for producer "${producer.name}"`);
-      }
-    };
-    this._findFilesRecursive = (dir) => {
-      const results = [];
-      for (const entry of fs11.readdirSync(dir)) {
-        const fullPath = import_path11.default.join(dir, entry);
-        if (fs11.statSync(fullPath).isDirectory())
-          results.push(...this._findFilesRecursive(fullPath));
-        else
-          results.push(fullPath);
-      }
-      return results;
-    };
   }
 };
 var ProducerEngine = new ProducerEngineClass();
@@ -18479,7 +18447,7 @@ var UsageManager = new UsageManagerClass();
 var UsageManager_default = UsageManager;
 
 // ../../packages/executors/src/OutputExecutor.ts
-var fs14 = __toESM(require("fs"));
+var fs13 = __toESM(require("fs"));
 
 // ../../packages/executors/src/ExecutorScope.ts
 var import_path13 = __toESM(require("path"));
@@ -18590,7 +18558,7 @@ var OutputExecutorClass = class {
         const currentPath = import_path14.default.dirname(ExecutorScope_default2.getMainPath(scope));
         const destinationName = this._composeFileName(consumer, output, this._getExtension(output));
         Logger_default.log(`Exporting consumer "${consumer.name}" to "${output.exportDestination}" as ${output.format} (${destinationName})`);
-        const filenameArray = fs14.readdirSync(currentPath).filter((filename) => filename.includes(".dataset"));
+        const filenameArray = fs13.readdirSync(currentPath).filter((filename) => filename.includes(".dataset"));
         for (const filename in filenameArray) {
           const destinationPath = this.getCompletedPath(destinationName, filename);
           const startingPath = import_path14.default.join(currentPath, filenameArray[filename]);
@@ -19345,7 +19313,7 @@ var ExecutorProgress = class {
 var ExecutorProgress_default = ExecutorProgress;
 
 // ../../packages/executors/src/ExecutorWriter.ts
-var fs17 = __toESM(require("fs"));
+var fs16 = __toESM(require("fs"));
 var import_readline8 = __toESM(require("readline"));
 var ExecutorWriter = class {
   constructor() {
@@ -19362,11 +19330,11 @@ var ExecutorWriter = class {
     };
     this.splitBySize = async (scope, sourcePath) => {
       const maxOutputFileSize = scope.limitFileSize * this.FAKE_GB;
-      const readStream = fs17.createReadStream(sourcePath);
+      const readStream = fs16.createReadStream(sourcePath);
       const reader = import_readline8.default.createInterface({ input: readStream, crlfDelay: Infinity });
       let writerIndex = 0;
       let destPath = this.getCompletedPath(sourcePath, writerIndex);
-      let writeStream = fs17.createWriteStream(destPath, { flags: "a" });
+      let writeStream = fs16.createWriteStream(destPath, { flags: "a" });
       for await (const line of reader) {
         if (readStream.bytesRead > maxOutputFileSize * (writerIndex + 1)) {
           writeStream.end();
@@ -19376,7 +19344,7 @@ var ExecutorWriter = class {
           });
           writerIndex++;
           destPath = this.getCompletedPath(sourcePath, writerIndex);
-          writeStream = fs17.createWriteStream(destPath, { flags: "a" });
+          writeStream = fs16.createWriteStream(destPath, { flags: "a" });
         }
         writeStream.write(line + "\n");
       }
@@ -19385,7 +19353,7 @@ var ExecutorWriter = class {
         writeStream.on("finish", resolve);
         writeStream.on("error", reject);
       });
-      await fs17.promises.unlink(sourcePath);
+      await fs16.promises.unlink(sourcePath);
     };
     /**
      * Manage the Writestream for main.dataset