@forzalabs/remora 1.1.15 → 1.2.2

package/workers/ExecutorWorker.js

@@ -146,7 +146,7 @@ var require_has_flag = __commonJS({
 var require_supports_colors = __commonJS({
   "../../node_modules/logform/node_modules/@colors/colors/lib/system/supports-colors.js"(exports2, module2) {
     "use strict";
-    var os2 = require("os");
+    var os3 = require("os");
     var hasFlag = require_has_flag();
     var env = process.env;
     var forceColor = void 0;
@@ -184,7 +184,7 @@ var require_supports_colors = __commonJS({
       }
       var min = forceColor ? 1 : 0;
       if (process.platform === "win32") {
-        var osRelease = os2.release().split(".");
+        var osRelease = os3.release().split(".");
         if (Number(process.versions.node.split(".")[0]) >= 8 && Number(osRelease[0]) >= 10 && Number(osRelease[2]) >= 10586) {
           return Number(osRelease[2]) >= 14931 ? 3 : 2;
         }
@@ -5428,7 +5428,7 @@ var require_winston_transport = __commonJS({
 var require_console = __commonJS({
   "../../packages/logger/node_modules/winston/lib/winston/transports/console.js"(exports2, module2) {
     "use strict";
-    var os2 = require("os");
+    var os3 = require("os");
     var { LEVEL, MESSAGE } = require_triple_beam();
     var TransportStream = require_winston_transport();
     module2.exports = class Console extends TransportStream {
@@ -5442,7 +5442,7 @@ var require_console = __commonJS({
         this.name = options.name || "console";
         this.stderrLevels = this._stringArrayToSet(options.stderrLevels);
         this.consoleWarnLevels = this._stringArrayToSet(options.consoleWarnLevels);
-        this.eol = typeof options.eol === "string" ? options.eol : os2.EOL;
+        this.eol = typeof options.eol === "string" ? options.eol : os3.EOL;
         this.forceConsole = options.forceConsole || false;
         this._consoleLog = console.log.bind(console);
         this._consoleWarn = console.warn.bind(console);
@@ -9654,15 +9654,15 @@ var require_index_cjs = __commonJS({
       };
     }
     function wrapConversion(toModel, graph) {
-      const path17 = [graph[toModel].parent, toModel];
+      const path18 = [graph[toModel].parent, toModel];
       let fn = convert$1[graph[toModel].parent][toModel];
       let cur = graph[toModel].parent;
       while (graph[cur].parent) {
-        path17.unshift(graph[cur].parent);
+        path18.unshift(graph[cur].parent);
         fn = link(convert$1[graph[cur].parent][cur], fn);
         cur = graph[cur].parent;
       }
-      fn.conversion = path17;
+      fn.conversion = path18;
       return fn;
     }
     function route(fromModel) {
@@ -10292,7 +10292,7 @@ var require_node2 = __commonJS({
 var require_tail_file = __commonJS({
   "../../packages/logger/node_modules/winston/lib/winston/tail-file.js"(exports2, module2) {
     "use strict";
-    var fs18 = require("fs");
+    var fs19 = require("fs");
     var { StringDecoder } = require("string_decoder");
     var { Stream } = require_readable();
     function noop() {
@@ -10313,7 +10313,7 @@ var require_tail_file = __commonJS({
         stream.emit("end");
         stream.emit("close");
       };
-      fs18.open(options.file, "a+", "0644", (err, fd) => {
+      fs19.open(options.file, "a+", "0644", (err, fd) => {
         if (err) {
           if (!iter) {
             stream.emit("error", err);
@@ -10325,10 +10325,10 @@ var require_tail_file = __commonJS({
         }
         (function read() {
           if (stream.destroyed) {
-            fs18.close(fd, noop);
+            fs19.close(fd, noop);
             return;
           }
-          return fs18.read(fd, buffer, 0, buffer.length, pos, (error, bytes) => {
+          return fs19.read(fd, buffer, 0, buffer.length, pos, (error, bytes) => {
             if (error) {
               if (!iter) {
                 stream.emit("error", error);
@@ -10387,15 +10387,15 @@ var require_tail_file = __commonJS({
 var require_file = __commonJS({
   "../../packages/logger/node_modules/winston/lib/winston/transports/file.js"(exports2, module2) {
     "use strict";
-    var fs18 = require("fs");
-    var path17 = require("path");
+    var fs19 = require("fs");
+    var path18 = require("path");
     var asyncSeries = require_series();
     var zlib2 = require("zlib");
     var { MESSAGE } = require_triple_beam();
     var { Stream, PassThrough } = require_readable();
     var TransportStream = require_winston_transport();
     var debug = require_node2()("winston:file");
-    var os2 = require("os");
+    var os3 = require("os");
     var tailFile = require_tail_file();
     module2.exports = class File extends TransportStream {
       /**
@@ -10418,14 +10418,14 @@ var require_file = __commonJS({
         this._onError = this._onError.bind(this);
         if (options.filename || options.dirname) {
           throwIf("filename or dirname", "stream");
-          this._basename = this.filename = options.filename ? path17.basename(options.filename) : "winston.log";
-          this.dirname = options.dirname || path17.dirname(options.filename);
+          this._basename = this.filename = options.filename ? path18.basename(options.filename) : "winston.log";
+          this.dirname = options.dirname || path18.dirname(options.filename);
           this.options = options.options || { flags: "a" };
         } else if (options.stream) {
           console.warn("options.stream will be removed in winston@4. Use winston.transports.Stream");
           throwIf("stream", "filename", "maxsize");
           this._dest = this._stream.pipe(this._setupStream(options.stream));
-          this.dirname = path17.dirname(this._dest.path);
+          this.dirname = path18.dirname(this._dest.path);
         } else {
           throw new Error("Cannot log to file without filename or stream.");
         }
@@ -10433,7 +10433,7 @@ var require_file = __commonJS({
         this.rotationFormat = options.rotationFormat || false;
         this.zippedArchive = options.zippedArchive || false;
         this.maxFiles = options.maxFiles || null;
-        this.eol = typeof options.eol === "string" ? options.eol : os2.EOL;
+        this.eol = typeof options.eol === "string" ? options.eol : os3.EOL;
         this.tailable = options.tailable || false;
         this.lazy = options.lazy || false;
         this._size = 0;
@@ -10588,11 +10588,11 @@ var require_file = __commonJS({
           options = {};
         }
         options = normalizeQuery(options);
-        const file = path17.join(this.dirname, this.filename);
+        const file = path18.join(this.dirname, this.filename);
         let buff = "";
         let results = [];
         let row = 0;
-        const stream = fs18.createReadStream(file, {
+        const stream = fs19.createReadStream(file, {
           encoding: "utf8"
         });
         stream.on("error", (err) => {
@@ -10693,7 +10693,7 @@ var require_file = __commonJS({
        * TODO: Refactor me.
        */
       stream(options = {}) {
-        const file = path17.join(this.dirname, this.filename);
+        const file = path18.join(this.dirname, this.filename);
         const stream = new Stream();
         const tail = {
           file,
@@ -10743,8 +10743,8 @@ var require_file = __commonJS({
        */
       stat(callback) {
         const target = this._getFile();
-        const fullpath = path17.join(this.dirname, target);
-        fs18.stat(fullpath, (err, stat) => {
+        const fullpath = path18.join(this.dirname, target);
+        fs19.stat(fullpath, (err, stat) => {
           if (err && err.code === "ENOENT") {
             debug("ENOENT\xA0ok", fullpath);
             this.filename = target;
@@ -10847,9 +10847,9 @@ var require_file = __commonJS({
        * @returns {WritableStream} Stream that writes to disk for the active file.
        */
       _createStream(source) {
-        const fullpath = path17.join(this.dirname, this.filename);
+        const fullpath = path18.join(this.dirname, this.filename);
         debug("create stream start", fullpath, this.options);
-        const dest = fs18.createWriteStream(fullpath, this.options).on("error", (err) => debug(err)).on("close", () => debug("close", dest.path, dest.bytesWritten)).on("open", () => {
+        const dest = fs19.createWriteStream(fullpath, this.options).on("error", (err) => debug(err)).on("close", () => debug("close", dest.path, dest.bytesWritten)).on("open", () => {
           debug("file open ok", fullpath);
           this.emit("open", fullpath);
           source.pipe(dest);
@@ -10872,16 +10872,16 @@ var require_file = __commonJS({
        */
       _incFile(callback) {
         debug("_incFile", this.filename);
-        const ext = path17.extname(this._basename);
-        const basename = path17.basename(this._basename, ext);
+        const ext = path18.extname(this._basename);
+        const basename = path18.basename(this._basename, ext);
         const tasks = [];
         if (this.zippedArchive) {
           tasks.push(
             function(cb) {
               const num = this._created > 0 && !this.tailable ? this._created : "";
               this._compressFile(
-                path17.join(this.dirname, `${basename}${num}${ext}`),
-                path17.join(this.dirname, `${basename}${num}${ext}.gz`),
+                path18.join(this.dirname, `${basename}${num}${ext}`),
+                path18.join(this.dirname, `${basename}${num}${ext}.gz`),
                 cb
               );
             }.bind(this)
@@ -10906,8 +10906,8 @@ var require_file = __commonJS({
        * @private
        */
       _getFile() {
-        const ext = path17.extname(this._basename);
-        const basename = path17.basename(this._basename, ext);
+        const ext = path18.extname(this._basename);
+        const basename = path18.basename(this._basename, ext);
         const isRotation = this.rotationFormat ? this.rotationFormat() : this._created;
         return !this.tailable && this._created ? `${basename}${isRotation}${ext}` : `${basename}${ext}`;
       }
@@ -10927,8 +10927,8 @@ var require_file = __commonJS({
         const isOldest = oldest !== 0 ? oldest : "";
         const isZipped = this.zippedArchive ? ".gz" : "";
         const filePath = `${basename}${isOldest}${ext}${isZipped}`;
-        const target = path17.join(this.dirname, filePath);
-        fs18.unlink(target, callback);
+        const target = path18.join(this.dirname, filePath);
+        fs19.unlink(target, callback);
       }
       /**
        * Roll files forward based on integer, up to maxFiles. e.g. if base if
@@ -10950,20 +10950,20 @@ var require_file = __commonJS({
         for (let x = this.maxFiles - 1; x > 1; x--) {
           tasks.push(function(i, cb) {
             let fileName = `${basename}${i - 1}${ext}${isZipped}`;
-            const tmppath = path17.join(this.dirname, fileName);
-            fs18.exists(tmppath, (exists) => {
+            const tmppath = path18.join(this.dirname, fileName);
+            fs19.exists(tmppath, (exists) => {
               if (!exists) {
                 return cb(null);
               }
               fileName = `${basename}${i}${ext}${isZipped}`;
-              fs18.rename(tmppath, path17.join(this.dirname, fileName), cb);
+              fs19.rename(tmppath, path18.join(this.dirname, fileName), cb);
             });
           }.bind(this, x));
         }
         asyncSeries(tasks, () => {
-          fs18.rename(
-            path17.join(this.dirname, `${basename}${ext}${isZipped}`),
-            path17.join(this.dirname, `${basename}1${ext}${isZipped}`),
+          fs19.rename(
+            path18.join(this.dirname, `${basename}${ext}${isZipped}`),
+            path18.join(this.dirname, `${basename}1${ext}${isZipped}`),
             callback
           );
         });
@@ -10977,22 +10977,22 @@ var require_file = __commonJS({
        * @private
        */
       _compressFile(src, dest, callback) {
-        fs18.access(src, fs18.F_OK, (err) => {
+        fs19.access(src, fs19.F_OK, (err) => {
           if (err) {
             return callback();
           }
           var gzip = zlib2.createGzip();
-          var inp = fs18.createReadStream(src);
-          var out = fs18.createWriteStream(dest);
+          var inp = fs19.createReadStream(src);
+          var out = fs19.createWriteStream(dest);
           out.on("finish", () => {
-            fs18.unlink(src, callback);
+            fs19.unlink(src, callback);
           });
           inp.pipe(gzip).pipe(out);
         });
       }
       _createLogDirIfNotExist(dirPath) {
-        if (!fs18.existsSync(dirPath)) {
-          fs18.mkdirSync(dirPath, { recursive: true });
+        if (!fs19.existsSync(dirPath)) {
+          fs19.mkdirSync(dirPath, { recursive: true });
         }
       }
     };
@@ -11076,9 +11076,9 @@ var require_http = __commonJS({
         };
         const auth = options.params.auth || null;
         delete options.params.auth;
-        const path17 = options.params.path || null;
+        const path18 = options.params.path || null;
         delete options.params.path;
-        this._request(options, auth, path17, (err, res, body) => {
+        this._request(options, auth, path18, (err, res, body) => {
           if (res && res.statusCode !== 200) {
             err = new Error(`Invalid HTTP Status Code: ${res.statusCode}`);
           }
@@ -11106,12 +11106,12 @@ var require_http = __commonJS({
           method: "stream",
           params: options
         };
-        const path17 = options.params.path || null;
+        const path18 = options.params.path || null;
         delete options.params.path;
         const auth = options.params.auth || null;
         delete options.params.auth;
         let buff = "";
-        const req = this._request(options, auth, path17);
+        const req = this._request(options, auth, path18);
         stream.destroy = () => req.destroy();
         req.on("data", (data) => {
           data = (buff + data).split(/\n+/);
@@ -11137,14 +11137,14 @@ var require_http = __commonJS({
        * @param {string} path - request path
        * @param {function} callback - Continuation to respond to when complete.
        */
-      _request(options, auth, path17, callback) {
+      _request(options, auth, path18, callback) {
         options = options || {};
         auth = auth || this.auth;
-        path17 = path17 || this.path || "";
+        path18 = path18 || this.path || "";
         if (this.batch) {
-          this._doBatch(options, callback, auth, path17);
+          this._doBatch(options, callback, auth, path18);
         } else {
-          this._doRequest(options, callback, auth, path17);
+          this._doRequest(options, callback, auth, path18);
         }
       }
       /**
@@ -11154,18 +11154,18 @@ var require_http = __commonJS({
        * @param {Object?} auth - authentication options
        * @param {string} path - request path
        */
-      _doBatch(options, callback, auth, path17) {
+      _doBatch(options, callback, auth, path18) {
         this.batchOptions.push(options);
         if (this.batchOptions.length === 1) {
           const me = this;
           this.batchCallback = callback;
           this.batchTimeoutID = setTimeout(function() {
             me.batchTimeoutID = -1;
-            me._doBatchRequest(me.batchCallback, auth, path17);
+            me._doBatchRequest(me.batchCallback, auth, path18);
           }, this.batchInterval);
         }
         if (this.batchOptions.length === this.batchCount) {
-          this._doBatchRequest(this.batchCallback, auth, path17);
+          this._doBatchRequest(this.batchCallback, auth, path18);
         }
       }
       /**
@@ -11174,14 +11174,14 @@ var require_http = __commonJS({
        * @param {Object?} auth - authentication options
        * @param {string} path - request path
        */
-      _doBatchRequest(callback, auth, path17) {
+      _doBatchRequest(callback, auth, path18) {
         if (this.batchTimeoutID > 0) {
           clearTimeout(this.batchTimeoutID);
           this.batchTimeoutID = -1;
         }
         const batchOptionsCopy = this.batchOptions.slice();
         this.batchOptions = [];
-        this._doRequest(batchOptionsCopy, callback, auth, path17);
+        this._doRequest(batchOptionsCopy, callback, auth, path18);
       }
       /**
        * Make a request to a winstond server or any http server which can
@@ -11191,7 +11191,7 @@ var require_http = __commonJS({
        * @param {Object?} auth - authentication options
        * @param {string} path - request path
        */
-      _doRequest(options, callback, auth, path17) {
+      _doRequest(options, callback, auth, path18) {
         const headers = Object.assign({}, this.headers);
         if (auth && auth.bearer) {
           headers.Authorization = `Bearer ${auth.bearer}`;
@@ -11201,7 +11201,7 @@ var require_http = __commonJS({
           method: "POST",
           host: this.host,
           port: this.port,
-          path: `/${path17.replace(/^\//, "")}`,
+          path: `/${path18.replace(/^\//, "")}`,
           headers,
           auth: auth && auth.username && auth.password ? `${auth.username}:${auth.password}` : "",
           agent: this.agent
@@ -11236,7 +11236,7 @@ var require_stream3 = __commonJS({
     "use strict";
     var isStream = require_is_stream();
     var { MESSAGE } = require_triple_beam();
-    var os2 = require("os");
+    var os3 = require("os");
     var TransportStream = require_winston_transport();
     module2.exports = class Stream extends TransportStream {
       /**
@@ -11252,7 +11252,7 @@ var require_stream3 = __commonJS({
         this._stream = options.stream;
         this._stream.setMaxListeners(Infinity);
         this.isObjectMode = options.stream._writableState.objectMode;
-        this.eol = typeof options.eol === "string" ? options.eol : os2.EOL;
+        this.eol = typeof options.eol === "string" ? options.eol : os3.EOL;
       }
       /**
        * Core logging method exposed to Winston.
@@ -11635,7 +11635,7 @@ var require_exception_stream = __commonJS({
 var require_exception_handler = __commonJS({
   "../../packages/logger/node_modules/winston/lib/winston/exception-handler.js"(exports2, module2) {
     "use strict";
-    var os2 = require("os");
+    var os3 = require("os");
     var asyncForEach = require_forEach();
     var debug = require_node2()("winston:exception");
     var once = require_one_time();
@@ -11730,8 +11730,8 @@ var require_exception_handler = __commonJS({
        */
       getOsInfo() {
         return {
-          loadavg: os2.loadavg(),
-          uptime: os2.uptime()
+          loadavg: os3.loadavg(),
+          uptime: os3.uptime()
         };
       }
       /**
@@ -11873,7 +11873,7 @@ var require_rejection_stream = __commonJS({
 var require_rejection_handler = __commonJS({
   "../../packages/logger/node_modules/winston/lib/winston/rejection-handler.js"(exports2, module2) {
     "use strict";
-    var os2 = require("os");
+    var os3 = require("os");
     var asyncForEach = require_forEach();
     var debug = require_node2()("winston:rejection");
     var once = require_one_time();
@@ -11970,8 +11970,8 @@ var require_rejection_handler = __commonJS({
        */
       getOsInfo() {
         return {
-          loadavg: os2.loadavg(),
-          uptime: os2.uptime()
+          loadavg: os3.loadavg(),
+          uptime: os3.uptime()
         };
       }
       /**
@@ -13351,6 +13351,10 @@ var Logger = class {
       console.info(message);
       FileLogService_default.write("INFO", String(message));
     };
+    this.warn = (message) => {
+      console.warn(message);
+      FileLogService_default.write("WARN", String(message));
+    };
     this.flush = () => FileLogService_default.flush();
     this.close = () => FileLogService_default.close();
     this.error = (error) => {
@@ -13494,7 +13498,7 @@ var import_promises = __toESM(require("fs/promises"), 1);
 
 // ../../packages/constants/src/Constants.ts
 var CONSTANTS = {
-  cliVersion: "1.1.15",
+  cliVersion: "1.2.1",
   backendVersion: 1,
   backendPort: 5088,
   workerVersion: 2,
@@ -14217,7 +14221,7 @@ var Environment = new EnvironmentClass();
 var Environment_default = Environment;
 
 // ../../packages/executors/src/ConsumerExecutor.ts
-var import_path14 = __toESM(require("path"));
+var import_path15 = __toESM(require("path"));
 var import_fs9 = __toESM(require("fs"));
 var import_readline6 = __toESM(require("readline"));
 var import_promises8 = __toESM(require("fs/promises"));
@@ -14602,6 +14606,11 @@ var AutoMapperEngineClass = class {
 };
 var AutoMapperEngine = new AutoMapperEngineClass();
 
+// ../../packages/engines/src/producer/ProducerEngine.ts
+var fs11 = __toESM(require("fs"), 1);
+var os = __toESM(require("os"), 1);
+var import_path11 = __toESM(require("path"), 1);
+
 // ../../packages/drivers/src/DeltaShareDriver.ts
 var DeltaShareSourceDriver = class {
   constructor() {
@@ -15436,8 +15445,8 @@ var HttpClientClass = class {
         throw new Error(err.message ?? "An error occurred on GET.");
       }
     };
-    this.getWorkerUrl = (path17) => {
-      const cleanPath = path17.startsWith("/") ? path17.substring(1) : path17;
+    this.getWorkerUrl = (path18) => {
+      const cleanPath = path18.startsWith("/") ? path18.substring(1) : path18;
       return `${this.workerHost}/${cleanPath}`;
     };
     this.workerHost = ProcessENVManager_default.getEnvVariable("REMORA_WORKER_HOST") || "http://worker:5069";
@@ -16507,12 +16516,12 @@ var noopTracker = { _operations: {}, measure: () => {
 }, getOperations: () => ({}) };
 var ProducerEngineClass = class {
   constructor() {
-    this.readFile = async (producer, options) => {
+    this.readFile = async (producer, options, sourceOverride) => {
       Affirm_default(producer, "Invalid producer");
       Affirm_default(options, "Invalid options");
       if (options.readmode === "lines")
         Affirm_default(options.lines, "Invalid lines");
-      const source = Environment_default.getSource(producer.source);
+      const source = sourceOverride ?? Environment_default.getSource(producer.source);
       Affirm_default(source, `No source found for producer "${producer.name}" with name "${producer.source}"`);
       const driver = await DriverFactory_default.instantiateSource(source);
       Affirm_default(driver, `No driver found for producer "${producer.name}" with driver type "${source.engine}"`);
@@ -16586,8 +16595,22 @@ var ProducerEngineClass = class {
         case "local":
         case "aws-s3":
         case "delta-share": {
-          const fileData = await this.readFile(producer, { readmode: "lines", lines: { from: 0, to: sampleSize } });
-          rawData = fileData.data;
+          const { compressionType } = producer.settings;
+          let tempDir = null;
+          try {
+            let effectiveProducer = producer;
+            let effectiveSource = source;
+            if (compressionType) {
+              tempDir = fs11.mkdtempSync(import_path11.default.join(os.tmpdir(), "remora-sample-"));
+              const decompressedFileKey = await this._decompressForSampling(producer, source, tempDir);
+              effectiveProducer = { ...producer, settings: { ...producer.settings, fileKey: decompressedFileKey, compressionType: void 0 } };
+              effectiveSource = { ...source, authentication: { ...source.authentication, path: tempDir } };
+            }
+            const fileData = await this.readFile(effectiveProducer, { readmode: "lines", lines: { from: 0, to: sampleSize } }, effectiveSource);
+            rawData = fileData.data;
+          } finally {
+            if (tempDir) fs11.rmSync(tempDir, { recursive: true, force: true });
+          }
           break;
         }
         default:
@@ -16672,13 +16695,54 @@ var ProducerEngineClass = class {
       }
       return dimensions;
     };
+    this._decompressForSampling = async (producer, source, tempDir) => {
+      const { fileKey, compressionType } = producer.settings;
+      const sourcePath = source.authentication["path"];
+      switch (compressionType?.toUpperCase()) {
+        case "GZ": {
+          const decompressedName = fileKey.replace(/\.gz$/i, "");
+          const outputPath = import_path11.default.join(tempDir, decompressedName);
+          const outputDir = import_path11.default.dirname(outputPath);
+          if (!fs11.existsSync(outputDir)) fs11.mkdirSync(outputDir, { recursive: true });
+          await ParseCompression_default.decompressToFile(compressionType, fileKey, sourcePath, outputPath);
+          return decompressedName;
+        }
+        case "ZIP":
+        case "TAR": {
+          const extractDir = import_path11.default.join(tempDir, "extracted");
+          await ParseCompression_default.decompressToFile(compressionType, fileKey, sourcePath, extractDir);
+          const files = this._findFilesRecursive(extractDir);
+          Affirm_default(files.length > 0, `No files found after decompressing "${fileKey}"`);
+          const firstFile = files[0];
+          const relativePath = import_path11.default.relative(extractDir, firstFile);
+          const destPath = import_path11.default.join(tempDir, relativePath);
+          const destDir = import_path11.default.dirname(destPath);
+          if (!fs11.existsSync(destDir)) fs11.mkdirSync(destDir, { recursive: true });
+          fs11.renameSync(firstFile, destPath);
+          return relativePath;
+        }
+        default:
+          throw new Error(`Unsupported compression type "${compressionType}" for producer "${producer.name}"`);
+      }
+    };
+    this._findFilesRecursive = (dir) => {
+      const results = [];
+      for (const entry of fs11.readdirSync(dir)) {
+        const fullPath = import_path11.default.join(dir, entry);
+        if (fs11.statSync(fullPath).isDirectory())
+          results.push(...this._findFilesRecursive(fullPath));
+        else
+          results.push(fullPath);
+      }
+      return results;
+    };
   }
 };
 var ProducerEngine = new ProducerEngineClass();
 var ProducerEngine_default = ProducerEngine;
 
 // ../../packages/engines/src/ai/DeveloperEngine.ts
-var import_path11 = __toESM(require("path"), 1);
+var import_path12 = __toESM(require("path"), 1);
 var import_promises6 = __toESM(require("fs/promises"), 1);
 var import_dayjs2 = __toESM(require("dayjs"), 1);
 var import_customParseFormat2 = __toESM(require("dayjs/plugin/customParseFormat"), 1);
@@ -16705,7 +16769,7 @@ var DeveloperEngineClass = class {
         _version: producer._version ?? 1
       };
       mappedProducer["$schema"] = producer["$schema"];
-      const producerPath = import_path11.default.join(process.cwd(), "remora", "producers", `${producer.name}.json`);
+      const producerPath = import_path12.default.join(process.cwd(), "remora", "producers", `${producer.name}.json`);
       await import_promises6.default.writeFile(producerPath, JSON.stringify(mappedProducer, null, 4), "utf-8");
       return { producer: mappedProducer, fields: typeDefinitions };
     };
@@ -16964,8 +17028,8 @@ var DeveloperEngineClass = class {
       Affirm_default(fileType, "Producer must have a fileType setting for mock data generation");
       const mockRecords = this.generateMockRecords(producer.dimensions, records);
       const basePath = source.authentication.path || process.cwd();
-      const filePath = import_path11.default.join(basePath, fileKey.replace("%", "mock"));
-      await import_promises6.default.mkdir(import_path11.default.dirname(filePath), { recursive: true });
+      const filePath = import_path12.default.join(basePath, fileKey.replace("%", "mock"));
+      await import_promises6.default.mkdir(import_path12.default.dirname(filePath), { recursive: true });
       const content = this.formatMockData(mockRecords, fileType, delimiter);
       await import_promises6.default.writeFile(filePath, content, "utf-8");
       return { filePath, recordCount: records };
@@ -17812,6 +17876,112 @@ var TransformationEngineClass = class {
 var TransformationEngine = new TransformationEngineClass();
 var TransformationEngine_default = TransformationEngine;
 
+// ../../packages/engines/src/transform/DataValidationEngine.ts
+var DataValidationEngineClass = class {
+  constructor() {
+    this.applyValidations = (value, validations, fieldKey) => {
+      for (const validation of validations) {
+        const passed = this.evaluateRule(value, validation.rule);
+        if (!passed) {
+          return {
+            valid: false,
+            message: this.buildMessage(value, validation.rule, fieldKey),
+            onFail: validation.onFail
+          };
+        }
+      }
+      return { valid: true };
+    };
+    this.evaluateRule = (value, rule) => {
+      if ("required" in rule)
+        return Algo_default.hasVal(value);
+      if ("min" in rule) {
+        if (!Algo_default.hasVal(value)) return true;
+        const num = Number(value);
+        return !isNaN(num) && num >= rule.min;
+      }
+      if ("max" in rule) {
+        if (!Algo_default.hasVal(value)) return true;
+        const num = Number(value);
+        return !isNaN(num) && num <= rule.max;
+      }
+      if ("regex" in rule) {
+        if (!Algo_default.hasVal(value)) return true;
+        return new RegExp(rule.regex).test(String(value));
+      }
+      if ("min_length" in rule) {
+        if (!Algo_default.hasVal(value)) return true;
+        return String(value).length >= rule.min_length;
+      }
+      if ("max_length" in rule) {
+        if (!Algo_default.hasVal(value)) return true;
+        return String(value).length <= rule.max_length;
+      }
+      if ("in" in rule) {
+        return rule.in.includes(value);
+      }
+      if ("not_in" in rule) {
+        return !rule.not_in.includes(value);
+      }
+      return true;
+    };
+    this.buildMessage = (value, rule, fieldKey) => {
+      const preview = Algo_default.hasVal(value) ? JSON.stringify(value) : "null/undefined";
+      if ("required" in rule) return `Field "${fieldKey}" is required but got ${preview}`;
+      if ("min" in rule) return `Field "${fieldKey}" value ${preview} is below minimum ${rule.min}`;
+      if ("max" in rule) return `Field "${fieldKey}" value ${preview} exceeds maximum ${rule.max}`;
+      if ("regex" in rule) return `Field "${fieldKey}" value ${preview} does not match pattern "${rule.regex}"`;
+      if ("min_length" in rule) return `Field "${fieldKey}" value ${preview} is shorter than minimum length ${rule.min_length}`;
+      if ("max_length" in rule) return `Field "${fieldKey}" value ${preview} exceeds maximum length ${rule.max_length}`;
+      if ("in" in rule) return `Field "${fieldKey}" value ${preview} is not in the allowed values`;
+      if ("not_in" in rule) return `Field "${fieldKey}" value ${preview} is in the disallowed values`;
+      return `Field "${fieldKey}" failed validation`;
+    };
+    this.evaluateDatasetValidations = (validations, context) => {
+      const results = [];
+      for (const validation of validations) {
+        const result = this.evaluateDatasetRule(validation, context);
+        if (result) results.push(result);
+      }
+      return results;
+    };
+    this.extractUniqueFieldKeys = (validations) => {
+      return validations.filter((v) => "unique_fields" in v.rule).flatMap((v) => v.rule.unique_fields);
+    };
+    this.hasRule = (validations, ruleKey) => {
+      return validations.some((v) => ruleKey in v.rule);
+    };
+    this.evaluateDatasetRule = (validation, context) => {
+      const { rule, onFail } = validation;
+      const { rowCount, hasDuplicateRows, duplicateFields } = context;
+      if ("not_empty" in rule) {
+        if (rowCount === 0)
+          return { message: "Dataset is empty", onFail };
+      }
+      if ("min_rows" in rule) {
+        if (rowCount < rule.min_rows)
+          return { message: `Dataset has ${rowCount} rows, expected at least ${rule.min_rows}`, onFail };
+      }
+      if ("max_rows" in rule) {
+        if (rowCount > rule.max_rows)
+          return { message: `Dataset has ${rowCount} rows, expected at most ${rule.max_rows}`, onFail };
+      }
+      if ("no_duplicates" in rule) {
+        if (hasDuplicateRows)
+          return { message: "Dataset contains duplicate rows", onFail };
+      }
+      if ("unique_fields" in rule) {
+        const failedFields = rule.unique_fields.filter((f) => duplicateFields.includes(f));
+        if (failedFields.length > 0)
+          return { message: `Duplicate values found in field(s): ${failedFields.join(", ")}`, onFail };
+      }
+      return null;
+    };
+  }
+};
+var DataValidationEngine = new DataValidationEngineClass();
+var DataValidationEngine_default = DataValidationEngine;
+
 // ../../packages/engines/src/usage/DataframeManager.ts
 var DataframeManagerClass = class {
   fill(points, from, to, onlyLastValue, maintainLastValue) {
@@ -18309,10 +18479,10 @@ var UsageManager = new UsageManagerClass();
 var UsageManager_default = UsageManager;
 
 // ../../packages/executors/src/OutputExecutor.ts
-var fs13 = __toESM(require("fs"));
+var fs14 = __toESM(require("fs"));
 
 // ../../packages/executors/src/ExecutorScope.ts
-var import_path12 = __toESM(require("path"));
+var import_path13 = __toESM(require("path"));
 var import_fs8 = __toESM(require("fs"));
 var import_promises7 = __toESM(require("fs/promises"));
 var ExecutorScopeClass2 = class {
@@ -18320,7 +18490,7 @@ var ExecutorScopeClass2 = class {
     this.WORKERS_FOLDER = "workers";
     this.PRODUCERS_FOLDER = "producers";
     this.getWorkerPath = (scope, workerId) => {
-      return import_path12.default.join(
+      return import_path13.default.join(
         Constants_default.defaults.REMORA_PATH,
         Constants_default.defaults.PRODUCER_TEMP_FOLDER,
         // A specific execution sits entirely in this folder, so at the end it's safe to delete it entirely
@@ -18330,7 +18500,7 @@ var ExecutorScopeClass2 = class {
       );
     };
     this.getProducerPath = (scope, producer, sourceFileKey) => {
-      return import_path12.default.join(
+      return import_path13.default.join(
         Constants_default.defaults.REMORA_PATH,
         Constants_default.defaults.PRODUCER_TEMP_FOLDER,
         // A specific execution sits entirely in this folder, so at the end it's safe to delete it entirely
@@ -18341,7 +18511,7 @@ var ExecutorScopeClass2 = class {
       );
     };
     this.getMainPath = (scope) => {
-      return import_path12.default.join(
+      return import_path13.default.join(
         Constants_default.defaults.REMORA_PATH,
         Constants_default.defaults.PRODUCER_TEMP_FOLDER,
         scope.folder,
@@ -18349,7 +18519,7 @@ var ExecutorScopeClass2 = class {
       );
     };
     this.clearScope = async (scope) => {
-      const scopePath = import_path12.default.join(
+      const scopePath = import_path13.default.join(
         Constants_default.defaults.REMORA_PATH,
         Constants_default.defaults.PRODUCER_TEMP_FOLDER,
         scope.folder
@@ -18359,7 +18529,7 @@ var ExecutorScopeClass2 = class {
       }
     };
     this.ensurePath = (fileUri) => {
-      const dir = import_path12.default.dirname(fileUri);
+      const dir = import_path13.default.dirname(fileUri);
       if (!import_fs8.default.existsSync(dir))
         import_fs8.default.mkdirSync(dir, { recursive: true });
       if (!import_fs8.default.existsSync(fileUri))
@@ -18371,7 +18541,7 @@ var ExecutorScope2 = new ExecutorScopeClass2();
 var ExecutorScope_default2 = ExecutorScope2;
 
 // ../../packages/executors/src/OutputExecutor.ts
-var import_path13 = __toESM(require("path"));
+var import_path14 = __toESM(require("path"));
 var OutputExecutorClass = class {
   constructor() {
     this._getInternalRecordFormat = (consumer) => {
@@ -18417,13 +18587,13 @@ var OutputExecutorClass = class {
       for (const output of consumer.outputs) {
         const destination = Environment_default.getSource(output.exportDestination);
         const driver = await DriverFactory_default.instantiateDestination(destination);
-        const currentPath = import_path13.default.dirname(ExecutorScope_default2.getMainPath(scope));
+        const currentPath = import_path14.default.dirname(ExecutorScope_default2.getMainPath(scope));
         const destinationName = this._composeFileName(consumer, output, this._getExtension(output));
         Logger_default.log(`Exporting consumer "${consumer.name}" to "${output.exportDestination}" as ${output.format} (${destinationName})`);
-        const filenameArray = fs13.readdirSync(currentPath).filter((filename) => filename.includes(".dataset"));
+        const filenameArray = fs14.readdirSync(currentPath).filter((filename) => filename.includes(".dataset"));
         for (const filename in filenameArray) {
           const destinationPath = this.getCompletedPath(destinationName, filename);
-          const startingPath = import_path13.default.join(currentPath, filenameArray[filename]);
+          const startingPath = import_path14.default.join(currentPath, filenameArray[filename]);
           if (output.format === internalFormat) {
             results.push(await driver.move(startingPath, destinationPath));
           } else {
@@ -18483,8 +18653,8 @@ var OutputExecutor_default = OutputExecutor;
 var ConsumerExecutorClass = class {
   constructor() {
     this._getWorkPath = (consumer, executionId) => {
-      const execFolder = import_path14.default.join(consumer.name, executionId);
-      const workPath = import_path14.default.join("./remora", Constants_default.defaults.PRODUCER_TEMP_FOLDER, execFolder, ".dataset");
+      const execFolder = import_path15.default.join(consumer.name, executionId);
+      const workPath = import_path15.default.join("./remora", Constants_default.defaults.PRODUCER_TEMP_FOLDER, execFolder, ".dataset");
       return workPath;
     };
     this._clearWorkPath = async (workPath) => {
@@ -18495,7 +18665,7 @@ var ConsumerExecutorClass = class {
       } catch (error) {
       }
       try {
-        const dir = import_path14.default.dirname(workPath);
+        const dir = import_path15.default.dirname(workPath);
         if (import_fs9.default.existsSync(dir)) {
           await import_promises8.default.rmdir(dir);
         }
@@ -18503,7 +18673,7 @@ var ConsumerExecutorClass = class {
       }
     };
     this._ensurePath = (pathUri) => {
-      const dir = import_path14.default.dirname(pathUri);
+      const dir = import_path15.default.dirname(pathUri);
       if (!import_fs9.default.existsSync(dir))
         import_fs9.default.mkdirSync(dir, { recursive: true });
       if (!import_fs9.default.existsSync(pathUri))
@@ -18570,6 +18740,32 @@ var ConsumerExecutorClass = class {
           }
         }
       }
+      for (const field of fields) {
+        const { cField } = field;
+        const fieldKey = cField.alias ?? cField.key;
+        if (cField.validate && cField.validate.length > 0) {
+          const result = DataValidationEngine_default.applyValidations(record[fieldKey], cField.validate, fieldKey);
+          if (!result.valid) {
+            const errorMessage = `Validation failed for field "${fieldKey}" (index: ${recordIndex}): ${result.message}`;
+            switch (result.onFail) {
+              case "set_default":
+                record[fieldKey] = cField.default;
+                break;
+              case "skip":
+                return null;
+              case "warn":
+                Logger_default.warn(errorMessage);
+                break;
+              case "fail":
+              default: {
+                const err = new Error(errorMessage);
+                Logger_default.error(err);
+                throw err;
+              }
+            }
+          }
+        }
+      }
       try {
         for (const dimension of dimensions) {
           const field = fields.find((x) => x.cField.key === dimension.name);
@@ -18810,6 +19006,48 @@ var ConsumerExecutorClass = class {
           return false;
       }
     };
+    this.processDatasetValidation = async (consumer, datasetPath) => {
+      const validations = consumer.validate;
+      if (!validations || validations.length === 0) return [];
+      const internalRecordFormat = OutputExecutor_default._getInternalRecordFormat(consumer);
+      const internalFields = ConsumerManager_default.getExpandedFields(consumer);
+      let rowCount = 0;
+      const seenRows = /* @__PURE__ */ new Set();
+      const fieldValueSets = /* @__PURE__ */ new Map();
+      let hasDuplicateRows = false;
+      const duplicateFields = [];
+      const uniqueFieldKeys = DataValidationEngine_default.extractUniqueFieldKeys(validations);
+      const checkDuplicateRows = DataValidationEngine_default.hasRule(validations, "no_duplicates");
+      for (const fieldKey of uniqueFieldKeys) {
+        fieldValueSets.set(fieldKey, /* @__PURE__ */ new Set());
+      }
+      const reader = import_fs9.default.createReadStream(datasetPath);
+      const lineReader = import_readline6.default.createInterface({ input: reader, crlfDelay: Infinity });
+      for await (const line of lineReader) {
+        rowCount++;
+        if (checkDuplicateRows) {
+          if (seenRows.has(line))
+            hasDuplicateRows = true;
+          else
+            seenRows.add(line);
+        }
+        if (uniqueFieldKeys.length > 0) {
+          const record = internalRecordFormat === "CSV" || internalRecordFormat === "TXT" ? LineParser_default._internalParseCSV(line, internalFields) : LineParser_default._internalParseJSON(line);
+          for (const fieldKey of uniqueFieldKeys) {
+            const valueSet = fieldValueSets.get(fieldKey);
+            const val = String(record[fieldKey] ?? "");
+            if (valueSet.has(val)) {
+              if (!duplicateFields.includes(fieldKey))
+                duplicateFields.push(fieldKey);
+            } else {
+              valueSet.add(val);
+            }
+          }
+        }
+      }
+      lineReader.close();
+      return DataValidationEngine_default.evaluateDatasetValidations(validations, { rowCount, hasDuplicateRows, duplicateFields });
+    };
     /**
      * Compares two values, handling numbers, strings, and dates
      * Returns: negative if a < b, positive if a > b, 0 if equal
@@ -18835,7 +19073,7 @@ var import_fs10 = __toESM(require("fs"));
 var import_readline7 = __toESM(require("readline"));
 
 // ../../packages/executors/src/ProducerExecutor.ts
-var import_path15 = __toESM(require("path"));
+var import_path16 = __toESM(require("path"));
 var ProducerExecutorClass = class {
   constructor() {
     this.ready = async (producer, scope) => {
@@ -18859,7 +19097,7 @@ var ProducerExecutorClass = class {
       counter = performance.now();
       for (const dimension of dimensions) {
         if (dimension.prodDimension.sourceFilename === true)
-          record[dimension.name] = import_path15.default.basename(chunk.fileUri);
+          record[dimension.name] = import_path16.default.basename(chunk.fileUri);
         const maskType = ProducerManager_default.getMask(dimension.prodDimension);
         if (Algo_default.hasVal(maskType))
           record[dimension.name] = CryptoEngine_default.hashValue(maskType, record[dimension.name]?.toString(), dimension.prodDimension.type);
@@ -19049,7 +19287,7 @@ var Executor_default = Executor;
 var import_os = __toESM(require("os"));
 var import_fs11 = __toESM(require("fs"));
 var import_promises9 = __toESM(require("fs/promises"));
-var import_path16 = __toESM(require("path"));
+var import_path17 = __toESM(require("path"));
 var import_workerpool = __toESM(require("workerpool"));
 
 // ../../packages/executors/src/ExecutorProgress.ts
@@ -19107,7 +19345,7 @@ var ExecutorProgress = class {
 var ExecutorProgress_default = ExecutorProgress;
 
 // ../../packages/executors/src/ExecutorWriter.ts
-var fs16 = __toESM(require("fs"));
+var fs17 = __toESM(require("fs"));
 var import_readline8 = __toESM(require("readline"));
 var ExecutorWriter = class {
   constructor() {
@@ -19124,11 +19362,11 @@ var ExecutorWriter = class {
     };
     this.splitBySize = async (scope, sourcePath) => {
       const maxOutputFileSize = scope.limitFileSize * this.FAKE_GB;
-      const readStream = fs16.createReadStream(sourcePath);
+      const readStream = fs17.createReadStream(sourcePath);
       const reader = import_readline8.default.createInterface({ input: readStream, crlfDelay: Infinity });
       let writerIndex = 0;
       let destPath = this.getCompletedPath(sourcePath, writerIndex);
-      let writeStream = fs16.createWriteStream(destPath, { flags: "a" });
+      let writeStream = fs17.createWriteStream(destPath, { flags: "a" });
       for await (const line of reader) {
         if (readStream.bytesRead > maxOutputFileSize * (writerIndex + 1)) {
           writeStream.end();
@@ -19138,7 +19376,7 @@ var ExecutorWriter = class {
           });
           writerIndex++;
           destPath = this.getCompletedPath(sourcePath, writerIndex);
-          writeStream = fs16.createWriteStream(destPath, { flags: "a" });
+          writeStream = fs17.createWriteStream(destPath, { flags: "a" });
         }
         writeStream.write(line + "\n");
       }
@@ -19147,7 +19385,7 @@ var ExecutorWriter = class {
         writeStream.on("finish", resolve);
         writeStream.on("error", reject);
       });
-      await fs16.promises.unlink(sourcePath);
+      await fs17.promises.unlink(sourcePath);
     };
     /**
      * Manage the Writestream for main.dataset
@@ -19217,7 +19455,7 @@ var ExecutorOrchestratorClass = class {
       };
       const workerPath = this._getWorkerPath();
       Logger_default.log(`Initializing worker pool from ${workerPath} (heap limit: ${Constants_default.defaults.MIN_RUNTIME_HEAP_MB}MB)`);
-      return import_workerpool.default.pool(import_path16.default.join(workerPath, "ExecutorWorker.js"), options);
+      return import_workerpool.default.pool(import_path17.default.join(workerPath, "ExecutorWorker.js"), options);
     };
     this.launch = async (request) => {
       Affirm_default(request, "Invalid options");
@@ -19227,7 +19465,7 @@ var ExecutorOrchestratorClass = class {
       const tracker = new ExecutorPerformance_default();
       const _progress = new ExecutorProgress_default(logProgress);
       const { usageId } = UsageManager_default.startUsage(consumer, details);
-      const scope = { id: usageId, folder: `${consumer.name}_${usageId}`, workersId: [], limitFileSize: consumer.MaximumFileSize };
+      const scope = { id: usageId, folder: `${consumer.name}_${usageId}`, workersId: [], limitFileSize: consumer.maximumFileSize };
       const pool = this.createPool();
       try {
         const start = performance.now();
@@ -19322,6 +19560,30 @@ var ExecutorOrchestratorClass = class {
           postOperation.totalOutputCount = unifiedOutputCount;
           Logger_default.log(`[${usageId}] Pivot complete: ${unifiedOutputCount} rows in ${Math.round(performance.now() - counter)}ms`);
         }
+        if (consumer.validate && consumer.validate.length > 0) {
+          Logger_default.log(`[${usageId}] Running dataset-level validations`);
+          counter = performance.now();
+          const validationResults = await ConsumerExecutor_default.processDatasetValidation(consumer, ExecutorScope_default2.getMainPath(scope));
+          tracker.measure("dataset-validation", performance.now() - counter);
+          for (const result of validationResults) {
+            if (result.onFail === "fail") {
+              const err = new Error(`Dataset validation failed for consumer "${consumer.name}": ${result.message}`);
+              Logger_default.error(err);
+              throw err;
+            } else if (result.onFail === "warn") {
+              Logger_default.warn(`Dataset validation warning for consumer "${consumer.name}": ${result.message}`);
+            }
+          }
+          Logger_default.log(`[${usageId}] Dataset validations complete in ${Math.round(performance.now() - counter)}ms`);
+        }
+        if (scope.limitFileSize) {
+          Logger_default.log(`[${usageId}] Splitting output by size limit (${scope.limitFileSize})`);
+          counter = performance.now();
+          const writer = new ExecutorWriter_default();
+          await writer.splitBySize(scope, ExecutorScope_default2.getMainPath(scope));
+          tracker.measure("split-by-size", performance.now() - counter);
+          Logger_default.log(`[${usageId}] Split complete in ${Math.round(performance.now() - counter)}ms`);
+        }
         counter = performance.now();
         Logger_default.log(`[${usageId}] Exporting results to ${consumer.outputs.length} output(s)`);
         const exportRes = await OutputExecutor_default.exportResult(consumer, ConsumerManager_default.getExpandedFields(consumer), scope);
@@ -19429,20 +19691,19 @@ var ExecutorOrchestratorClass = class {
     this._getWorkerPath = () => {
       const currentDir = __dirname;
       if (ProcessENVManager_default.getEnvVariable("NODE_ENV") === "dev" || ProcessENVManager_default.getEnvVariable("NODE_ENV") === "development")
-        return import_path16.default.resolve("./.build/workers");
+        return import_path17.default.resolve("./.build/workers");
       const forcedPath = ProcessENVManager_default.getEnvVariable("REMORA_WORKERS_PATH");
       if (forcedPath && forcedPath.length > 0)
-        return import_path16.default.join(__dirname, forcedPath);
+        return import_path17.default.join(__dirname, forcedPath);
       if (!currentDir.includes(".build")) {
-        return import_path16.default.join(__dirname, "../workers");
+        return import_path17.default.join(__dirname, "../workers");
       } else {
-        return import_path16.default.resolve("./.build/workers");
+        return import_path17.default.resolve("./.build/workers");
       }
     };
     this.reconcileExecutorThreadsResults = async (scope, executorResults, tracker) => {
       const mainPath = ExecutorScope_default2.getMainPath(scope);
       ConsumerExecutor_default._ensurePath(mainPath);
-      const writer = new ExecutorWriter_default();
       if (executorResults.length > 1) {
         Logger_default.log(`[${scope.id}] Merging ${executorResults.length} worker output files into ${mainPath}`);
         const perf = performance.now();
@@ -19458,10 +19719,6 @@ var ExecutorOrchestratorClass = class {
         Logger_default.log(`[${scope.id}] Single worker \u2014 renaming output to ${mainPath}`);
         await import_promises9.default.rename(executorResults[0].resultUri, mainPath);
       }
-      if (scope.limitFileSize) {
-        Logger_default.log(`[${scope.id}] Splitting output by size limit (${scope.limitFileSize})`);
-        await writer.splitBySize(scope, mainPath);
-      }
       return mainPath;
     };
     this.performCleanupOperations = async (scope, tracker) => {